raphiaura/KytyPS5
1
0
Fork 0
mirror of https://github.com/KytyPS5/KytyPS5.git synced 2026-10-19 18:36:53 +02:00
Code Issues Packages Projects Releases Wiki Activity
Files
ba8b0c85b80e74de092c410f625a496831d18ead
KytyPS5/tests/ShaderRecompilerComputeTests.cpp
T
Nawid3333 ba8b0c85b8 shader: keep quiet NaNs with payload bits out of the packed f16 min/max sNaN path (#1579) 
V_PK_MIN_F16 and V_PK_MAX_F16 return the quieted NaN for a signaling NaN
operand and the other operand for a quiet one (per half, as V_MIN_F16 and
V_MAX_F16 do). The signaling test masked the half with 0x7c00 and checked
for a nonzero payload, so a quiet NaN with payload bits such as 0x7e01 was
taken as signaling and returned. Mask with 0x7e00 so the quiet bit must be
clear.

Two more vectors in PackedMinMaxF16NanAndSignedZeroEdges cover it.

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-17 05:12:37 +02:00

49012 lines
2.2 MiB
Plaintext
Raw Blame History

#define SPV_ENABLE_UTILITY_CODE
#include "common/assert.h"
#include "common/emulatorConfig.h"
#include "common/file.h"
#include "common/hostException.h"
#include "common/logging/log.h"
#include "common/subsystems.h"
#include "common/threads.h"
#include "gpu_test_shaders/gpu_test_ms_depth_spv.h"
#include "graphics/guest_gpu/command_processor/commandProcessor.h"
#include "graphics/guest_gpu/command_processor/pm4Dispatch.h"
#include "graphics/guest_gpu/gpu_defs.h"
#include "graphics/guest_gpu/gpu_format.h"
#include "graphics/guest_gpu/graphicsRun.h"
#include "graphics/guest_gpu/hardwareContext.h"
#include "graphics/guest_gpu/pm4.h"
#include "graphics/guest_gpu/tile.h"
#include "graphics/host_gpu/hostMemory.h"
#include "graphics/host_gpu/memoryTracker.h"
#include "graphics/host_gpu/pageManager.h"
#include "graphics/host_gpu/renderer/cache/bufferCache.h"
#include "graphics/host_gpu/renderer/cache/textureCache.h"
#include "graphics/host_gpu/renderer/colorRenderTarget.h"
#include "graphics/host_gpu/renderer/depthRenderTarget.h"
#include "graphics/host_gpu/renderer/image/blitHelper.h"
#include "graphics/host_gpu/renderer/image/image.h"
#include "graphics/host_gpu/renderer/image/imageView.h"
#include "graphics/host_gpu/renderer/image/textureCommon.h"
#include "graphics/host_gpu/renderer/image/tiler.h"
#include "graphics/host_gpu/renderer/pipeline/descriptors.h"
#include "graphics/host_gpu/renderer/pipeline/pipelineCache.h"
#include "graphics/host_gpu/renderer/debug.h"
#include "graphics/host_gpu/renderer/render.h"
#include "graphics/host_gpu/renderer/renderContext.h"
#include "graphics/host_gpu/renderer/renderDraw.h"
#include "graphics/host_gpu/renderer/renderTarget.h"
#include "graphics/host_gpu/renderer/sync.h"
#include "graphics/host_gpu/vulkanCommon.h"
#include <vulkan/vulkan_format_traits.hpp>
#include "graphics/presentation/window/windowInternal.h"
#include "graphics/shader/recompiler/BufferFormat.h"
#include "graphics/shader/recompiler/ShaderRecompiler.h"
#include "graphics/shader/recompiler/Tessellation.h"
#include "graphics/shader/recompiler/backend/spirv/SpirvBuilder.h"
#include "graphics/shader/recompiler/backend/spirv/SpirvEmitter.h"
#include "graphics/shader/recompiler/frontend/decode/ShaderDecoder.h"
#include "graphics/shader/recompiler/ir/ShaderIR.h"
#include "graphics/shader/recompiler/ir/passes/DeadCodeElimination.h"
#include "graphics/shader/recompiler/ir/passes/ShaderInfoCollection.h"
#include "graphics/shader/rectListShader.h"
#include "graphics/shader/shader.h"
#include "graphics/shader/shaderCompiler.h"
#include "kernel/eventQueue.h"
#include "kernel/memory.h"
#include "libs/agc.h"
#include "libs/dialog.h"
#include "libs/errno.h"
#include "loader/symbolDatabase.h"
#include "spirv-tools/libspirv.hpp"
#if __has_include("graphics/host_gpu/renderer/renderTargetBarriers.h")
#error "legacy render-target barrier API must remain deleted"
#endif
#if __has_include("graphics/host_gpu/transfer.h")
#error "legacy transfer facade must remain deleted"
#endif
#if __has_include("graphics/host_gpu/renderer/framebufferCache.h")
#error "classic framebuffer/render-pass cache must remain deleted"
#endif
#include <algorithm>
#include <array>
#include <atomic>
#include <bit>
#include <chrono>
#include <cinttypes>
#include <cmath>
#include <cstdint>
#include <cstdio>
#include <cstdlib>
#include <cstring>
#include <initializer_list>
#include <limits>
#include <memory>
#include <mutex>
#include <numeric>
#include <optional>
#include <semaphore>
#include <set>
#include <span>
#include <sstream>
#include <string>
#include <string_view>
#include <tuple>
#include <vector>
#if KYTY_PLATFORM != KYTY_PLATFORM_WINDOWS
#include <cerrno>
#include <sys/wait.h>
#include <unistd.h>
#endif
#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
#ifndef NOMINMAX
#define NOMINMAX
#endif
#include <windows.h>
#undef min
#undef max
#endif
namespace Libs {
void InitAgcDriver_1(Loader::SymbolDatabase *symbols);
}
namespace Libs::Graphics {
template <typename Cache>
concept HasGetDownloadBuffer =
requires(Cache &cache) { cache.GetDownloadBuffer(uint64_t{1}); };
static_assert(!HasGetDownloadBuffer<BufferCache>);
static_assert(std::same_as<decltype(&BufferCache::GetBuffer),
Buffer &(BufferCache::*)(BufferId)>);
static_assert(std::same_as<decltype(&TextureCache::GetImage),
Image &(TextureCache::*)(ImageId)>);
template <typename Cache>
concept HasSynchronizeImageToBuffer = requires(Cache &cache) {
cache.SynchronizeImageToBuffer(uint64_t{1}, uint64_t{1});
};
template <typename Cache>
concept HasObtainBufferForImageCopy = requires(Cache &cache) {
cache.ObtainBufferForImageCopy(uint64_t{1}, uint64_t{1});
};
template <typename Cache>
concept HasObtainBufferForImageWrite = requires(Cache &cache) {
cache.ObtainBufferForImageWrite(uint64_t{1}, uint64_t{1});
};
template <typename Cache>
concept HasDiscardGpuDirtyBytes = requires(Cache &cache) {
cache.DiscardGpuDirtyBytes(uint64_t{1}, uint64_t{1});
};
static_assert(!HasSynchronizeImageToBuffer<TextureCache>);
static_assert(!HasObtainBufferForImageCopy<BufferCache>);
static_assert(!HasObtainBufferForImageWrite<BufferCache>);
static_assert(!HasDiscardGpuDirtyBytes<BufferCache>);
template <typename Backing>
concept HasLegacyImageLayout = requires(Backing &backing) { backing.layout; };
static_assert(!HasLegacyImageLayout<VulkanImage>);
template <typename Manager>
concept HasPolicyDetileImage = requires { &Manager::DetileImage; };
template <typename Manager>
concept HasPolicyTileImageBacking = requires { &Manager::TileImageBacking; };
static_assert(!HasPolicyDetileImage<TileManager>);
static_assert(!HasPolicyTileImageBacking<TileManager>);
static_assert(BlitHelper::ColorToMsDepthLayout ==
vk::ImageLayout::eDepthStencilAttachmentOptimal);
struct BufferCacheTestAccess {
static_assert(std::same_as<decltype(BufferCache::m_slot_buffers),
Common::SlotVector<Buffer>>);
static uint64_t GcTick(const BufferCache &cache) { return cache.m_gc_tick; }
static void SetGarbageCollectionThresholds(BufferCache &cache,
uint64_t trigger,
uint64_t critical) {
cache.m_trigger_gc_memory = trigger;
cache.m_critical_gc_memory = critical;
}
static StreamBuffer &DownloadBuffer(BufferCache &cache) {
return cache.m_download_buffer;
}
static void SetUnusedStagingBufferSize(BufferCache &cache, uint64_t size) {
// Called immediately after cache construction, before any staging reservation.
std::destroy_at(&cache.m_staging_buffer);
std::construct_at(&cache.m_staging_buffer, cache.m_graphics, cache.m_scheduler,
MemoryUsage::Upload, size);
}
static bool SynchronizeBufferFromImage(BufferCache &cache, Buffer &buffer,
uint64_t address, uint64_t size) {
return cache.SynchronizeBufferFromImage(buffer, address, size);
}
static BufferId PageOwner(const BufferCache &cache, uint64_t address) {
const auto *owner = cache.m_page_table.Find(
address >> BufferCache::PageTable::kPageBits);
return owner == nullptr ? BufferId{} : *owner;
}
static bool IsBufferAllocated(const BufferCache &cache, BufferId id) {
return cache.m_slot_buffers.is_allocated(id);
}
};
struct StreamBufferTestAccess {
static bool NormalizeReservation(bool coherent, uint64_t atom, uint64_t &size,
uint64_t &alignment) {
return StreamBuffer::NormalizeReservation(coherent, atom, size, alignment);
}
};
struct ImageTestAccess {
static uint32_t CopyRows(uint64_t row_size, uint32_t rows,
uint64_t capacity) {
return Image::CopyRows(row_size, rows, capacity);
}
};
struct TileManagerTestAccess {
static uint32_t ConversionRows(uint64_t offset, uint64_t row_stride,
uint64_t active, uint32_t remaining,
uint64_t alignment, uint64_t max_range,
uint32_t max_groups) {
return TileManager::ConversionRows(offset, row_stride, active, remaining,
alignment, max_range, max_groups);
}
};
struct TextureCacheTestAccess {
static_assert(std::same_as<decltype(TextureCache::m_slot_images),
Common::SlotVector<Image>>);
static_assert(TextureCache::ImagePageTable::kPageBits == 20);
static_assert(TextureCache::ImagePageTable::kAddressSpaceBits == 44);
static_assert(TextureCache::ImagePageTable::kFirstLevelBits == 14);
static uint64_t GcTick(const TextureCache &cache) { return cache.m_gc_tick; }
static std::unique_lock<TrackingSpinLock> Lock(TextureCache &cache) {
return std::unique_lock(cache.m_lock);
}
static void ClearImage(TextureCache &cache, CommandBuffer &command, ImageId id,
const vk::ImageSubresourceRange &range,
const vk::ClearValue &clear) {
auto lock = Lock(cache);
cache.ClearImage(command, id, cache.GetImage(id).backing.format, range, clear);
}
static void ConfigureGarbageCollection(TextureCache &cache,
std::span<const ImageId> oldest,
uint64_t tick, uint64_t pressure) {
cache.m_trigger_gc_memory = 0;
cache.m_pressure_gc_memory = pressure;
cache.m_critical_gc_memory = UINT64_MAX;
cache.m_gc_tick = tick;
std::vector<ImageId> live;
cache.m_lru_cache = {};
cache.m_slot_images.ForEach([&](ImageId id, Image &image) {
if (image.registered) {
image.tick_accessed_last = cache.m_scheduler.CurrentTick();
live.push_back(id);
}
});
for (const auto id : oldest) {
const auto owner = cache.m_slot_images.try_get(id);
if (owner != nullptr && owner->registered) {
owner->tick_accessed_last = 0;
owner->lru_id = cache.m_lru_cache.Insert(id, 0);
}
}
for (const auto id : live) {
if (std::ranges::find(oldest, id) == oldest.end()) {
cache.m_slot_images[id].lru_id = cache.m_lru_cache.Insert(id, tick);
}
}
}
static bool Contains(const TextureCache &cache, ImageId id) {
const auto owner = cache.m_slot_images.try_get(id);
return owner != nullptr && owner->registered;
}
static std::vector<ImageId> FindImages(TextureCache &cache, uint64_t address,
uint64_t size, bool page_overlap) {
std::lock_guard lock(cache.m_lock);
const auto found = cache.FindImagesInRegion(address, size, page_overlap);
std::vector<ImageId> result;
result.reserve(found.size());
for (const auto id : found) {
result.push_back(id);
}
return result;
}
static size_t PageOwnerCount(TextureCache &cache, uint64_t address) {
std::lock_guard lock(cache.m_lock);
const auto *owners = cache.m_image_page_table.Find(static_cast<size_t>(
address >> TextureCache::ImagePageTable::kPageBits));
return owners == nullptr ? 0 : owners->size();
}
static size_t OwnedPageCount(TextureCache &cache, uint64_t address,
uint64_t size, ImageId id) {
std::lock_guard lock(cache.m_lock);
TextureCache::ImagePageTable::PageRange pages{};
if (!TextureCache::ImagePageTable::TryGetPageRange(address, size, pages)) {
return 0;
}
size_t count = 0;
for (size_t page = pages.first; page < pages.last_exclusive; ++page) {
const auto *owners = cache.m_image_page_table.Find(page);
count += owners != nullptr && owners->Contains(id) ? 1 : 0;
}
return count;
}
static void AddPageOwner(TextureCache &cache, uint64_t address, ImageId id) {
std::lock_guard lock(cache.m_lock);
cache
.m_image_page_table[static_cast<size_t>(
address >> TextureCache::ImagePageTable::kPageBits)]
.push_back(id);
}
static bool RemovePageOwner(TextureCache &cache, uint64_t address,
ImageId id) {
std::lock_guard lock(cache.m_lock);
auto *owners = cache.m_image_page_table.Find(static_cast<size_t>(
address >> TextureCache::ImagePageTable::kPageBits));
return owners != nullptr && owners->Erase(id);
}
static void SetQueryEpoch(TextureCache &cache, uint32_t epoch) {
std::lock_guard lock(cache.m_lock);
cache.m_image_query_epoch = epoch;
}
static uint32_t QueryEpoch(TextureCache &cache) {
std::lock_guard lock(cache.m_lock);
return cache.m_image_query_epoch;
}
static ImageId InsertImage(TextureCache &cache, const ImageInfo &info) {
std::lock_guard lock(cache.m_lock);
return cache.InsertImage(info);
}
static void DeleteImage(TextureCache &cache, ImageId id) {
std::lock_guard lock(cache.m_lock);
cache.DeleteImage(id);
}
static const Image *Owner(const TextureCache &cache, ImageId id) {
return cache.m_slot_images.try_get(id);
}
static bool PendingDownload(const TextureCache &cache, ImageId id) {
return cache.m_download_images.contains(id);
}
static void TrackDownload(TextureCache &cache, ImageId id) {
std::lock_guard lock(cache.m_lock);
cache.TrackImageDownload(id, cache.m_slot_images[id]);
}
static void AssociateStencil(TextureCache &cache, ImageId depth,
GuestRange stencil) {
std::lock_guard lock(cache.m_lock);
(void)cache.AssociateStencil(depth, stencil);
}
static void SetLinearReadback(TextureCache &cache, bool enabled) {
cache.m_readback_linear_images = enabled;
}
static std::pair<uint8_t *, uint64_t>
MapDownload(TextureCache &cache, uint64_t size, uint64_t alignment) {
auto &download =
cache.m_buffer_cache.GetUtilityBuffer(MemoryUsage::Download);
auto mapping = download.Map(size, std::max<uint64_t>(alignment, 4));
if (mapping.first != nullptr) {
download.Commit();
}
return mapping;
}
static bool TryDownload(TextureCache &cache, ImageId id) {
return cache.DownloadImageMemory(id);
}
static void RegisterHtileMeta(TextureCache &cache, uint64_t address) {
std::lock_guard lock(cache.m_lock);
auto &metadata = cache.m_surface_metas[address];
metadata.type = TextureCache::MetaDataInfo::Type::HTile;
metadata.clear_mask.reset();
}
static TileManager &Tiler(TextureCache &cache) { return cache.m_tiler; }
};
struct RenderExecutorTestAccess {
static void DrawIndex(RenderExecutor &executor, CommandBuffer &command,
const DrawIndexArgs &args) {
executor.DrawIndex(0, command, args);
}
static void DrawAuto(RenderExecutor &executor, CommandBuffer &command,
const DrawAutoArgs &args) {
executor.DrawAuto(0, command, args);
}
static bool TryConsumeComputeImageClear(RenderExecutor &executor,
const ShaderComputeInputInfo &input, CommandBuffer &command,
uint32_t x, uint32_t y, uint32_t z, uint32_t mode) {
return executor.TryConsumeComputeImageClear(input, command, x, y, z, mode);
}
static void UpdateComputeMetaClears(RenderExecutor &executor,
const ShaderComputeInputInfo &input,
const CommandBuffer &buffer) {
executor.UpdateComputeMetaClears(input, buffer);
}
static TextureBinding
ResolveTexture(RenderExecutor &executor,
const ShaderRecompiler::IR::ImageResource &resource,
const ShaderRecompiler::IR::DescriptorValue &value) {
return executor.ResolveTexture(resource, value);
}
static auto PrepareGraphicsBindings(RenderExecutor &executor,
const ShaderStageRuntime &vertex,
const ShaderStageRuntime &pixel,
bool pixel_active) {
RenderExecutor::GraphicsBindings result;
executor.PrepareBindings(vertex, result.vertex[0]);
std::array<PreparedBindings *, 2> stages{&result.vertex[0], nullptr};
if (pixel_active) {
result.pixel.emplace();
executor.PrepareBindings(pixel, *result.pixel);
stages[1] = &*result.pixel;
}
executor.PrepareGraphicsBindings(
std::span{stages.data(), pixel_active ? 2u : 1u}, {});
return result;
}
static void PrepareGraphicsBindings(RenderExecutor &executor,
std::span<PreparedBindings *const> stages,
std::span<RenderColorInfo> colors) {
executor.PrepareGraphicsBindings(stages, colors);
}
static PipelineCache::Pipeline
CreateDescriptorPipeline(RenderExecutor &executor,
std::span<PreparedBindings *const> stages) {
std::vector<vk::DescriptorSetLayoutBinding> layout_bindings;
bool compute = false;
for (const auto *prepared : stages) {
const auto &program = *prepared->runtime->program;
compute |= program.stage == ShaderType::Compute;
const auto shader_stage = program.stage == ShaderType::Vertex
? vk::ShaderStageFlagBits::eVertex
: program.stage == ShaderType::Pixel
? vk::ShaderStageFlagBits::eFragment
: vk::ShaderStageFlagBits::eCompute;
for (auto mask = program.bindings.descriptor_mask; mask != 0; mask &= mask - 1) {
const auto index = std::countr_zero(mask);
const auto kind = static_cast<ShaderRecompiler::IR::DescriptorBindingKind>(index);
layout_bindings.push_back(
{ShaderRecompiler::IR::NativeBinding(program.stage, kind),
NativeDescriptorType(kind), program.bindings.descriptor_counts[index], shader_stage});
}
}
vk::DescriptorSetLayoutCreateInfo descriptor_info{};
descriptor_info.sType = vk::StructureType::eDescriptorSetLayoutCreateInfo;
descriptor_info.flags =
vk::DescriptorSetLayoutCreateFlagBits::ePushDescriptorKHR;
descriptor_info.bindingCount =
static_cast<uint32_t>(layout_bindings.size());
descriptor_info.pBindings = layout_bindings.data();
PipelineCache::Pipeline pipeline{};
auto &device = executor.m_context.GetGraphics().device;
EXIT_IF(device.createDescriptorSetLayout(&descriptor_info, nullptr,
&pipeline.descriptor_set_layout) !=
vk::Result::eSuccess);
vk::PipelineLayoutCreateInfo pipeline_info{};
pipeline_info.sType = vk::StructureType::ePipelineLayoutCreateInfo;
pipeline_info.setLayoutCount = 1;
pipeline_info.pSetLayouts = &pipeline.descriptor_set_layout;
const vk::PushConstantRange push_range {
compute ? vk::ShaderStageFlags {vk::ShaderStageFlagBits::eCompute}
: vk::ShaderStageFlags {vk::ShaderStageFlagBits::eVertex |
vk::ShaderStageFlagBits::eFragment},
0, ShaderRecompiler::IR::NativePushConstantSize};
pipeline_info.pushConstantRangeCount = 1;
pipeline_info.pPushConstantRanges = &push_range;
EXIT_IF(device.createPipelineLayout(&pipeline_info, nullptr,
&pipeline.pipeline_layout) !=
vk::Result::eSuccess);
pipeline.uses_push_descriptors = true;
return pipeline;
}
static PipelineCache::Pipeline CommitBindings(RenderExecutor &executor,
CommandBuffer &buffer,
PreparedBindings &bindings) {
std::array<PreparedBindings *, 1> stages{&bindings};
auto pipeline = CreateDescriptorPipeline(executor, stages);
const auto bind_point = bindings.runtime->program->stage == ShaderType::Compute
? vk::PipelineBindPoint::eCompute
: vk::PipelineBindPoint::eGraphics;
executor.CommitBindings(buffer, bind_point, pipeline, stages);
return pipeline;
}
static PipelineCache::Pipeline CommitBindings(RenderExecutor &executor,
CommandBuffer &buffer,
PreparedBindings &vertex,
PreparedBindings &pixel) {
std::array<PreparedBindings *, 2> stages{&vertex, &pixel};
auto pipeline = CreateDescriptorPipeline(executor, stages);
executor.CommitBindings(buffer, vk::PipelineBindPoint::eGraphics, pipeline,
stages);
return pipeline;
}
static void CommitBindings(RenderExecutor &executor, CommandBuffer &buffer,
const PipelineCache::Pipeline &pipeline,
PreparedBindings &vertex, PreparedBindings &pixel) {
std::array<PreparedBindings *, 2> stages{&vertex, &pixel};
executor.CommitBindings(buffer, vk::PipelineBindPoint::eGraphics, pipeline,
stages);
}
static void DestroyDescriptorPipelines(
RenderExecutor &executor,
std::span<const PipelineCache::Pipeline> pipelines) {
auto &device = executor.m_context.GetGraphics().device;
for (const auto &pipeline : pipelines) {
device.destroyPipelineLayout(pipeline.pipeline_layout, nullptr);
device.destroyDescriptorSetLayout(pipeline.descriptor_set_layout,
nullptr);
}
}
static void ResolveRenderDepthTarget(RenderExecutor &executor,
CommandBuffer &buffer,
RenderDepthInfo &depth) {
executor.ResolveRenderDepthTarget(buffer, depth);
}
static bool DepthStencilCopy(RenderExecutor &executor, CommandBuffer &buffer) {
return executor.DepthStencilCopy(buffer);
}
static void ResolveRenderColorTarget(RenderExecutor &executor,
CommandBuffer &buffer,
RenderColorInfo &color, uint32_t slot) {
executor.ResolveRenderColorTarget(buffer, color, 0, slot);
}
static void BindRenderTarget(RenderExecutor &executor, ImageId id) {
executor.BindRenderTarget(id);
}
static RenderState AcquireRenderTargets(RenderExecutor &executor,
CommandBuffer &buffer,
RenderColorInfo *colors,
uint32_t color_count,
RenderDepthInfo &depth,
std::span<PreparedBindings *const> stages = {},
vk::ImageAspectFlags *feedback_out = nullptr) {
vk::ImageAspectFlags feedback;
auto state = executor.AcquireRenderTargets(buffer, colors, color_count, depth,
feedback, stages);
if (feedback_out != nullptr) {
*feedback_out = feedback;
}
return state;
}
static void ResetBindings(RenderExecutor &executor) {
executor.ResetBindings();
}
static bool BoundImagesInOrder(const RenderExecutor &executor, ImageId first,
ImageId second) {
return executor.m_bound_images.size() == 2 &&
executor.m_bound_images[0] == first &&
executor.m_bound_images[1] == second;
}
};
namespace {
using u32 = uint32_t;
using BindingType = TextureCache::BindingType;
using ImageDesc = TextureCache::ImageDesc;
using ShaderOpcode = ShaderRecompiler::Decoder::Opcode;
ImageInfo MakeTilingImage(Prospero::BufferFormat format, uint32_t width,
uint32_t height, uint32_t levels, uint32_t depth,
Prospero::TileMode tile, uint64_t guest_size,
bool volume) {
ImageInfo info{};
info.guest_format = format;
info.type =
volume ? Prospero::ImageType::kColor3D : Prospero::ImageType::kColor2D;
info.extent = {width, height, volume ? depth : 1u};
info.resources = {levels, volume ? 1u : depth};
info.data.size = guest_size;
info.tile_mode = tile;
info.UpdateSize();
return info;
}
constexpr u32 InlineU32(u32 value) { return 128u + value; }
constexpr u32 Vgpr(u32 reg) { return 256u + reg; }
constexpr u32 EncodeSMovB32(u32 dst, u32 src) {
return 0x80000000u | (0x7du << 23u) | ((dst & 0x7fu) << 16u) | (0x03u << 8u) |
(src & 0xffu);
}
constexpr u32 EncodeSop1(u32 opcode, u32 dst, u32 src) {
return 0x80000000u | (0x7du << 23u) | ((dst & 0x7fu) << 16u) |
((opcode & 0xffu) << 8u) | (src & 0xffu);
}
constexpr u32 EncodeSop2(u32 opcode, u32 dst, u32 src0, u32 src1) {
return 0x80000000u | ((opcode & 0x7fu) << 23u) | ((dst & 0x7fu) << 16u) |
((src1 & 0xffu) << 8u) | (src0 & 0xffu);
}
constexpr u32 EncodeSopc(u32 opcode, u32 src0, u32 src1) {
return 0x80000000u | (0x7eu << 23u) | ((opcode & 0x7fu) << 16u) |
((src1 & 0xffu) << 8u) | (src0 & 0xffu);
}
constexpr u32 EncodeSopp(u32 opcode, u32 simm = 0) {
return 0x80000000u | (0x7fu << 23u) | ((opcode & 0x7fu) << 16u) |
(simm & 0xffffu);
}
constexpr u32 EncodeSopk(u32 opcode, u32 dst, u32 imm) {
return 0x80000000u | (((opcode + 0x60u) & 0x7fu) << 23u) |
((dst & 0x7fu) << 16u) | (imm & 0xffffu);
}
constexpr u32 EncodeVop1(u32 opcode, u32 dst, u32 src0) {
return (0x3fu << 25u) | ((dst & 0xffu) << 17u) | ((opcode & 0xffu) << 9u) |
(src0 & 0x1ffu);
}
constexpr u32 EncodeVop1Sdwa(u32 src0, u32 dst_sel = 6, u32 dst_u = 0,
u32 src0_sel = 6, u32 src0_sext = 0,
u32 src0_neg = 0, u32 src0_abs = 0, u32 s0 = 0,
u32 clamp = 0, u32 omod = 0) {
return (src0 & 0xffu) | ((dst_sel & 0x7u) << 8u) | ((dst_u & 0x3u) << 11u) |
((clamp & 0x1u) << 13u) | ((omod & 0x3u) << 14u) |
((src0_sel & 0x7u) << 16u) | ((src0_sext & 0x1u) << 19u) |
((src0_neg & 0x1u) << 20u) | ((src0_abs & 0x1u) << 21u) |
((s0 & 0x1u) << 23u);
}
constexpr u32 EncodeVop1Dpp(u32 src0, u32 dpp_ctrl = 0, u32 row_mask = 0xf,
u32 bank_mask = 0xf) {
return (src0 & 0xffu) | ((dpp_ctrl & 0x1ffu) << 8u) |
((bank_mask & 0xfu) << 24u) | ((row_mask & 0xfu) << 28u);
}
constexpr u32 EncodeVop2(u32 opcode, u32 dst, u32 src0, u32 src1) {
return ((opcode & 0x3fu) << 25u) | ((dst & 0xffu) << 17u) |
((src1 & 0xffu) << 9u) | (src0 & 0x1ffu);
}
constexpr u32 EncodeVop2Sdwa(u32 src0, u32 dst_sel = 6, u32 dst_u = 0,
u32 src0_sel = 6, u32 src1_sel = 6,
u32 src0_sext = 0, u32 src1_sext = 0,
u32 src0_neg = 0, u32 src0_abs = 0,
u32 src1_neg = 0, u32 src1_abs = 0, u32 s0 = 0,
u32 s1 = 0, u32 clamp = 0, u32 omod = 0) {
return (src0 & 0xffu) | ((dst_sel & 0x7u) << 8u) | ((dst_u & 0x3u) << 11u) |
((clamp & 0x1u) << 13u) | ((omod & 0x3u) << 14u) |
((src0_sel & 0x7u) << 16u) | ((src0_sext & 0x1u) << 19u) |
((src0_neg & 0x1u) << 20u) | ((src0_abs & 0x1u) << 21u) |
((s0 & 0x1u) << 23u) | ((src1_sel & 0x7u) << 24u) |
((src1_sext & 0x1u) << 27u) | ((src1_neg & 0x1u) << 28u) |
((src1_abs & 0x1u) << 29u) | ((s1 & 0x1u) << 31u);
}
constexpr u32 EncodeVop2Dpp(u32 src0, u32 dpp_ctrl = 0, u32 row_mask = 0xf,
u32 bank_mask = 0xf, u32 src0_neg = 0,
u32 src0_abs = 0, u32 src1_neg = 0,
u32 src1_abs = 0) {
return (src0 & 0xffu) | ((dpp_ctrl & 0x1ffu) << 8u) |
((src0_neg & 0x1u) << 20u) | ((src0_abs & 0x1u) << 21u) |
((src1_neg & 0x1u) << 22u) | ((src1_abs & 0x1u) << 23u) |
((bank_mask & 0xfu) << 24u) | ((row_mask & 0xfu) << 28u);
}
constexpr u32 EncodeVop3Word0(u32 opcode, u32 dst, u32 abs = 0, u32 op_sel = 0,
bool clamp = false) {
return (0x35u << 26u) | ((opcode & 0x3ffu) << 16u) | ((abs & 0x7u) << 8u) |
((op_sel & 0xfu) << 11u) | (clamp ? (1u << 15u) : 0u) | (dst & 0xffu);
}
constexpr u32 EncodeVop3BWord0(u32 opcode, u32 vdst, u32 sdst) {
return (0x35u << 26u) | ((opcode & 0x3ffu) << 16u) | ((sdst & 0x7fu) << 8u) |
(vdst & 0xffu);
}
constexpr u32 EncodeVop3Word1(u32 src0, u32 src1, u32 src2 = 0, u32 omod = 0,
u32 neg = 0) {
return (src0 & 0x1ffu) | ((src1 & 0x1ffu) << 9u) | ((src2 & 0x1ffu) << 18u) |
((omod & 0x3u) << 27u) | ((neg & 0x7u) << 29u);
}
constexpr u32 EncodeVop3pWord0(u32 opcode, u32 dst, u32 op_sel_hi = 0,
u32 op_sel = 0, u32 neg_hi = 0,
bool clamp = false) {
return (0x33u << 26u) | ((opcode & 0x7fu) << 16u) | ((neg_hi & 0x7u) << 8u) |
((op_sel & 0x7u) << 11u) | (((op_sel_hi >> 2u) & 0x1u) << 14u) |
(clamp ? (1u << 15u) : 0u) | (dst & 0xffu);
}
constexpr u32 EncodeVop3pWord1(u32 src0, u32 src1, u32 src2 = 0,
u32 op_sel_hi = 0, u32 neg = 0) {
return (src0 & 0x1ffu) | ((src1 & 0x1ffu) << 9u) | ((src2 & 0x1ffu) << 18u) |
((op_sel_hi & 0x3u) << 27u) | ((neg & 0x7u) << 29u);
}
constexpr u32 EncodeVopc(u32 opcode, u32 src0, u32 src1) {
return (0x3eu << 25u) | ((opcode & 0xffu) << 17u) | ((src1 & 0xffu) << 9u) |
(src0 & 0x1ffu);
}
constexpr u32 EncodeVopcSdwa(u32 src0, u32 sdst = 0, u32 sd = 0,
u32 src0_sel = 6, u32 src1_sel = 6,
u32 src0_sext = 0, u32 src1_sext = 0,
u32 src0_neg = 0, u32 src0_abs = 0,
u32 src1_neg = 0, u32 src1_abs = 0, u32 s0 = 0,
u32 s1 = 0) {
return (src0 & 0xffu) | ((sdst & 0x7fu) << 8u) | ((sd & 0x1u) << 15u) |
((src0_sel & 0x7u) << 16u) | ((src0_sext & 0x1u) << 19u) |
((src0_neg & 0x1u) << 20u) | ((src0_abs & 0x1u) << 21u) |
((s0 & 0x1u) << 23u) | ((src1_sel & 0x7u) << 24u) |
((src1_sext & 0x1u) << 27u) | ((src1_neg & 0x1u) << 28u) |
((src1_abs & 0x1u) << 29u) | ((s1 & 0x1u) << 31u);
}
constexpr u32 EncodeMubuf0(u32 opcode, u32 offset = 0, bool idxen = false,
bool offen = true, bool glc = false) {
return (0x38u << 26u) | ((opcode & 0xffu) << 18u) |
(offen ? (1u << 12u) : 0u) | (idxen ? (1u << 13u) : 0u) |
(glc ? (1u << 14u) : 0u) | (offset & 0xfffu);
}
constexpr u32 EncodeMubuf1(u32 vdata, u32 srsrc, u32 vaddr, u32 soffset = 128) {
return ((soffset & 0xffu) << 24u) | ((srsrc & 0x1fu) << 16u) |
((vdata & 0xffu) << 8u) | (vaddr & 0xffu);
}
constexpr u32 EncodeMtbuf0(u32 opcode, u32 dfmt, u32 nfmt, u32 offset = 0,
bool idxen = false, bool offen = true) {
return (0x3au << 26u) | (offset & 0xfffu) | (offen ? (1u << 12u) : 0u) |
(idxen ? (1u << 13u) : 0u) | ((opcode & 0x7u) << 16u) |
((dfmt & 0xfu) << 19u) | ((nfmt & 0x7u) << 23u);
}
constexpr u32 EncodeMtbuf1(u32 opcode, u32 vdata, u32 srsrc, u32 vaddr,
u32 soffset = 128) {
return (((opcode >> 3u) & 1u) << 21u) | ((soffset & 0xffu) << 24u) |
((srsrc & 0x1fu) << 16u) | ((vdata & 0xffu) << 8u) | (vaddr & 0xffu);
}
constexpr u32 EncodeSmem0(u32 opcode, u32 dst, u32 sbase = 0) {
return (0x3du << 26u) | ((opcode & 0xffu) << 18u) | ((dst & 0x7fu) << 6u) |
(sbase & 0x3fu);
}
constexpr u32 EncodeSmem1(u32 offset, u32 soffset = 0) {
return (offset & 0x1fffffu) | ((soffset & 0x7fu) << 25u);
}
constexpr u32 EncodeMimg0(u32 opcode, u32 dmask, u32 nsa_dwords = 0,
bool glc = false, u32 dim = 1, bool r128 = false) {
return (0x3cu << 26u) | ((opcode >> 7u) & 0x1u) |
((nsa_dwords & 0x3u) << 1u) | ((dim & 0x7u) << 3u) |
((dmask & 0xfu) << 8u) | (glc ? (1u << 13u) : 0u) |
(r128 ? (1u << 15u) : 0u) | ((opcode & 0x7fu) << 18u);
}
constexpr u32 EncodeMimg1(u32 vdata, u32 vaddr, u32 srsrc = 0, u32 ssamp = 0,
bool a16 = false, bool d16 = false) {
return ((ssamp & 0x1fu) << 21u) | ((srsrc & 0x1fu) << 16u) |
((vdata & 0xffu) << 8u) | (vaddr & 0xffu) | (a16 ? (1u << 30u) : 0u) |
(d16 ? (1u << 31u) : 0u);
}
constexpr u32 EncodeVintrp(u32 opcode, u32 dst, u32 attr, u32 chan, u32 src) {
return (0x32u << 26u) | ((dst & 0xffu) << 18u) | ((opcode & 0x3u) << 16u) |
((attr & 0x3fu) << 10u) | ((chan & 0x3u) << 8u) | (src & 0xffu);
}
constexpr u32 EncodeExp0(u32 target, u32 en, bool done = true,
bool compr = false, bool vm = false) {
return (0x3eu << 26u) | ((target & 0x3fu) << 4u) | (en & 0xfu) |
(compr ? (1u << 10u) : 0u) | (done ? (1u << 11u) : 0u) |
(vm ? (1u << 12u) : 0u);
}
constexpr u32 EncodeExp1(u32 src0, u32 src1, u32 src2, u32 src3) {
return (src0 & 0xffu) | ((src1 & 0xffu) << 8u) | ((src2 & 0xffu) << 16u) |
((src3 & 0xffu) << 24u);
}
constexpr u32 EncodeFlat0(u32 opcode, u32 segment, u32 offset = 0) {
return (0x37u << 26u) | ((opcode & 0x7fu) << 18u) |
((segment & 0x3u) << 14u) | (offset & 0xfffu);
}
constexpr u32 EncodeFlat1(u32 vdst, u32 saddr, u32 data, u32 addr) {
return ((vdst & 0xffu) << 24u) | ((saddr & 0x7fu) << 16u) |
((data & 0xffu) << 8u) | (addr & 0xffu);
}
constexpr u32 EncodeDs0(u32 opcode, u32 offset = 0, bool gds = false) {
return (0x36u << 26u) | ((opcode & 0xffu) << 18u) | (gds ? (1u << 17u) : 0u) |
(offset & 0xffffu);
}
constexpr u32 EncodeDs1Ex(u32 vdst, u32 data1, u32 data0, u32 addr) {
return ((vdst & 0xffu) << 24u) | ((data1 & 0xffu) << 16u) |
((data0 & 0xffu) << 8u) | (addr & 0xffu);
}
constexpr u32 EncodeDs1(u32 vdst, u32 data0, u32 addr) {
return EncodeDs1Ex(vdst, 0, data0, addr);
}
void AppendSMovLiteral(std::vector<u32> *code, u32 dst, u32 literal) {
code->push_back(EncodeSMovB32(dst, 255u));
code->push_back(literal);
}
void AppendVMovLiteral(std::vector<u32> *code, u32 dst, u32 literal) {
code->push_back(EncodeVop1(0x01, dst, 255u));
code->push_back(literal);
}
void AppendVMovU32(std::vector<u32> *code, u32 dst, u32 value) {
if (value <= 64u) {
code->push_back(EncodeVop1(0x01, dst, InlineU32(value)));
return;
}
AppendVMovLiteral(code, dst, value);
}
void AppendVop3(std::vector<u32> *code, u32 opcode, u32 dst, u32 src0, u32 src1,
u32 src2 = 0, u32 abs = 0, u32 op_sel = 0, bool clamp = false,
u32 omod = 0, u32 neg = 0) {
code->push_back(EncodeVop3Word0(opcode, dst, abs, op_sel, clamp));
code->push_back(EncodeVop3Word1(src0, src1, src2, omod, neg));
}
void AppendVop3B(std::vector<u32> *code, u32 opcode, u32 vdst, u32 sdst,
u32 src0, u32 src1, u32 src2 = 0) {
code->push_back(EncodeVop3BWord0(opcode, vdst, sdst));
code->push_back(EncodeVop3Word1(src0, src1, src2));
}
void AppendVop3p(std::vector<u32> *code, u32 opcode, u32 dst, u32 src0,
u32 src1, u32 src2 = 0, u32 op_sel_hi = 0, u32 op_sel = 0,
u32 neg_hi = 0, u32 neg = 0) {
code->push_back(EncodeVop3pWord0(opcode, dst, op_sel_hi, op_sel, neg_hi));
code->push_back(EncodeVop3pWord1(src0, src1, src2, op_sel_hi, neg));
}
void AppendBufferLoadDword(std::vector<u32> *code, u32 dst_vgpr,
u32 address_vgpr) {
code->push_back(EncodeMubuf0(0x0cu));
code->push_back(EncodeMubuf1(dst_vgpr, 0, address_vgpr));
}
void AppendBufferLoadOpcode(std::vector<u32> *code, u32 opcode, u32 dst_vgpr,
u32 address_vgpr) {
code->push_back(EncodeMubuf0(opcode));
code->push_back(EncodeMubuf1(dst_vgpr, 0, address_vgpr));
}
void AppendBufferStoreDword(std::vector<u32> *code, u32 value_vgpr,
u32 address_vgpr) {
code->push_back(EncodeMubuf0(0x1cu));
code->push_back(EncodeMubuf1(value_vgpr, 12, address_vgpr));
}
void AppendBufferStoreOpcode(std::vector<u32> *code, u32 opcode, u32 value_vgpr,
u32 address_vgpr, bool glc = false) {
code->push_back(EncodeMubuf0(opcode, 0, false, true, glc));
code->push_back(EncodeMubuf1(value_vgpr, 12, address_vgpr));
}
void AppendTBufferLoadOpcode(std::vector<u32> *code, u32 opcode, u32 dst_vgpr,
u32 address_vgpr) {
code->push_back(EncodeMtbuf0(opcode, 14, 7));
code->push_back(EncodeMtbuf1(opcode, dst_vgpr, 0, address_vgpr));
}
constexpr u32 BufferFormat(Prospero::BufferFormat format) {
return static_cast<uint32_t>(format);
}
void AppendTBufferLoadFormatOpcode(std::vector<u32> *code, u32 opcode,
u32 dst_vgpr, u32 address_vgpr,
Prospero::BufferFormat format) {
const auto value = BufferFormat(format);
code->push_back(EncodeMtbuf0(opcode, value & 0xfu, (value >> 4u) & 0x7u));
code->push_back(EncodeMtbuf1(opcode, dst_vgpr, 0, address_vgpr));
}
void AppendTBufferStoreOpcode(std::vector<u32> *code, u32 opcode,
u32 value_vgpr, u32 address_vgpr) {
code->push_back(EncodeMtbuf0(opcode, 14, 7));
code->push_back(EncodeMtbuf1(opcode, value_vgpr, 0, address_vgpr));
}
void AppendSmemLoadOpcode(std::vector<u32> *code, u32 opcode, u32 dst_sgpr,
u32 byte_offset) {
code->push_back(EncodeSmem0(opcode, dst_sgpr));
code->push_back(EncodeSmem1(byte_offset));
}
void AppendStoreVgpr(std::vector<u32> *code, u32 value_vgpr, u32 dword_index) {
AppendVMovU32(code, 31, dword_index * 4u);
AppendBufferStoreDword(code, value_vgpr, 31);
}
void AppendStoreVgprAtLaneDwordOffset(std::vector<u32> *code, u32 value_vgpr,
u32 lane_vgpr, u32 dword_offset) {
if (dword_offset == 0u) {
code->push_back(EncodeVop2(0x1a, 31, InlineU32(2), lane_vgpr));
} else {
AppendVMovU32(code, 31, dword_offset);
code->push_back(EncodeVop2(0x25, 31, Vgpr(lane_vgpr), 31));
code->push_back(EncodeVop2(0x1a, 31, InlineU32(2), 31));
}
AppendBufferStoreDword(code, value_vgpr, 31);
}
void AppendStoreSgpr(std::vector<u32> *code, u32 value_sgpr, u32 dword_index) {
code->push_back(EncodeVop1(0x01, 30, value_sgpr));
AppendStoreVgpr(code, 30, dword_index);
}
void AppendStoreSgprAtLaneDwordOffset(std::vector<u32> *code, u32 value_sgpr,
u32 lane_vgpr, u32 dword_offset) {
code->push_back(EncodeVop1(0x01, 30, value_sgpr));
AppendStoreVgprAtLaneDwordOffset(code, 30, lane_vgpr, dword_offset);
}
void AppendStoreSgprPair(std::vector<u32> *code, u32 value_sgpr,
u32 first_dword_index) {
AppendStoreSgpr(code, value_sgpr, first_dword_index);
AppendStoreSgpr(code, value_sgpr + 1u, first_dword_index + 1u);
}
void AppendEnd(std::vector<u32> *code) { code->push_back(0xbf810000u); }
std::string Hex(u32 value) {
char buffer[32] = {};
std::snprintf(buffer, sizeof(buffer), "0x%08" PRIx32, value);
return buffer;
}
[[noreturn]] void Fail(const char *shader_name, const char *stage,
const std::string &message) {
std::fprintf(stderr, "ShaderRecompilerComputeTests: %s failed at %s: %s\n",
shader_name, stage, message.c_str());
std::abort();
}
void Require(const char *shader_name, const char *stage, bool value,
const std::string &message);
void EnsureConfigInitialized() {
static bool config_initialized = false;
if (!config_initialized) {
static Common::Subsystems subsystems;
Common::InitializeThreads();
subsystems.Initialize<Config::Lifecycle>();
Config::ConfigOptions options;
options.printf_direction = Config::LogDirection::Silent;
Config::Load(options);
subsystems.Initialize<Log::Lifecycle>();
subsystems.Initialize<Libs::LibKernel::Memory::Lifecycle>();
ShaderInit();
config_initialized = true;
}
}
void RequireVk(const char *shader_name, const char *stage, vk::Result result,
const char *action) {
if (result != vk::Result::eSuccess) {
Fail(shader_name, stage,
std::string(action) + " returned " + vk::to_string(result));
}
}
void Require(const char *shader_name, const char *stage, bool value,
const std::string &message) {
if (!value) {
Fail(shader_name, stage, message);
}
}
void CheckErrorDialogLifecycle() {
namespace ErrorDialog = Libs::Dialog::ErrorDialog;
namespace SystemDialog = Libs::Dialog::SystemDialog;
struct Param {
int32_t size = 16;
int32_t error_code = static_cast<int32_t>(0x80550006u);
int32_t user_id = 1;
int32_t reserved = 0;
} param;
static_assert(sizeof(Param) == 16);
constexpr int invalid_state = static_cast<int>(0x80ed0005u);
constexpr int invalid_param = static_cast<int>(0x80ed0003u);
static std::vector<SystemDialog::VisualState> notifications;
notifications.clear();
SystemDialog::SetVisibilityCallback([] {
const auto visual = SystemDialog::GetVisualState();
SystemDialog::HostSnapshot snapshot{};
Require("ErrorDialog", "callback reentry",
SystemDialog::GetHostSnapshot(&snapshot) == visual.active &&
(ErrorDialog::ErrorDialogGetStatus() == 2) == visual.active,
"visibility callback observed inconsistent dialog state");
notifications.push_back(visual);
});
const auto initial_revision = SystemDialog::GetVisualState().revision;
const auto check_state = [&](const char *stage, int status,
size_t visibility_changes) {
const auto visual = SystemDialog::GetVisualState();
SystemDialog::HostSnapshot snapshot{};
Require("ErrorDialog", stage,
ErrorDialog::ErrorDialogGetStatus() == status &&
ErrorDialog::ErrorDialogUpdateStatus() == status &&
visual.active == (status == 2) && !visual.background &&
SystemDialog::GetHostSnapshot(&snapshot) == visual.active &&
visual.revision == initial_revision + visibility_changes &&
notifications.size() == visibility_changes &&
(notifications.empty() ||
(notifications.back().active == visual.active &&
notifications.back().revision == visual.revision)),
"status, host visibility, or visibility notification changed "
"unexpectedly");
};
check_state("uninitialized", 0, 0);
Require("ErrorDialog", "uninitialized operations",
ErrorDialog::ErrorDialogTerminate() ==
static_cast<int>(0x80ed0001u) &&
ErrorDialog::ErrorDialogClose() == invalid_state,
"uninitialized dialog accepted termination or close");
Require("ErrorDialog", "initialize",
ErrorDialog::ErrorDialogInitialize() == 0 &&
ErrorDialog::ErrorDialogInitialize() ==
static_cast<int>(0x80ed0002u),
"initialization did not reject a second initialization");
Param invalid = param;
invalid.size--;
Require("ErrorDialog", "invalid open",
ErrorDialog::ErrorDialogOpen(nullptr) == invalid_param &&
ErrorDialog::ErrorDialogOpen(&invalid) == invalid_param &&
ErrorDialog::ErrorDialogClose() == invalid_state,
"invalid parameters or close were accepted before opening");
check_state("failed open preserves initialized state", 1, 0);
Require("ErrorDialog", "open", ErrorDialog::ErrorDialogOpen(&param) == 0,
"valid error dialog did not open");
SystemDialog::HostSnapshot first{};
Require("ErrorDialog", "copied error code",
SystemDialog::GetHostSnapshot(&first) &&
first.error_code == param.error_code,
"host dialog did not expose the guest error code");
param.error_code = static_cast<int32_t>(0x8055000au);
for (int poll = 0; poll < 8; ++poll) {
check_state("polling waits for acknowledgement", 2, 1);
}
Require(
"ErrorDialog", "running failures",
ErrorDialog::ErrorDialogOpen(&param) == invalid_state &&
!SystemDialog::HostClose(first.generation + 1),
"an invalid open or stale acknowledgement replaced the active dialog");
SystemDialog::HostSnapshot current{};
Require("ErrorDialog", "failed open preserves active dialog",
SystemDialog::GetHostSnapshot(&current) &&
current.generation == first.generation &&
current.error_code == first.error_code,
"active error code or generation changed after a failed open");
check_state("failed operations preserve running state", 2, 1);
Require("ErrorDialog", "acknowledge",
SystemDialog::HostClose(first.generation),
"current host acknowledgement did not finish the dialog");
check_state("acknowledged", 3, 2);
Require("ErrorDialog", "finished operations",
!SystemDialog::HostClose(first.generation) &&
ErrorDialog::ErrorDialogClose() == invalid_state &&
ErrorDialog::ErrorDialogOpen(&invalid) == invalid_param,
"finished dialog accepted duplicate completion or an invalid open");
check_state("failed operations preserve finished state", 3, 2);
Require("ErrorDialog", "reopen and stale acknowledgement",
ErrorDialog::ErrorDialogOpen(&param) == 0 &&
SystemDialog::GetHostSnapshot(&current) &&
current.generation != first.generation &&
current.error_code == param.error_code &&
!SystemDialog::HostClose(first.generation),
"reopened dialog accepted acknowledgement from its predecessor");
check_state("reopened", 2, 3);
Require("ErrorDialog", "guest close", ErrorDialog::ErrorDialogClose() == 0,
"guest close did not finish the running dialog");
check_state("closed", 3, 4);
Require("ErrorDialog", "terminate running dialog",
ErrorDialog::ErrorDialogOpen(&param) == 0 &&
SystemDialog::GetHostSnapshot(&current) &&
ErrorDialog::ErrorDialogTerminate() == 0 &&
!SystemDialog::HostClose(current.generation),
"termination failed to remove the active dialog");
check_state("terminated", 0, 6);
Require("ErrorDialog", "reinitialize and stale acknowledgement",
ErrorDialog::ErrorDialogInitialize() == 0 &&
ErrorDialog::ErrorDialogOpen(&param) == 0 &&
!SystemDialog::HostClose(current.generation),
"reinitialization reused an old dialog generation");
check_state("reinitialized", 2, 7);
Require("ErrorDialog", "final termination",
ErrorDialog::ErrorDialogTerminate() == 0 &&
ErrorDialog::ErrorDialogTerminate() ==
static_cast<int>(0x80ed0001u),
"termination did not restore the uninitialized state");
check_state("final state", 0, 8);
SystemDialog::SetVisibilityCallback(nullptr);
Require("ErrorDialog", "lifecycle without a visibility listener",
ErrorDialog::ErrorDialogInitialize() == 0 &&
ErrorDialog::ErrorDialogOpen(&param) == 0 &&
ErrorDialog::ErrorDialogClose() == 0 &&
ErrorDialog::ErrorDialogTerminate() == 0 &&
ErrorDialog::ErrorDialogGetStatus() == 0 &&
!SystemDialog::GetVisualState().active &&
SystemDialog::GetVisualState().revision ==
initial_revision + 10 &&
notifications.size() == 8,
"guest lifecycle depended on a registered host visibility listener");
std::printf("[host] %-32s ok\n", "ErrorDialogLifecycle");
}
void CheckSigninDialogLifecycle() {
namespace Signin = Libs::Dialog::SigninDialog;
namespace Error = Libs::Dialog::ErrorDialog;
namespace Host = Libs::Dialog::SystemDialog;
struct Param {
int32_t size = 16;
int32_t user_id = Config::GetUserId();
int32_t reserved[2] = {};
} param;
struct Result {
int32_t result = -1;
int32_t reserved[3] = {-1, -1, -1};
} result;
constexpr int not_initialized = static_cast<int>(0x81350001u);
constexpr int already_initialized = static_cast<int>(0x81350002u);
constexpr int invalid_param = static_cast<int>(0x81350003u);
constexpr int invalid_state = static_cast<int>(0x81350005u);
Require("SigninDialog", "uninitialized",
Signin::SigninDialogOpen(&param) == not_initialized &&
Signin::SigninDialogClose() == not_initialized &&
Signin::SigninDialogGetResult(&result) == not_initialized,
"uninitialized operations returned the wrong error");
Require("SigninDialog", "initialize and close",
Signin::SigninDialogInitialize() == 0 &&
Signin::SigninDialogClose() == 0 &&
Signin::SigninDialogUpdateStatus() == 3,
"close must also succeed from INITIALIZED");
Param invalid = param;
invalid.reserved[0] = 1;
Require("SigninDialog", "parameter validation",
Signin::SigninDialogOpen(nullptr) == invalid_param &&
Signin::SigninDialogOpen(&invalid) == invalid_param,
"invalid parameters were accepted");
invalid = param;
invalid.user_id = -1;
Require("SigninDialog", "user validation",
Signin::SigninDialogOpen(&invalid) == static_cast<int>(0x81350007u),
"an invalid user was accepted");
Host::SetVisibilityCallback([] {
const auto visual = Host::GetVisualState();
Require("SigninDialog", "callback reentry",
visual.background == (Signin::SigninDialogGetStatus() == 2),
"foreground state did not follow the dialog lifetime");
});
Require("SigninDialog", "open", Signin::SigninDialogOpen(&param) == 0,
"valid sign-in dialog did not open");
Host::HostSnapshot first{};
Require(
"SigninDialog", "visible and backgrounded",
Host::GetHostSnapshot(&first) && first.kind == Host::Kind::Signin &&
Host::GetVisualState().active && Host::GetVisualState().background,
"sign-in dialog was not visible or did not background the application");
for (int poll = 0; poll < 8; ++poll) {
Require(
"SigninDialog", "polling waits for the user",
Signin::SigninDialogUpdateStatus() == 2 &&
Signin::SigninDialogGetStatus() == 2 &&
Signin::SigninDialogInitialize() == already_initialized &&
Signin::SigninDialogGetResult(&result) == invalid_state,
"polling silently finished sign-in or lost the active initialization");
}
Require(
"SigninDialog", "user cancellation",
Host::HostClose(first.generation) &&
Signin::SigninDialogGetResult(&result) == 0 && result.result == 1 &&
result.reserved[0] == 0 && result.reserved[1] == 0 &&
result.reserved[2] == 0 && !Host::GetVisualState().background &&
!Host::GetVisualState().active && Signin::SigninDialogClose() == 0,
"user cancellation did not finish sign-in and restore the foreground");
const int32_t error_param[4] = {16, static_cast<int32_t>(0x80550006u), 1, 0};
Host::HostSnapshot error{};
Require("SigninDialog", "shared presentation generations",
Error::ErrorDialogInitialize() == 0 &&
Error::ErrorDialogOpen(error_param) == 0 &&
Host::GetHostSnapshot(&error) &&
error.kind == Host::Kind::Error &&
error.generation != first.generation &&
!Host::HostClose(first.generation),
"an old sign-in action affected an error dialog");
Host::HostSnapshot overlay{};
Require("SigninDialog", "overlapping dialogs",
Signin::SigninDialogOpen(&param) == 0 &&
Host::GetHostSnapshot(&overlay) &&
overlay.kind == Host::Kind::Signin &&
!Host::HostClose(error.generation) &&
Host::HostClose(overlay.generation) &&
Host::GetHostSnapshot(&overlay) &&
overlay.kind == Host::Kind::Error &&
overlay.generation == error.generation &&
!Host::GetVisualState().background &&
Host::HostClose(error.generation) &&
Error::ErrorDialogTerminate() == 0,
"closing sign-in did not restore the pending error dialog");
Host::HostSnapshot reopened{};
Require(
"SigninDialog", "reopen and abort",
Signin::SigninDialogOpen(&param) == 0 &&
Host::GetHostSnapshot(&reopened) &&
!Host::HostClose(first.generation) &&
!Host::HostClose(error.generation) &&
Signin::SigninDialogTerminate() == 0 &&
!Host::HostClose(reopened.generation) &&
!Host::GetVisualState().active &&
!Host::GetVisualState().background &&
Signin::SigninDialogTerminate() == not_initialized,
"abort did not remove the dialog or a stale host action was accepted");
Host::SetVisibilityCallback(nullptr);
std::printf("[host] %-32s ok\n", "SigninDialogLifecycle");
}
#if KYTY_PLATFORM != KYTY_PLATFORM_WINDOWS
template <typename Function>
void ExpectFatal(const char *name, Function function) {
const pid_t pid = ::fork();
Require(name, "fatal subprocess", pid >= 0,
"fork failed while starting fatal shader test");
if (pid == 0) {
function();
::_exit(0);
}
int status = 0;
pid_t waited = -1;
do {
waited = ::waitpid(pid, &status, 0);
} while (waited < 0 && errno == EINTR);
Require(name, "fatal subprocess", waited == pid,
"waitpid failed while collecting fatal shader test");
Require(name, "fatal exit",
WIFEXITED(status) && WEXITSTATUS(status) == (321 & 0xff),
"fatal shader path did not terminate with status 321");
std::printf("[host] %-32s ok\n", name);
}
#endif
void CheckLeastRecentlyUsedCacheOrdering() {
Common::LeastRecentlyUsedCache<uint32_t, uint64_t> cache;
const auto first = cache.Insert(1, 1);
const auto middle = cache.Insert(2, 2);
const auto last = cache.Insert(3, 3);
(void)first;
(void)last;
cache.Touch(middle, 4);
std::vector<uint32_t> visited;
cache.ForEachItemBelow(4, [&](uint32_t value) {
visited.push_back(value);
return visited.size() > 3;
});
Require("LeastRecentlyUsedCache", "non-tail touch traversal",
visited == std::vector<uint32_t>{1, 3, 2},
"touching a non-tail item left a cycle or changed LRU order");
std::printf("[host] %-32s ok\n", "LeastRecentlyUsedCache");
}
struct BdaMapping {
uint64_t guest_base = 0;
u32 backing_offset = 0;
};
struct TestCase {
const char *name = "";
std::vector<u32> code;
std::vector<u32> initial;
std::vector<u32> expected;
std::vector<ShaderOpcode> opcodes;
u32 image_width = 4;
u32 image_height = 4;
std::vector<u32> sampled_image_rgba;
std::vector<std::vector<u32>> sampled_image_rgba_mips;
vk::Format sampled_image_format = vk::Format::eR32G32B32A32Sfloat;
vk::Filter sampler_filter = vk::Filter::eNearest;
u32 sampled_image_dwords_per_pixel = 4;
vk::ImageType sampled_image_type = vk::ImageType::e2D;
vk::ImageViewType sampled_image_view_type = vk::ImageViewType::e2D;
u32 sampled_image_layers = 1;
u32 sampled_image_view_base_mip = 0;
u32 sampled_image_view_base_layer = 0;
u32 sampled_image_view_layers = 0;
std::vector<u32> storage_image_rgba;
std::vector<u32> expected_storage_image_rgba;
vk::Format storage_image_format = vk::Format::eR32G32B32A32Sfloat;
u32 storage_image_dwords_per_pixel = 4;
std::vector<u32> storage_image_r32ui;
std::vector<u32> expected_storage_image_r32ui;
std::vector<std::string> required_spirv;
std::vector<std::string> forbidden_spirv;
ShaderComputeInputInfo compute_info = [] {
ShaderComputeInputInfo info{};
info.lds_size_dwords = 1024;
return info;
}();
bool has_compute_info = false;
u32 dispatch_x = 1;
u32 dispatch_y = 1;
u32 dispatch_z = 1;
std::array<u32, 64> user_data{};
bool has_user_data = false;
u32 image_descriptor_swizzle = DstSel(4, 5, 6, 7);
bool compile_only = false;
size_t storage_buffer_range_bytes = 0;
std::vector<u32> storage_buffer_offsets;
std::vector<std::pair<u32, u32>> storage_buffer_ranges;
std::vector<BdaMapping> bda_mappings;
std::optional<uint64_t> fault_free_address;
bool expand_shader_data_storage = false;
bool expected_force_point_sampler = false;
float expected_float_tolerance = 0.0f;
std::vector<u32> gds_initial;
std::vector<u32> expected_gds;
std::vector<std::pair<std::string, size_t>> decoded_counts;
std::vector<std::pair<std::string, size_t>> ir_counts;
u32 expected_mip_descriptors = 0;
std::optional<std::vector<u32>> expected_buffer_resources;
};
struct GraphicsCase {
const char *name = "";
std::vector<u32> fragment_code;
std::vector<u32> expected_pixel;
std::vector<ShaderOpcode> opcodes;
std::array<u32, 64> user_data{};
bool has_user_data = false;
std::vector<u32> push_constants;
std::vector<u32> pixel_interpolator_settings;
bool pixel_no_perspective = false;
bool provoking_vtx_last = false;
std::vector<u32> vertices;
bool pixel_ancillary = false;
bool pixel_front_face = false;
bool pixel_front_face_all_bits = false;
u32 layers = 1;
vk::SampleCountFlagBits samples = vk::SampleCountFlagBits::e1;
bool pixel_position_w = false;
float vertex_clip_w = 1.0f;
bool pixel_depth_export = false;
u32 pixel_perspective_sample_vgpr = UINT32_MAX;
u32 pixel_perspective_centroid_vgpr = UINT32_MAX;
u32 pixel_custom_interpolation_mask = 0;
u32 width = 1;
std::vector<u32> object_ids;
};
struct CompiledShader {
std::vector<u32> spirv;
ShaderRecompiler::IR::Program program;
ShaderRecompiler::IR::ResourceSnapshot resources;
std::vector<u32> packed_user_data;
};
std::array<u32, 64> MakeNativeUserData(const std::array<u32, 64> *source) {
std::array<u32, 64> data{};
data[3] = 3u << 28u;
data[50] = 1u << 20u;
data[51] = 3u << 28u;
if (source != nullptr) {
data = *source;
}
return data;
}
bool ReadTestMemory(void *userdata, uint64_t address, std::span<u32> values) {
const auto *data = static_cast<const std::vector<u32> *>(userdata);
if (data == nullptr || address % 4u != 0 || values.size() > data->size() ||
address / 4u > data->size() - values.size()) {
return false;
}
std::copy_n(data->begin() + address / 4u, values.size(), values.begin());
return true;
}
void ValidateSpirv(const char *shader_name, const std::vector<u32> &spirv) {
spvtools::SpirvTools tools(SPV_ENV_VULKAN_1_2);
std::string messages;
tools.SetMessageConsumer([&messages](spv_message_level_t, const char *,
const spv_position_t &position,
const char *message) {
char buffer[1024] = {};
std::snprintf(buffer, sizeof(buffer), "%zu:%zu: %s\n", position.line,
position.column, message);
messages += buffer;
});
spvtools::ValidatorOptions options;
options.SetAllowOffsetTextureOperand(true);
if (!tools.Validate(spirv.data(), spirv.size(), options)) {
Fail(shader_name, "SPIR-V validation", messages);
}
}
size_t CountText(const std::string &text, const std::string &needle) {
size_t count = 0;
for (size_t offset = 0;
(offset = text.find(needle, offset)) != std::string::npos;
offset += needle.size()) {
count++;
}
return count;
}
void CheckPixelParameterAliases() {
constexpr const char *name = "PixelParameterAliases";
ShaderPixelInputInfo pixel{};
pixel.input_num = 2;
const std::array<uint32_t, 2> pair = {0, 1};
for (const auto flat : {0u, 0x400u}) {
pixel.interpolator_settings[0] = flat | 3u;
pixel.interpolator_settings[1] = flat | 3u;
Require(name, "same-mode nonidentity alias",
ShaderPixelParameterLocation(pixel, pair, 0) == 3u &&
ShaderPixelParameterLocation(pixel, pair, 1) == 3u,
"aliases of one export with the same interpolation mode must "
"share that physical location");
}
pixel.interpolator_settings[0] = 3u;
pixel.interpolator_settings[1] = 0x423u;
pixel.custom_interpolation_mask = 2u;
Require(name, "smooth/custom alias",
ShaderPixelParameterLocation(pixel, pair, 0) == 3u &&
ShaderPixelParameterLocation(pixel, pair, 1) == 3u,
"custom interpolation must retain its shared vertex export");
pixel.input_num = 3;
pixel.custom_interpolation_mask = 0;
pixel.interpolator_settings[0] = 0x400u;
pixel.interpolator_settings[1] = 0u;
pixel.interpolator_settings[2] = 1u;
const std::array<uint32_t, 3> active = {0, 1, 2};
Require(name, "ordinary mixed aliases",
ShaderPixelParameterLocation(pixel, active, 0) == 0u &&
ShaderPixelParameterLocation(pixel, active, 1) == 0u &&
ShaderPixelParameterLocation(pixel, active, 2) == 1u,
"flat/smooth aliases must retain the real vertex export location");
std::vector<uint32_t> ordinary_key, rectangle_key, provoking_key;
BuildStageStaticKey(pixel, ordinary_key);
pixel.parameter_mode = ShaderPixelParameterMode::LastVertex;
BuildStageStaticKey(pixel, provoking_key);
pixel.parameter_mode = ShaderPixelParameterMode::Rectangle;
BuildStageStaticKey(pixel, rectangle_key);
Require(name, "interpolation specialization",
ordinary_key != rectangle_key && ordinary_key != provoking_key,
"rectangle and provoking-vertex modes must participate in the shader key");
Require(name, "reserved physical locations",
ShaderPixelParameterLocation(pixel, active, 0) == 0u &&
ShaderPixelParameterLocation(pixel, active, 1) == 2u &&
ShaderPixelParameterLocation(pixel, active, 2) == 1u,
"a synthesized opposite-mode alias must not occupy another "
"vertex export's physical location");
std::printf("[host] %-32s ok\n", name);
}
void CheckRectListShaders() {
constexpr const char *name = "RectListShaders";
ShaderRecompiler::IR::CompiledShaderInfo program{};
program.info.inputs.push_back(
{ShaderRecompiler::IR::StageInputKind::Parameter, 0, 4, "in_param_0"});
program.info.inputs.push_back(
{ShaderRecompiler::IR::StageInputKind::Parameter, 1, 4, "in_param_1"});
ShaderVertexInputInfo vertex{};
ShaderRecompiler::IR::CompiledShaderInfo vertex_program{.param_export_mask = 1u};
ShaderRecompiler::IR::ResourceSnapshot empty_snapshot;
vertex.stage.program = &vertex_program;
vertex.stage.resources = &empty_snapshot;
ShaderPixelInputInfo pixel{};
pixel.parameter_mode = ShaderPixelParameterMode::Rectangle;
pixel.input_num = 2;
pixel.interpolator_settings[0] = 0x400u;
pixel.interpolator_settings[1] = 0;
pixel.stage.program = &program;
pixel.stage.resources = &empty_snapshot;
std::vector<uint32_t> perspective_specialization;
BuildStageStaticKey(pixel, perspective_specialization);
pixel.ps_no_perspective = true;
std::vector<uint32_t> no_perspective_specialization;
BuildStageStaticKey(pixel, no_perspective_specialization);
pixel.ps_no_perspective = false;
Require(name, "pipeline identity",
perspective_specialization != no_perspective_specialization,
"pixel interpolation mode must participate in the shader and "
"pipeline key");
const std::array<uint32_t, 2> active_inputs = {0, 1};
Require(name, "duplicate mapping",
ShaderPixelParameterLocation(pixel, active_inputs, 0) == 0 &&
ShaderPixelParameterLocation(pixel, active_inputs, 1) == 1,
"duplicate pixel mappings must receive distinct effective output "
"locations");
const auto shaders = BuildRectListShaders(vertex, &pixel);
Require(name, "SPIR-V version",
shaders.control.size() > 1 && shaders.control[1] == 0x00010500u &&
shaders.evaluation.size() > 1 &&
shaders.evaluation[1] == 0x00010500u,
"vector selection requires SPIR-V 1.5");
ValidateSpirv(name, shaders.control);
ValidateSpirv(name, shaders.evaluation);
spvtools::SpirvTools tools(SPV_ENV_VULKAN_1_2);
std::string control_text;
std::string evaluation_text;
Require(name, "control disassembly",
tools.Disassemble(shaders.control, &control_text),
"failed to disassemble rectangle-list tessellation control shader");
Require(
name, "evaluation disassembly",
tools.Disassemble(shaders.evaluation, &evaluation_text),
"failed to disassemble rectangle-list tessellation evaluation shader");
Require(name, "control execution mode",
control_text.find("TessellationControl") != std::string::npos &&
control_text.find("OutputVertices 4") != std::string::npos,
"rectangle-list control shader must produce four control points");
Require(name, "evaluation execution modes",
evaluation_text.find("TessellationEvaluation") != std::string::npos &&
evaluation_text.find("Quads") != std::string::npos &&
evaluation_text.find("SpacingEqual") != std::string::npos &&
evaluation_text.find("VertexOrderCw") != std::string::npos,
"rectangle-list evaluation shader has the wrong patch modes");
Require(name, "no geometry stage",
control_text.find("Geometry") == std::string::npos &&
evaluation_text.find("Geometry") == std::string::npos,
"rectangle-list expansion must not use geometry shaders");
Require(name, "flat broadcast",
CountText(control_text, "OpVectorTimesScalar") == 6 &&
CountText(control_text, "OpSelect %v4float") == 2,
"flat parameters must use guest vertex zero instead of reconstructed "
"values");
Require(name, "remapped interface",
CountText(control_text, " Location 0") == 2 &&
CountText(control_text, " Location 1") == 1 &&
CountText(evaluation_text, " Location 0") == 2 &&
CountText(evaluation_text, " Location 1") == 2,
"duplicate pixel mappings must share one vertex input and keep "
"distinct patch outputs");
for (const auto flat : {0u, 0x400u}) {
pixel.interpolator_settings[0] = flat;
pixel.interpolator_settings[1] = flat;
const auto shared = BuildRectListShaders(vertex, &pixel);
ValidateSpirv(name, shared.control);
ValidateSpirv(name, shared.evaluation);
Require(name, "same-mode alias disassembly",
tools.Disassemble(shared.control, &control_text) &&
tools.Disassemble(shared.evaluation, &evaluation_text),
"failed to disassemble shared rectangle-list parameters");
Require(name, "same-mode interface deduplication",
CountText(control_text, " Location 0") == 2 &&
CountText(evaluation_text, " Location 0") == 2 &&
control_text.find(" Location 1") == std::string::npos &&
evaluation_text.find(" Location 1") == std::string::npos,
"same-mode aliases must produce one input and one output at "
"the shared location in each rectangle-list stage");
}
const auto position_only = BuildRectListShaders(vertex, nullptr);
ValidateSpirv(name, position_only.control);
ValidateSpirv(name, position_only.evaluation);
}
void CheckSpirvEmission(const TestCase &test, const std::vector<u32> &spirv) {
std::vector<u32> result_types(spirv[3]);
for (size_t offset = 5; offset < spirv.size();) {
const auto words = std::span<const u32>(spirv).subspan(offset, spirv[offset] >> 16u);
const auto opcode = static_cast<spv::Op>(words[0] & 0xffffu);
bool has_result = false, has_type = false;
spv::HasResultAndType(opcode, &has_result, &has_type);
if (has_result && has_type) result_types[words[2]] = words[1];
if (opcode == spv::OpBitcast) {
Require(test.name, "bitcast changes representation",
words[1] != result_types[words[3]],
"emitted a redundant bitcast between identical SPIR-V types");
}
offset += words.size();
}
if (test.required_spirv.empty() && test.forbidden_spirv.empty()) {
return;
}
spvtools::SpirvTools tools(SPV_ENV_VULKAN_1_2);
std::string text;
if (!tools.Disassemble(spirv, &text)) {
Fail(test.name, "SPIR-V disassembly",
"failed to disassemble emitted SPIR-V");
}
for (const auto &required : test.required_spirv) {
if (text.find(required) == std::string::npos) {
Fail(test.name, "SPIR-V disassembly",
std::string("missing required text: ") + required);
}
}
for (const auto &forbidden : test.forbidden_spirv) {
if (text.find(forbidden) != std::string::npos) {
Fail(test.name, "SPIR-V disassembly",
std::string("found forbidden text: ") + forbidden);
}
}
}
CompiledShader CompileCase(const TestCase &test, u32 host_subgroup_size = 64) {
auto user_data =
MakeNativeUserData(test.has_user_data ? &test.user_data : nullptr);
const auto uses_image =
std::any_of(test.opcodes.begin(), test.opcodes.end(), [](auto op) {
return op >= ShaderOpcode::IMAGE_GET_RESINFO &&
op <= ShaderOpcode::IMAGE_GATHER4H;
});
if (uses_image && ((user_data[3] >> 28u) & 0xfu) == 0) {
user_data[3] = static_cast<uint32_t>(Prospero::ImageType::kColor2D) << 28u;
}
if (uses_image) {
user_data[3] = (user_data[3] & ~0xfffu) | test.image_descriptor_swizzle;
}
if (!test.has_user_data) {
user_data[2] = static_cast<u32>(test.initial.size() * sizeof(u32));
}
ShaderRecompiler::CompileOptions options;
options.stage = ShaderType::Compute;
options.maintenance8_enabled = true;
options.dump_ir = true;
auto compute_info = test.compute_info;
compute_info.host_subgroup_size = host_subgroup_size;
options.input_info.compute = &compute_info;
options.user_data = user_data;
if (test.has_compute_info) {
options.wave_size = test.compute_info.wave_size;
}
auto translated = ShaderRecompiler::TranslateProgram(test.code, options);
auto resource_plan =
ShaderRecompiler::IR::ExtractResourcePlan(translated.program);
ShaderRecompiler::IR::ResourceSnapshot resources;
ShaderRecompiler::IR::ResourceSpecialization specialization;
const std::array workgroup_counts{test.dispatch_x, test.dispatch_y, test.dispatch_z};
const ShaderRecompiler::IR::SrtRuntime runtime{
.user_data = options.user_data,
.shader_base = reinterpret_cast<uint64_t>(test.code.data()),
.read_memory = ReadTestMemory,
.userdata = const_cast<std::vector<u32> *>(&test.initial),
.read_specialization_memory = ReadTestMemory,
.workgroup_counts = workgroup_counts,
};
Require(test.name, "resource materialization",
ShaderRecompiler::IR::MaterializeResources(
resource_plan, runtime, resources, specialization),
"translated resources could not be materialized");
auto result = ShaderRecompiler::CompileProgram(
std::move(translated), options, specialization);
for (const auto &[text, expected] : test.decoded_counts) {
const auto actual = CountText(result.decoded_dump, text);
Require(test.name, "decoded RDNA2", actual == expected,
text + " count=" + std::to_string(actual) +
", expected=" + std::to_string(expected));
}
for (const auto &[text, expected] : test.ir_counts) {
const auto actual = CountText(result.ir_dump, text);
Require(test.name, "typed IR", actual == expected,
text + " count=" + std::to_string(actual) +
", expected=" + std::to_string(expected));
}
if (test.expected_mip_descriptors != 0) {
const auto image = std::ranges::find_if(
result.program.info.images, [](const auto &resource) {
return resource.mip_mode ==
ShaderRecompiler::IR::ImageMipMode::Dynamic;
});
Require(test.name, "dynamic mip specialization",
image != result.program.info.images.end() &&
image->mip_count == test.expected_mip_descriptors,
"runtime descriptor range did not specialize the mip count");
const auto kind = ShaderRecompiler::IR::DescriptorBindingForImage(*image);
Require(test.name, "dynamic mip binding",
kind && image->descriptor_index != UINT32_MAX &&
image->descriptor_index + image->mip_count <=
result.program.bindings.descriptor_counts[static_cast<size_t>(*kind)],
"dynamic image did not receive one descriptor per mip");
}
if (test.expand_shader_data_storage) {
auto &block = *result.program.blocks.front();
for (u32 reg = 0; reg < 33; reg++) {
auto &user_data = block.AppendNewInst(
ShaderRecompiler::IR::ValueOpcode::GetUserData,
{ShaderRecompiler::IR::Value(
static_cast<ShaderRecompiler::IR::ScalarReg>(reg))});
block.AppendNewInst(ShaderRecompiler::IR::ValueOpcode::ReferenceU32,
{ShaderRecompiler::IR::Value(&user_data)});
}
result.program.shader_info_complete = false;
ShaderRecompiler::IR::CollectShaderInfo(result.program, options.input_info);
result.spirv = ShaderRecompiler::Spirv::EmitProgram(
result.program, options.input_info, 0u, options.maintenance8_enabled);
const auto shader_data = result.program.bindings.descriptor_counts[
static_cast<size_t>(ShaderRecompiler::IR::DescriptorBindingKind::ShaderData)];
Require(test.name, "binding layout",
result.program.info.user_data_registers.size() == 33 &&
!result.program.bindings.UsesPushData() &&
shader_data != 0,
"oversized shader data did not use its storage fallback");
}
Require(test.name, "SPIR-V emit", !result.spirv.empty(),
"recompiler returned empty SPIR-V");
ValidateSpirv(test.name, result.spirv);
CheckSpirvEmission(test, result.spirv);
if (test.expected_buffer_resources) {
std::vector<u32> resources;
for (u32 i = 0; i < result.program.info.buffers.size(); ++i) {
if (result.program.info.buffers[i].descriptor_index != UINT32_MAX) resources.push_back(i);
}
Require(test.name, "live buffer bindings",
resources == *test.expected_buffer_resources,
"dead buffer changed the surviving descriptor order");
}
std::vector<u32> packed_user_data;
for (const auto reg : result.program.info.user_data_registers) {
packed_user_data.push_back(
resources.user_data[reg - result.program.user_data_base]);
}
packed_user_data.resize(result.program.bindings.ShaderDataDwords());
for (u32 resource = 0; resource < result.program.info.buffers.size(); ++resource) {
const auto index = result.program.info.buffers[resource].descriptor_index;
if (index == UINT32_MAX) continue;
const auto offset = resource < test.storage_buffer_offsets.size()
? test.storage_buffer_offsets[resource] : 0u;
Require(test.name, "shader data", offset < 256,
"storage buffer offset is not representable");
packed_user_data[result.program.bindings.memory_offset_dword + index / 4u] |=
offset << ((index % 4u) * 8u);
}
return {std::move(result.spirv), std::move(result.program),
std::move(resources), std::move(packed_user_data)};
}
std::array<u32, 64> MakeStructuredStorageBufferData(u32 stride_bytes,
u32 num_records,
bool add_tid = false,
u32 format = 0) {
std::array<u32, 64> data{};
data[1] = (stride_bytes & 0x3fffu) << 16u;
data[2] = num_records;
data[3] = DstSel(4, 5, 6, 7) | (1u << 24u) |
(stride_bytes == 0 ? 3u << 28u : 0u);
if (add_tid) {
data[3] |= 1u << 23u;
}
data[3] |= (format & 0x7fu) << 12u;
return data;
}
std::array<u32, 64> MakeStorageTextureData(Prospero::BufferFormat format) {
std::array<u32, 64> data{};
data[0] = 0x1000u;
data[1] = (static_cast<uint32_t>(format) & 0x1ffu) << 20u;
data[3] = static_cast<uint32_t>(Prospero::ImageType::kColor2D) << 28u;
return data;
}
std::string StorageUint2DImageBindingName(bool atomic) {
ShaderRecompiler::IR::ImageResource image{};
image.resource_class = ShaderRecompiler::IR::ImageResourceClass::Storage;
image.numeric_class = Prospero::TextureNumericClass::Uint;
image.dimension = ShaderRecompiler::Decoder::ImageDimension::Dim2D;
image.atomic = atomic;
const auto binding = ShaderRecompiler::IR::DescriptorBindingForImage(image);
Require("StorageUint2DImageBindingName", "descriptor binding", binding.has_value(),
"storage image has no descriptor binding");
return "image_" + std::to_string(static_cast<uint32_t>(*binding));
}
CompiledShader CompileFragmentCase(const GraphicsCase &test) {
const auto user_data =
MakeNativeUserData(test.has_user_data ? &test.user_data : nullptr);
ShaderPixelInputInfo pixel_info{};
pixel_info.input_num =
test.pixel_interpolator_settings.empty()
? 1u
: static_cast<u32>(test.pixel_interpolator_settings.size());
pixel_info.ps_no_perspective = test.pixel_no_perspective;
pixel_info.parameter_mode = test.provoking_vtx_last ? ShaderPixelParameterMode::LastVertex
: ShaderPixelParameterMode::FirstVertex;
pixel_info.ps_ancillary = test.pixel_ancillary;
pixel_info.ps_front_face = test.pixel_front_face;
pixel_info.ps_front_face_all_bits = test.pixel_front_face_all_bits;
pixel_info.ps_pos_w = test.pixel_position_w;
pixel_info.ps_depth_export_enable = test.pixel_depth_export;
pixel_info.ps_system_input_base = 2;
pixel_info.ps_perspective_sample_vgpr = test.pixel_perspective_sample_vgpr;
pixel_info.ps_perspective_centroid_vgpr = test.pixel_perspective_centroid_vgpr;
pixel_info.custom_interpolation_mask = test.pixel_custom_interpolation_mask;
pixel_info.object_primitive_id = !test.object_ids.empty();
for (u32 i = 0; i < std::size(pixel_info.interpolator_settings); i++) {
pixel_info.interpolator_settings[i] = i;
}
for (u32 i = 0; i < test.pixel_interpolator_settings.size() &&
i < std::size(pixel_info.interpolator_settings);
i++) {
pixel_info.interpolator_settings[i] = test.pixel_interpolator_settings[i];
}
ShaderRecompiler::CompileOptions options;
options.stage = ShaderType::Pixel;
options.maintenance8_enabled = true;
options.dump_ir = false;
options.input_info.pixel = &pixel_info;
options.user_data = user_data;
auto translated =
ShaderRecompiler::TranslateProgram(test.fragment_code, options);
auto resource_plan =
ShaderRecompiler::IR::ExtractResourcePlan(translated.program);
ShaderRecompiler::IR::ResourceSnapshot resources;
ShaderRecompiler::IR::ResourceSpecialization specialization;
const ShaderRecompiler::IR::SrtRuntime runtime{
.user_data = options.user_data,
.shader_base = reinterpret_cast<uint64_t>(test.fragment_code.data()),
};
Require(test.name, "resource materialization",
ShaderRecompiler::IR::MaterializeResources(
resource_plan, runtime, resources, specialization),
"translated resources could not be materialized");
auto result = ShaderRecompiler::CompileProgram(
std::move(translated), options, specialization,
static_cast<u32>(pixel_info.object_primitive_id));
Require(test.name, "SPIR-V emit", !result.spirv.empty(),
"recompiler returned empty SPIR-V");
ValidateSpirv(test.name, result.spirv);
std::vector<u32> packed_user_data;
for (const auto reg : result.program.info.user_data_registers) {
packed_user_data.push_back(
resources.user_data[reg - result.program.user_data_base]);
}
packed_user_data.resize(result.program.bindings.ShaderDataDwords());
return {std::move(result.spirv), std::move(result.program),
std::move(resources), std::move(packed_user_data)};
}
std::array<u32, 64> MakeSampledTextureData(Prospero::BufferFormat format) {
return MakeStorageTextureData(format);
}
namespace TestSpv {
std::vector<u32> MakePassthroughVertexSpirv(bool layered, float clip_w = 1.0f) {
using ShaderRecompiler::Spirv::Builder;
Builder b;
const auto void_type = b.Type(spv::OpTypeVoid);
const auto uint_type = b.Type(spv::OpTypeInt, 32, 0);
const auto int_type = b.Type(spv::OpTypeInt, 32, 1);
const auto float_type = b.Type(spv::OpTypeFloat, 32);
const auto vec2_type = b.Type(spv::OpTypeVector, float_type, 2);
const auto vec4_type = b.Type(spv::OpTypeVector, float_type, 4);
const auto per_vertex_type =
b.DecoratedType(spv::OpTypeStruct, {{spv::OpDecorate, {spv::DecorationBlock}},
{spv::OpMemberDecorate,
{0, spv::DecorationBuiltIn, spv::BuiltInPosition}}}, vec4_type);
const auto ptr_input_vec2 =
b.Type(spv::OpTypePointer, spv::StorageClassInput, vec2_type);
const auto ptr_input_vec4 =
b.Type(spv::OpTypePointer, spv::StorageClassInput, vec4_type);
const auto ptr_output_vec4 =
b.Type(spv::OpTypePointer, spv::StorageClassOutput, vec4_type);
const auto ptr_output_per_vertex =
b.Type(spv::OpTypePointer, spv::StorageClassOutput, per_vertex_type);
const auto func_type = b.Type(spv::OpTypeFunction, void_type);
const auto const_u32_0 = b.Constant(spv::OpConstant, uint_type, 0);
const auto const_f32_0 =
b.Constant(spv::OpConstant, float_type, 0x00000000u);
const auto const_f32_w =
b.Constant(spv::OpConstant, float_type, std::bit_cast<u32>(clip_w));
const auto in_pos =
b.DefineGlobalVariable(ptr_input_vec2, spv::StorageClassInput);
const auto in_color =
b.DefineGlobalVariable(ptr_input_vec4, spv::StorageClassInput);
const auto out_color =
b.DefineGlobalVariable(ptr_output_vec4, spv::StorageClassOutput);
const auto per_vertex =
b.DefineGlobalVariable(ptr_output_per_vertex, spv::StorageClassOutput);
u32 instance = 0;
u32 layer = 0;
std::vector<u32> interfaces = {in_pos, in_color, per_vertex, out_color};
if (layered) {
instance = b.DefineGlobalVariable(
b.Type(spv::OpTypePointer, spv::StorageClassInput, int_type),
spv::StorageClassInput);
layer = b.DefineGlobalVariable(
b.Type(spv::OpTypePointer, spv::StorageClassOutput, int_type),
spv::StorageClassOutput);
b.AddAnnotation(spv::OpDecorate, instance, spv::DecorationBuiltIn,
spv::BuiltInInstanceIndex);
b.AddAnnotation(spv::OpDecorate, layer, spv::DecorationBuiltIn, spv::BuiltInLayer);
b.RequireVersion(0x00010500u);
b.RequireCapability(spv::CapabilityShaderLayer);
interfaces.insert(interfaces.end(), {instance, layer});
}
const auto main = b.AllocateId();
const auto label = b.AllocateId();
const auto pos2 = b.AllocateId();
const auto color4 = b.AllocateId();
const auto pos_x = b.AllocateId();
const auto pos_y = b.AllocateId();
const auto position = b.AllocateId();
const auto position_ptr = b.AllocateId();
b.RequireCapability(spv::CapabilityShader);
b.AddMemoryModel(spv::AddressingModelLogical, spv::MemoryModelGLSL450);
b.AddEntryPoint(spv::ExecutionModelVertex, main, "main", interfaces);
b.AddAnnotation(spv::OpDecorate, in_pos, spv::DecorationLocation, 0);
b.AddAnnotation(spv::OpDecorate, in_color, spv::DecorationLocation, 1);
b.AddAnnotation(spv::OpDecorate, out_color, spv::DecorationLocation, 0);
b.AddFunction(spv::OpFunction, void_type, main, spv::FunctionControlMaskNone,
func_type);
b.AddFunction(spv::OpLabel, label);
b.AddFunction(spv::OpLoad, vec2_type, pos2, in_pos);
b.AddFunction(spv::OpLoad, vec4_type, color4, in_color);
b.AddFunction(spv::OpCompositeExtract, float_type, pos_x, pos2, 0);
b.AddFunction(spv::OpCompositeExtract, float_type, pos_y, pos2, 1);
b.AddFunction(spv::OpCompositeConstruct, vec4_type, position, pos_x, pos_y,
const_f32_0, const_f32_w);
b.AddFunction(spv::OpAccessChain, ptr_output_vec4, position_ptr, per_vertex,
const_u32_0);
b.AddFunction(spv::OpStore, position_ptr, position);
b.AddFunction(spv::OpStore, out_color, color4);
if (layered) {
const auto index = b.AllocateId();
b.AddFunction(spv::OpLoad, int_type, index, instance);
b.AddFunction(spv::OpStore, layer, index);
}
b.AddFunction(spv::OpReturn);
b.AddFunction(spv::OpFunctionEnd);
return b.Build();
}
} // namespace TestSpv
class VulkanHarness {
public:
VulkanHarness() { Init(); }
~VulkanHarness() { Destroy(); }
VulkanHarness(const VulkanHarness &) = delete;
VulkanHarness &operator=(const VulkanHarness &) = delete;
struct Buffer {
vk::Buffer buffer = nullptr;
vk::DeviceMemory memory = nullptr;
vk::DeviceAddress device_address = 0;
vk::DeviceSize size = 0;
bool coherent = false;
};
struct Image {
vk::Image image = nullptr;
vk::DeviceMemory memory = nullptr;
vk::ImageView view = nullptr;
vk::Format format = vk::Format::eUndefined;
vk::ImageLayout layout = vk::ImageLayout::eUndefined;
u32 width = 0;
u32 height = 0;
u32 layers = 1;
u32 mip_levels = 1;
};
[[nodiscard]] vk::Device Device() const { return m_device; }
[[nodiscard]] u32 SubgroupSize() const {
vk::PhysicalDeviceSubgroupProperties subgroup{};
vk::PhysicalDeviceProperties2 properties{};
properties.pNext = &subgroup;
m_physical_device.getProperties2(&properties);
Require("VulkanHarness", "subgroup vote support",
bool(subgroup.supportedOperations & vk::SubgroupFeatureFlagBits::eVote),
"device does not support subgroup All/Any votes");
return subgroup.subgroupSize;
}
// A consumer GPU may lack the production rasterization features the fragment
// cases need. The compute cases still run on such a device, so the harness reports
// the capability instead of failing the whole binary before the first case.
[[nodiscard]] bool RasterizationSupported() const {
return m_rasterization_supported;
}
void SkipRasterizationCases(u32 count) { m_skipped_cases += count; }
[[nodiscard]] u32 SkippedCaseCount() const { return m_skipped_cases; }
[[nodiscard]] GraphicContext &RuntimeContext() {
EnsureRuntimeContext();
return m_runtime_context;
}
[[nodiscard]] RenderContext &RuntimeRenderer() {
EnsureRuntimeContext();
return Renderer();
}
void CheckHostImageAllocation() {
constexpr const char *name = "HostImageAllocation";
auto &graphics = RuntimeContext();
VulkanImage image;
vk::ImageCreateInfo create{};
create.flags = vk::ImageCreateFlagBits::eMutableFormat;
create.imageType = vk::ImageType::e2D;
create.format = vk::Format::eR8G8B8A8Unorm;
create.extent = {16, 8, 1};
create.mipLevels = 3;
create.arrayLayers = 4;
create.samples = vk::SampleCountFlagBits::e1;
create.tiling = vk::ImageTiling::eOptimal;
create.usage = vk::ImageUsageFlagBits::eTransferSrc |
vk::ImageUsageFlagBits::eTransferDst |
vk::ImageUsageFlagBits::eSampled;
create.initialLayout = vk::ImageLayout::eUndefined;
for (uint32_t pass = 0; pass < 2; ++pass) {
Require(name, "create", graphics.CreateImage(create, image),
"could not allocate native image");
Require(name, "allocation metadata",
image.image != nullptr && image.allocation != nullptr &&
image.format == create.format &&
image.image_type == create.imageType &&
image.extent == create.extent &&
image.layers == create.arrayLayers &&
image.mip_levels == create.mipLevels &&
image.samples == static_cast<uint32_t>(create.samples) &&
image.usage == create.usage && image.flags == create.flags,
"native image metadata did not follow its creation descriptor");
Require(name, "fresh synchronization state",
image.state.layout == create.initialLayout &&
image.state.access_mask == vk::AccessFlags2{} &&
image.state.pl_stage == vk::PipelineStageFlagBits2::eAllCommands &&
image.subresource_states.empty(),
"new allocation retained synchronization state from an old image");
VkMemoryPropertyFlags properties{};
vmaGetAllocationMemoryProperties(graphics.allocator, image.allocation,
&properties);
Require(name, "allocation policy",
(properties & VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT) != 0,
"native image lost its device-local memory requirement");
// A reused presentation image may still describe its previous recording.
// No commands use this fixture; seed that bookkeeping before retirement.
image.state = {vk::PipelineStageFlagBits2::eTransfer,
vk::AccessFlagBits2::eTransferWrite,
vk::ImageLayout::eTransferDstOptimal};
image.subresource_states.assign(image.layers * image.mip_levels, image.state);
graphics.DeleteImage(image);
Require(name, "retirement",
image.image == nullptr && image.allocation == nullptr,
"retired native image retained an allocation handle");
create.flags = vk::ImageCreateFlagBits::e2DArrayCompatible;
create.imageType = vk::ImageType::e3D;
create.format = vk::Format::eR32Uint;
create.extent = {8, 4, 2};
create.mipLevels = 2;
create.arrayLayers = 1;
create.usage = vk::ImageUsageFlagBits::eTransferSrc |
vk::ImageUsageFlagBits::eTransferDst;
}
std::printf("[host] %-32s ok\n", name);
}
void CheckDescriptorHeapLargeSet() {
EnsureRuntimeContext();
std::array<vk::DescriptorSetLayoutBinding, 3> bindings{};
constexpr std::array stages{
vk::ShaderStageFlagBits::eVertex,
vk::ShaderStageFlagBits::eFragment,
vk::ShaderStageFlagBits::eCompute,
};
for (uint32_t i = 0; i < bindings.size(); i++) {
bindings[i].binding = i;
bindings[i].descriptorType = vk::DescriptorType::eSampler;
bindings[i].descriptorCount = 16;
bindings[i].stageFlags = stages[i];
}
vk::DescriptorSetLayoutCreateInfo create{};
create.sType = vk::StructureType::eDescriptorSetLayoutCreateInfo;
create.bindingCount = static_cast<uint32_t>(bindings.size());
create.pBindings = bindings.data();
vk::DescriptorSetLayout layout = nullptr;
RequireVk("DescriptorHeapLargeSet", "layout",
m_device.createDescriptorSetLayout(&create, nullptr, &layout),
"vkCreateDescriptorSetLayout");
vk::PipelineLayoutCreateInfo pipeline_create{};
pipeline_create.sType = vk::StructureType::ePipelineLayoutCreateInfo;
pipeline_create.setLayoutCount = 1;
pipeline_create.pSetLayouts = &layout;
vk::PipelineLayout pipeline_layout = nullptr;
RequireVk("DescriptorHeapLargeSet", "pipeline layout",
m_device.createPipelineLayout(&pipeline_create, nullptr,
&pipeline_layout),
"vkCreatePipelineLayout");
{
RenderContext context(m_runtime_context);
auto &scheduler = context.GetCommandScheduler();
HW::Context registers{};
HW::UserConfig user_config{};
HW::Shader shaders{};
scheduler.Begin(registers, user_config, shaders);
const auto commit = [&] {
const auto set = context.GetDescriptorHeap().Commit(layout);
Require("DescriptorHeapLargeSet", "allocation", set != nullptr,
"ordinary descriptor heap could not allocate an oversized sampler set");
scheduler.Current().Handle().bindDescriptorSets(
vk::PipelineBindPoint::eCompute, pipeline_layout, 0, 1, &set, 0,
nullptr);
};
for (uint32_t i = 0; i < 22; i++) {
commit();
}
scheduler.Finish();
for (uint32_t i = 0; i < 21; i++) {
commit();
}
}
m_device.destroyPipelineLayout(pipeline_layout, nullptr);
m_device.destroyDescriptorSetLayout(layout, nullptr);
std::printf("[host] %-32s ok\n", "DescriptorHeapLargeSet");
}
void CheckGraphicsPushConstantBank() {
constexpr const char *name = "GraphicsPushConstantStages";
EnsureRuntimeContext();
RenderContext context(m_runtime_context);
auto &scheduler = context.GetCommandScheduler();
HW::Context registers{};
HW::UserConfig user_config{};
HW::Shader shaders{};
scheduler.Begin(registers, user_config, shaders);
ShaderRecompiler::IR::CompiledShaderInfo vertex_program{};
vertex_program.stage = ShaderType::Vertex;
vertex_program.bindings.push_data_start_dword = 2;
vertex_program.info.user_data_registers = {0, 1};
vertex_program.bindings.memory_offset_dword = 2;
ShaderRecompiler::IR::CompiledShaderInfo pixel_program{};
pixel_program.stage = ShaderType::Pixel;
pixel_program.bindings.push_data_start_dword = 0;
pixel_program.info.user_data_registers = {0, 1};
pixel_program.bindings.memory_offset_dword = 2;
ShaderRecompiler::IR::ResourceSnapshot vertex_snapshot;
vertex_snapshot.user_data = {0x11111111u, 0x22222222u};
ShaderStageRuntime vertex_runtime{&vertex_program, &vertex_snapshot};
ShaderRecompiler::IR::ResourceSnapshot pixel_snapshot;
pixel_snapshot.user_data = {0x33333333u, 0x44444444u};
ShaderStageRuntime pixel_runtime{&pixel_program, &pixel_snapshot};
PreparedBindings vertex;
PreparedBindings pixel;
auto &executor = context.GetRenderExecutor();
executor.PrepareBindings(vertex_runtime, vertex);
executor.PrepareBindings(pixel_runtime, pixel);
const auto *vertex_storage = vertex.shader_data.data();
const auto *pixel_storage = pixel.shader_data.data();
vertex.buffer_sources.push_back({});
vertex.buffers.push_back({});
vertex.gds.offset = 1;
vertex.flattened_srt.offset = 1;
vertex.shader_data_buffer.offset = 1;
vertex_snapshot.user_data = {0x55555555u, 0x66666666u};
pixel_snapshot.user_data = {0x77777777u, 0x88888888u};
executor.PrepareBindings(vertex_runtime, vertex);
executor.PrepareBindings(pixel_runtime, pixel);
std::array<PreparedBindings *, 2> stages{&vertex, &pixel};
RenderExecutorTestAccess::PrepareGraphicsBindings(executor, stages, {});
Require(name, "prepared binding scratch",
vertex.shader_data.data() == vertex_storage &&
pixel.shader_data.data() == pixel_storage &&
vertex.buffer_sources.empty() && vertex.buffers.empty() &&
vertex.gds.offset == 0 && vertex.flattened_srt.offset == 0 &&
vertex.shader_data_buffer.offset == 0,
"repeated shader binding preparation allocated or retained stale state");
const auto pipeline = RenderExecutorTestAccess::CommitBindings(
context.GetRenderExecutor(), scheduler.Current(), vertex, pixel);
Require(name, "shared push data",
vertex.shader_data ==
std::vector<uint32_t>{0x55555555u, 0x66666666u} &&
pixel.shader_data ==
std::vector<uint32_t>{0x77777777u, 0x88888888u},
"graphics stages did not commit their shared push data");
scheduler.Finish();
RenderExecutorTestAccess::DestroyDescriptorPipelines(
context.GetRenderExecutor(), std::span {&pipeline, 1u});
std::printf("[host] %-32s ok\n", name);
}
void CheckSchedulerTimeline() {
EnsureRuntimeContext();
CommandScheduler scheduler(Renderer(), m_runtime_context);
HW::Context registers{};
HW::UserConfig user_config{};
HW::Shader shaders{};
scheduler.Begin(registers, user_config, shaders);
auto *command = &scheduler.Current();
const auto original_handle = command->Handle();
HW::Context next_registers{};
HW::UserConfig next_user_config{};
HW::Shader next_shaders{};
scheduler.Begin(next_registers, next_user_config, next_shaders);
Require("SchedulerTimeline", "guest context rebind",
&scheduler.Current() == command &&
command->Handle() == original_handle &&
&command->GetRegisters() == &next_registers &&
&command->GetUserConfig() == &next_user_config &&
&command->GetShaders() == &next_shaders,
"rebinding guest state replaced the active command recording");
scheduler.FlushAndWait();
Require("SchedulerTimeline", "guest context after recording restart",
scheduler.Active() && &scheduler.Current() == command &&
&command->GetRegisters() == &next_registers &&
&command->GetUserConfig() == &next_user_config &&
&command->GetShaders() == &next_shaders,
"restarting command recording lost the current guest context");
scheduler.Begin(registers, user_config, shaders);
const auto first_tick = scheduler.CurrentTick();
uint32_t completed = 0;
scheduler.DeferOperation([owned = std::make_unique<uint32_t>(1),
&completed] { completed = *owned; });
scheduler.Flush();
scheduler.Wait(first_tick);
Require("SchedulerTimeline", "wait boundary",
scheduler.CurrentTick() == first_tick + 1 &&
scheduler.IsFree(first_tick) && completed == 0,
"timeline wait released a resource in the middle of an operation");
scheduler.PopPendingOperations();
Require("SchedulerTimeline", "first tick", completed == 1,
"operation boundary did not release its deferred operation");
const auto second_tick = scheduler.CurrentTick();
std::atomic<bool> priority_in_callback{false};
scheduler.DeferPriorityOperation([owned = std::make_unique<uint32_t>(2),
&completed, &priority_in_callback] {
priority_in_callback = CommandScheduler::InDeferredOperation();
completed = *owned;
});
scheduler.Finish();
scheduler.DrainPriorityOperations();
Require("SchedulerTimeline", "finish",
scheduler.CurrentTick() == second_tick + 1 &&
scheduler.IsFree(second_tick) && completed == 2 &&
priority_in_callback.load() &&
!CommandScheduler::InDeferredOperation(),
"scheduler completion drain lost its tick or callback context");
const auto implicit_flush_tick = scheduler.CurrentTick();
scheduler.DeferOperation([owned = std::make_unique<uint32_t>(3),
&completed] { completed = *owned; });
scheduler.Wait(implicit_flush_tick);
Require("SchedulerTimeline", "implicit flush",
scheduler.CurrentTick() == implicit_flush_tick + 1 &&
scheduler.IsFree(implicit_flush_tick) && completed == 2,
"waiting for the current tick released a deferred resource");
scheduler.PopPendingOperations();
Require("SchedulerTimeline", "implicit boundary", completed == 3,
"operation boundary did not release the implicit tick callback");
vk::SemaphoreTypeCreateInfo timeline_type{};
timeline_type.sType = vk::StructureType::eSemaphoreTypeCreateInfo;
timeline_type.semaphoreType = vk::SemaphoreType::eTimeline;
vk::SemaphoreCreateInfo timeline_create{};
timeline_create.sType = vk::StructureType::eSemaphoreCreateInfo;
timeline_create.pNext = &timeline_type;
vk::Semaphore external_timeline = nullptr;
Require("SchedulerTimeline", "external timeline create",
m_runtime_context.device.createSemaphore(&timeline_create, nullptr,
&external_timeline) ==
vk::Result::eSuccess &&
external_timeline != nullptr,
"failed to create the external timeline semaphore");
constexpr uint64_t external_wait_value = 9;
const auto external_wait_tick = scheduler.CurrentTick();
SubmitInfo external_wait;
external_wait.AddWait(external_timeline, external_wait_value);
constexpr size_t blocked_submission_count = 6;
std::array<vk::CommandBuffer, blocked_submission_count> blocked_handles{};
for (size_t i = 0; i < blocked_submission_count; ++i) {
blocked_handles[i] = scheduler.Current().Handle();
if (i == 0) {
scheduler.Flush(external_wait);
} else {
scheduler.Flush();
}
}
bool distinct_handles = true;
for (size_t i = 0; i < blocked_handles.size(); ++i) {
for (size_t j = i + 1; j < blocked_handles.size(); ++j) {
distinct_handles &= blocked_handles[i] != blocked_handles[j];
}
}
const auto last_blocked_tick = scheduler.CurrentTick() - 1;
Require("SchedulerTimeline", "external timeline overflow",
distinct_handles &&
last_blocked_tick ==
external_wait_tick + blocked_submission_count - 1 &&
!scheduler.IsFree(last_blocked_tick),
"blocked submissions did not grow the command-buffer pool");
vk::SemaphoreSignalInfo signal_info{};
signal_info.sType = vk::StructureType::eSemaphoreSignalInfo;
signal_info.semaphore = external_timeline;
signal_info.value = external_wait_value;
Require("SchedulerTimeline", "external timeline signal",
m_runtime_context.device.signalSemaphore(&signal_info) ==
vk::Result::eSuccess,
"failed to signal the external timeline semaphore");
scheduler.Wait(last_blocked_tick);
m_runtime_context.device.destroySemaphore(external_timeline, nullptr);
scheduler.Flush();
scheduler.Flush();
Require(
"SchedulerTimeline", "timeline pool reuse",
std::ranges::find(blocked_handles, scheduler.Current().Handle()) !=
blocked_handles.end(),
"completed command buffers were not reused after timeline progress");
scheduler.Shutdown();
CommandScheduler draining(Renderer(), m_runtime_context);
HW::Context drain_registers{};
HW::UserConfig drain_user_config{};
HW::Shader drain_shaders{};
draining.Begin(drain_registers, drain_user_config, drain_shaders);
uint32_t reentrant = 0;
draining.DeferOperation([&draining, &reentrant] {
reentrant++;
draining.DeferOperation([&reentrant] { reentrant++; });
});
std::binary_semaphore priority_entered{0};
std::binary_semaphore release_priority{0};
draining.DeferPriorityOperation(
[&draining, &priority_entered, &release_priority] {
priority_entered.release();
draining.Shutdown();
release_priority.acquire();
});
std::jthread shutdown_thread([&draining] { draining.Shutdown(); });
priority_entered.acquire();
std::jthread second_shutdown_thread([&draining] { draining.Shutdown(); });
std::atomic<bool> concurrent_completed{false};
std::jthread concurrent_defer([&draining, &concurrent_completed] {
draining.DeferOperation(
[&concurrent_completed] { concurrent_completed = true; });
});
release_priority.release();
shutdown_thread.join();
second_shutdown_thread.join();
concurrent_defer.join();
Require("SchedulerTimeline", "shutdown drain",
reentrant == 2 && concurrent_completed.load(),
"shutdown lost reentrant or concurrent deferred work");
std::printf("[host] %-32s ok\n", "SchedulerTimeline");
}
void CheckGpuMappedRangeLifecycle() {
EnsureRuntimeContext();
RenderContext context(m_runtime_context);
auto &scheduler = context.GetCommandScheduler();
HW::Context registers{};
HW::UserConfig user_config{};
HW::Shader shaders{};
scheduler.Begin(registers, user_config, shaders);
context.InitializeGpu(nullptr);
auto &resources = context;
constexpr uint64_t base = 0x0000000200000000ull;
constexpr uint64_t page = 0x4000;
const auto cpu_only_tick = scheduler.CurrentTick();
resources.MapMemory(base, page * 4);
resources.MapMemory(base + page * 2, page * 4);
Require("GpuMappedRangeLifecycle", "union",
resources.IsMapped(base, page * 6) &&
!resources.IsMapped(base, page * 7),
"overlapping maps did not form one interval union");
resources.UnmapMemory(base + page * 2, page * 2);
Require("GpuMappedRangeLifecycle", "subtract",
resources.IsMapped(base, page * 2) &&
resources.IsMapped(base + page * 4, page * 2) &&
!resources.IsMapped(base, page * 6),
"partial unmap did not punch the expected interval hole");
resources.UnmapMemory(base + page * 2, page * 2);
Require("GpuMappedRangeLifecycle", "idempotent unmap",
resources.IsMapped(base, page * 2) &&
resources.IsMapped(base + page * 4, page * 2),
"unmapping an absent interval changed neighboring mappings");
resources.UnmapMemory(base, page * 6);
Require("GpuMappedRangeLifecycle", "clear",
!resources.IsMapped(base, page * 6),
"full unmap did not clear the interval union");
constexpr uint64_t old_prt = base + page * 8;
constexpr uint64_t new_prt = base + page * 16;
resources.MapMemory(old_prt, page * 4);
resources.UnmapMemory(old_prt, page * 4);
resources.MapMemory(new_prt, page * 6);
Require("GpuMappedRangeLifecycle", "PRT replacement",
!resources.IsMapped(old_prt, page * 4) &&
resources.IsMapped(new_prt, page * 6) &&
scheduler.CurrentTick() == cpu_only_tick,
"CPU-only unmaps submitted GPU work or lost full PRT coverage");
scheduler.Finish();
context.ShutdownGpu();
std::printf("[host] %-32s ok\n", "GpuMappedRangeLifecycle");
}
void CheckStreamBufferRing() {
EnsureRuntimeContext();
RenderContext context(m_runtime_context);
auto &scheduler = context.GetCommandScheduler();
HW::Context registers{};
HW::UserConfig user_config{};
HW::Shader shaders{};
scheduler.Begin(registers, user_config, shaders);
auto &resources = context;
auto &cache = resources.GetBufferCache();
constexpr std::array<std::pair<MemoryUsage, uint64_t>, 4> utilities{{
{MemoryUsage::Upload, 512ull << 20},
{MemoryUsage::Stream, 64ull << 20},
{MemoryUsage::Download, 64ull << 20},
{MemoryUsage::DeviceLocal, 128ull << 20},
}};
std::array<vk::Buffer, utilities.size()> handles{};
for (size_t i = 0; i < utilities.size(); i++) {
auto &utility = cache.GetUtilityBuffer(utilities[i].first);
handles[i] = utility.Handle();
Require("StreamBufferRing", "utility ownership",
utility.Usage() == utilities[i].first &&
utility.Size() == utilities[i].second &&
utility.Handle() != nullptr,
"BufferCache utility buffer does not match the fixed layout");
}
for (size_t i = 0; i < handles.size(); i++) {
for (size_t j = i + 1; j < handles.size(); j++) {
Require("StreamBufferRing", "utility identity",
handles[i] != handles[j],
"utility usages alias the same native buffer");
}
}
auto *fixed_download = &cache.GetUtilityBuffer(MemoryUsage::Download);
const auto fixed_handle = fixed_download->Handle();
const auto [oversized_download, oversized_offset] =
fixed_download->Map(fixed_download->Size() + 4, 64, false);
const auto [download_probe, download_probe_offset] =
fixed_download->Map(64, 64, false);
Require("StreamBufferRing", "fixed download utility",
oversized_download == nullptr && oversized_offset == 0 &&
download_probe != nullptr && download_probe_offset == 0 &&
&cache.GetUtilityBuffer(MemoryUsage::Download) ==
fixed_download &&
fixed_download->Size() == (64ull << 20) &&
fixed_download->Handle() == fixed_handle,
"oversized download replaced or corrupted the fixed shared ring");
fixed_download->Commit();
const std::array<uint8_t, 16> overflow_data{
0xa5, 0xa5, 0xa5, 0xa5, 0xa5, 0xa5, 0xa5, 0xa5,
0xa5, 0xa5, 0xa5, 0xa5, 0xa5, 0xa5, 0xa5, 0xa5};
auto &upload_stream = cache.GetUtilityBuffer(MemoryUsage::Stream);
const auto stream_offset =
upload_stream.Copy(overflow_data.data(), overflow_data.size(), 16);
Require("StreamBufferRing", "cache-owned stream upload",
upload_stream.Handle() != nullptr &&
stream_offset < upload_stream.Size() &&
std::memcmp(upload_stream.Mapped().data() + stream_offset,
overflow_data.data(), overflow_data.size()) == 0,
"cache-owned stream upload lost bytes");
const auto atom =
m_runtime_context.physical_device_properties.limits.nonCoherentAtomSize;
uint64_t policy_size = 17;
uint64_t policy_alignment = 12;
uint64_t coherent_size = policy_size;
uint64_t coherent_alignment = policy_alignment;
Require("StreamBufferRing", "non-coherent reservation policy",
StreamBufferTestAccess::NormalizeReservation(
false, 256, policy_size, policy_alignment) &&
policy_size == 256 && policy_alignment == 768 &&
StreamBufferTestAccess::NormalizeReservation(
true, 256, coherent_size, coherent_alignment) &&
coherent_size == 17 && coherent_alignment == 12,
"non-coherent reservations are not isolated at the atom/texel LCM");
const auto small_ring_size = std::max<uint64_t>(64, atom * 2);
StreamBuffer stream(m_runtime_context, scheduler, MemoryUsage::Upload,
small_ring_size);
const auto first_size =
stream.IsCoherent() ? small_ring_size - 16 : small_ring_size - atom + 1;
constexpr uint64_t blocked_size = 24;
const auto [first, first_offset] = stream.Map(first_size, 16, false);
Require("StreamBufferRing", "first allocation",
first != nullptr && first_offset == 0,
"initial stream allocation failed");
std::memset(first, 0x5a, static_cast<size_t>(first_size));
stream.Commit();
const auto [oversize, oversize_offset] =
stream.Map(small_ring_size + 1, 1, false);
Require("StreamBufferRing", "oversize",
oversize == nullptr && oversize_offset == 0,
"oversize allocation was accepted");
const auto [blocked, blocked_offset] = stream.Map(blocked_size, 1, false);
const auto [blocked_again, blocked_again_offset] =
stream.Map(blocked_size, 1, false);
Require("StreamBufferRing", "transactional failure",
blocked == nullptr && blocked_offset == 0 &&
blocked_again == nullptr && blocked_again_offset == 0,
"failed wrap mutated stream state");
scheduler.Flush();
const auto [wrapped, wrapped_offset] = stream.Map(blocked_size, 1);
Require("StreamBufferRing", "watched wrap",
wrapped != nullptr && wrapped_offset == 0,
"completed watch did not release the wrapped allocation");
stream.Commit();
const auto download_alignment = atom / std::gcd<uint64_t>(atom, 12) * 12;
const auto download_ring_size = std::max<uint64_t>(
64, download_alignment + std::max<uint64_t>(atom, 16));
StreamBuffer download(m_runtime_context, scheduler, MemoryUsage::Download,
download_ring_size);
const auto [download_data, download_offset] = download.Map(16, 8);
Require("StreamBufferRing", "download allocation",
download_data != nullptr && download_offset == 0,
"download stream is not host visible");
download.Commit();
scheduler.Flush();
const auto [second_download, second_download_offset] = download.Map(16, 12);
Require("StreamBufferRing", "download atom isolation",
second_download != nullptr &&
(download.IsCoherent() ||
(download_offset % atom == 0 &&
second_download_offset % atom == 0 &&
second_download_offset >= download_offset + atom)) &&
second_download_offset % 12 == 0,
"download reservations share a non-coherent atom or lost caller "
"alignment");
download.Commit();
scheduler.Finish();
download.Invalidate(download_offset, 16);
std::printf("[host] %-32s ok\n", "StreamBufferRing");
}
void CheckGpuSuspendPoint() {
constexpr const char *name = "GpuSuspendPoint";
EnsureRuntimeContext();
auto &context = Renderer();
context.InitializeGpu(nullptr);
auto &gpu = context.GetGpu();
auto &scheduler = context.GetCommandScheduler();
CommandProcessor setup(context, 0);
vk::SemaphoreTypeCreateInfo timeline_type{};
timeline_type.sType = vk::StructureType::eSemaphoreTypeCreateInfo;
timeline_type.semaphoreType = vk::SemaphoreType::eTimeline;
vk::SemaphoreCreateInfo timeline_create{};
timeline_create.sType = vk::StructureType::eSemaphoreCreateInfo;
timeline_create.pNext = &timeline_type;
vk::Semaphore gate = nullptr;
Require(name, "timeline create",
m_runtime_context.device.createSemaphore(
&timeline_create, nullptr, &gate) == vk::Result::eSuccess,
"failed to create the native completion gate");
const auto signal = [&](uint64_t value) {
vk::SemaphoreSignalInfo info{};
info.sType = vk::StructureType::eSemaphoreSignalInfo;
info.semaphore = gate;
info.value = value;
Require(name, "timeline signal",
m_runtime_context.device.signalSemaphore(&info) ==
vk::Result::eSuccess,
"failed to release native GPU work");
};
const auto finish = [&] {
gpu.WaitForIdle();
gpu.SendCommandSync([&] {
scheduler.Finish();
scheduler.WaitPriorityOperations(scheduler.CurrentTick() - 1);
});
};
const auto write = [](uint32_t *packet, uint32_t *destination,
uint32_t value, bool predicated = false) {
const auto address = reinterpret_cast<uint64_t>(destination);
packet[0] = KYTY_PM4(5, Pm4::IT_WRITE_DATA, 0) | uint32_t{predicated};
packet[1] = 0;
packet[2] = static_cast<uint32_t>(address);
packet[3] = static_cast<uint32_t>(address >> 32u);
packet[4] = value;
};
const auto cache_ticks = [&] {
std::pair<uint64_t, uint64_t> ticks;
gpu.SendCommandSync([&] {
ticks = {TextureCacheTestAccess::GcTick(context.GetTextureCache()),
BufferCacheTestAccess::GcTick(context.GetBufferCache())};
});
return ticks;
};
const auto initial_cache_ticks = cache_ticks();
uint32_t graphics_value = 0;
uint32_t async_value = 0;
std::array<uint32_t, 5> graphics_list{}, compute_list{};
write(graphics_list.data(), &graphics_value, 17);
write(compute_list.data(), &async_value, 29);
// A frame can contain many lists; completing them must not age either cache.
for (uint32_t i = 0; i < 32; ++i) {
gpu.Submit(graphics_list, {});
gpu.SubmitCompute(0x20, compute_list);
}
finish();
Require(name, "command-list cache lifetime",
graphics_value == 17 && async_value == 29 &&
cache_ticks() == initial_cache_ticks,
"individual graphics or compute lists aged the caches within a frame");
gpu.SendCommandSync([&] {
setup.BufferInit();
SubmitInfo blocked;
blocked.AddWait(gate, 1);
scheduler.Flush(blocked);
});
std::binary_semaphore first_returned{0};
std::jthread first([&] {
gpu.SuspendPoint();
first_returned.release();
});
const bool first_nonblocking =
first_returned.try_acquire_for(std::chrono::seconds(2));
if (!first_nonblocking) {
signal(1);
}
first.join();
Require(name, "first call", first_nonblocking,
"the first suspend point waited for native GPU completion");
// Let the marker reach Vulkan while its completion remains gated.
std::binary_semaphore marker_parsed{0};
std::jthread parser([&] {
gpu.WaitForIdle();
marker_parsed.release();
});
const bool parser_nonblocking =
marker_parsed.try_acquire_for(std::chrono::seconds(2));
if (!parser_nonblocking) {
signal(1);
}
parser.join();
Require(name, "marker insertion", parser_nonblocking,
"inserting a suspend point blocked the GPU command thread");
Require(name, "frame cache maintenance",
cache_ticks() == std::pair{initial_cache_ticks.first + 1,
initial_cache_ticks.second + 1},
"a suspend point did not age both caches exactly once");
std::binary_semaphore second_entered{0};
std::binary_semaphore second_returned{0};
std::jthread second([&] {
second_entered.release();
gpu.SuspendPoint();
second_returned.release();
});
second_entered.acquire();
const bool second_overtook_completion =
second_returned.try_acquire_for(std::chrono::milliseconds(100));
uint32_t compute_value = 0;
std::array<uint32_t, 5> compute_commands{};
write(compute_commands.data(), &compute_value, 33);
bool host_lane_responded = false;
std::binary_semaphore compute_progress{0};
std::jthread compute([&] {
gpu.SubmitCompute(0x20, compute_commands);
gpu.SendCommandSync([&] { host_lane_responded = true; });
gpu.WaitForIdle();
compute_progress.release();
});
const bool compute_did_not_wait =
compute_progress.try_acquire_for(std::chrono::seconds(2));
signal(1);
second.join();
compute.join();
finish();
Require(name, "one outstanding point", !second_overtook_completion,
"a second suspend point returned before the first native completion");
Require(name, "independent queue progress",
compute_did_not_wait && host_lane_responded && compute_value == 33,
"waiting for a previous suspend point blocked compute submissions "
"or the host command lane");
Require(name, "frame count", gpu.GetFrameNum() == 2,
"suspend-point insertion did not advance the frame counter once");
Require(name, "async cache lifetime",
cache_ticks() == std::pair{initial_cache_ticks.first + 2,
initial_cache_ticks.second + 2},
"asynchronous work aged the caches between frame boundaries");
// A marker must stay behind a stalled graphics stream, then clear its
// predicate before the next stream. Async compute releases that stream.
alignas(16) uint64_t predicate = 0;
uint32_t label = 0;
uint32_t prefix = 0;
uint32_t skipped = 0;
uint32_t after_reset = 0;
const auto predicate_address = reinterpret_cast<uint64_t>(&predicate);
const auto label_address = reinterpret_cast<uint64_t>(&label);
std::array<uint32_t, 21> before{};
before[0] = KYTY_PM4(4, Pm4::IT_SET_PREDICATION, 0);
before[1] = (3u << 16u) | (1u << 8u);
before[2] = static_cast<uint32_t>(predicate_address);
before[3] = static_cast<uint32_t>(predicate_address >> 32u);
write(before.data() + 4, &skipped, 11, true);
write(before.data() + 9, &prefix, 22);
before[14] = KYTY_PM4(7, Pm4::IT_WAIT_REG_MEM, 0);
before[15] = 0x10u | 3u;
before[16] = static_cast<uint32_t>(label_address);
before[17] = static_cast<uint32_t>(label_address >> 32u);
before[18] = 1;
before[19] = UINT32_MAX;
std::array<uint32_t, 5> after{};
write(after.data(), &after_reset, 44, true);
std::binary_semaphore lane_entered{0};
std::binary_semaphore release_lane{0};
gpu.SendCommand([&] {
lane_entered.release();
release_lane.acquire();
});
lane_entered.acquire();
gpu.Submit(before, {});
std::binary_semaphore queued_returned{0};
std::jthread queued([&] {
gpu.SuspendPoint();
queued_returned.release();
});
const bool queued_nonblocking =
queued_returned.try_acquire_for(std::chrono::seconds(2));
if (!queued_nonblocking) {
release_lane.release();
gpu.SendCommandSync([&] { label = 1; });
queued.join();
finish();
Require(name, "queued insertion", false,
"suspend-point insertion drained pending guest graphics");
}
queued.join();
gpu.Submit(after, {});
release_lane.release();
bool prefix_reached = false;
bool graphics_ordered = false;
const auto deadline = std::chrono::steady_clock::now() +
std::chrono::seconds(2);
do {
gpu.SendCommandSync([&] {
prefix_reached = prefix == 22;
graphics_ordered = skipped == 0 && after_reset == 0;
});
std::this_thread::yield();
} while (!prefix_reached && std::chrono::steady_clock::now() < deadline);
std::array<uint32_t, 5> release_graphics{};
write(release_graphics.data(), &label, 1);
gpu.SubmitCompute(0x20, release_graphics);
finish();
Require(name, "graphics FIFO",
prefix_reached && graphics_ordered && skipped == 0,
"suspend point or later graphics overtook a stalled graphics stream");
Require(name, "graphics reset", after_reset == 44,
"suspend point preserved the previous frame's predicate");
// Delay the completion callback beyond the owner's shutdown when possible.
// Its credit must remain alive independently of the GuestGpu object.
std::binary_semaphore priority_entered{0};
std::binary_semaphore release_priority{0};
gpu.SendCommandSync([&] {
scheduler.DeferPriorityOperation([&] {
priority_entered.release();
release_priority.acquire();
});
SubmitInfo blocked;
blocked.AddWait(gate, 2);
scheduler.Flush(blocked);
});
gpu.SuspendPoint();
gpu.WaitForIdle();
std::binary_semaphore shutdown_returned{0};
std::jthread shutdown([&] {
context.ShutdownGpu();
shutdown_returned.release();
});
const bool shutdown_overtook_native =
shutdown_returned.try_acquire_for(std::chrono::milliseconds(100));
signal(2);
priority_entered.acquire();
if (!shutdown_overtook_native) {
(void)shutdown_returned.try_acquire_for(std::chrono::seconds(2));
}
release_priority.release();
shutdown.join();
scheduler.DrainPriorityOperations();
m_runtime_context.device.destroySemaphore(gate, nullptr);
Require(name, "shutdown native completion", !shutdown_overtook_native,
"shutdown returned while the suspend point was still executing");
std::printf("[host] %-32s ok\n", name);
}
void CheckGpuCommandLane() {
EnsureRuntimeContext();
auto &context = Renderer();
context.InitializeGpu(nullptr);
auto &gpu = context.GetGpu();
const auto caller_thread = std::this_thread::get_id();
std::thread::id gpu_thread;
std::thread::id nested_thread;
uint32_t order = 0;
gpu.SendCommand([&order] { order = 1; });
gpu.SendCommandSync([&] {
gpu_thread = std::this_thread::get_id();
Require("GpuCommandLane", "FIFO", order == 1,
"synchronous command overtook an older host command");
Require("GpuCommandLane", "GPU context", GuestGpu::IsGpuThread(),
"host command did not run on the GPU thread");
gpu.SendCommandSync(
[&nested_thread] { nested_thread = std::this_thread::get_id(); });
order = 2;
});
Require(
"GpuCommandLane", "dispatch thread",
order == 2 && gpu_thread != caller_thread &&
nested_thread == gpu_thread,
"host command did not run on the GPU thread or nested sync deadlocked");
{
const auto make_release_mem =
[](uint32_t data_sel, uint32_t interrupt_selector, void *destination,
uint64_t value, uint32_t action = 0x28u,
uint32_t gcr_cntl = 1u << 9u) {
std::array<uint32_t, 8> packet{};
const auto address = reinterpret_cast<uint64_t>(destination);
const auto event_index = (action >= 0x2fu ? 6u : 5u);
packet[0] = KYTY_PM4(8, Pm4::IT_NOP, Pm4::R_RELEASE_MEM);
packet[1] = action | (event_index << 8u) | (gcr_cntl << 12u);
packet[2] = (interrupt_selector << 24u) | (data_sel << 29u);
packet[3] = static_cast<uint32_t>(address);
packet[4] = static_cast<uint32_t>(address >> 32u);
packet[5] = static_cast<uint32_t>(value);
packet[6] = static_cast<uint32_t>(value >> 32u);
packet[7] = 0x123u;
return packet;
};
vk::SemaphoreTypeCreateInfo timeline_type{};
timeline_type.sType = vk::StructureType::eSemaphoreTypeCreateInfo;
timeline_type.semaphoreType = vk::SemaphoreType::eTimeline;
vk::SemaphoreCreateInfo timeline_create{};
timeline_create.sType = vk::StructureType::eSemaphoreCreateInfo;
timeline_create.pNext = &timeline_type;
vk::Semaphore blocked_timeline = nullptr;
Require("GpuCommandLane", "blocked timeline create",
m_runtime_context.device.createSemaphore(
&timeline_create, nullptr, &blocked_timeline) ==
vk::Result::eSuccess &&
blocked_timeline != nullptr,
"failed to create the blocked GPU timeline");
constexpr uint64_t release_tick = 1;
auto &gpu_scheduler = context.GetCommandScheduler();
auto processor = std::make_unique<CommandProcessor>(context, 0);
gpu.SendCommandSync([&] {
processor->BufferInit();
SubmitInfo blocked_submit;
blocked_submit.AddWait(blocked_timeline, release_tick);
gpu_scheduler.Flush(blocked_submit);
});
std::binary_semaphore parser_complete{0};
alignas(uint64_t) uint64_t interrupt_only_gds_label = UINT64_MAX;
alignas(uint64_t) uint64_t cb_db_release_label = UINT64_MAX;
bool parser_kept_nonblocking_boundaries = false;
bool host_resource_retired = false;
bool cpu_unmap_did_not_submit = false;
std::jthread no_gpu_wait([&] {
gpu.SendCommandSync([&] {
processor->BufferInit();
const auto tick_before = gpu_scheduler.CurrentTick();
gpu_scheduler.DeferOperation([&] { host_resource_retired = true; });
constexpr uint64_t cpu_only_address = 0x0000000200400000ull;
context.MapMemory(cpu_only_address, 0x4000);
context.UnmapMemory(cpu_only_address, 0x4000);
cpu_unmap_did_not_submit =
gpu_scheduler.CurrentTick() == tick_before &&
!host_resource_retired &&
!context.IsMapped(cpu_only_address, 0x4000);
processor->TriggerEvent(0x16u, 0u);
auto packet = make_release_mem(0, 0, nullptr, 0);
Pm4Execution execution;
const auto result = processor->Process(execution, packet);
processor->IncrementDe();
const bool cache_and_counter_did_not_submit =
result == Pm4ProcessResult::Complete &&
gpu_scheduler.CurrentTick() == tick_before;
const auto cb_db_tick = gpu_scheduler.CurrentTick();
auto cb_db_release =
make_release_mem(2, 0, &cb_db_release_label, 0, 0x14u, 0);
Pm4Execution cb_db_execution;
const auto cb_db_result =
processor->Process(cb_db_execution, cb_db_release);
const bool cb_db_release_did_not_submit =
cb_db_result == Pm4ProcessResult::Complete &&
cb_db_release_label == 0 &&
gpu_scheduler.CurrentTick() == cb_db_tick;
for (const auto gcr : {0u, 1u << 9u}) {
const auto before = Sync::ReadReferenceClock();
auto timestamp =
make_release_mem(3, 0, &cb_db_release_label, UINT64_MAX, 0x14u, gcr);
Pm4Execution timestamp_execution;
const auto timestamp_result =
processor->Process(timestamp_execution, timestamp);
Require("GpuCommandLane", "timestamp write",
timestamp_result == Pm4ProcessResult::Complete &&
cb_db_release_label >= before &&
cb_db_release_label <= Sync::ReadReferenceClock() &&
gpu_scheduler.CurrentTick() == cb_db_tick,
"timestamp write failed, returned an invalid time, or "
"submitted extra GPU work");
}
auto gds_interrupt_only =
make_release_mem(5, 1, &interrupt_only_gds_label, 1ull << 16u);
Pm4Execution gds_interrupt_execution;
const auto interrupt_tick = gpu_scheduler.CurrentTick();
const auto interrupt_result =
processor->Process(gds_interrupt_execution, gds_interrupt_only);
parser_kept_nonblocking_boundaries =
cache_and_counter_did_not_submit &&
cb_db_release_did_not_submit &&
interrupt_result == Pm4ProcessResult::Complete &&
gpu_scheduler.CurrentTick() == interrupt_tick + 1 &&
interrupt_only_gds_label == UINT64_MAX;
});
parser_complete.release();
});
const bool parser_did_not_wait_gpu =
parser_complete.try_acquire_for(std::chrono::seconds(2));
vk::SemaphoreSignalInfo signal_info{};
signal_info.sType = vk::StructureType::eSemaphoreSignalInfo;
signal_info.semaphore = blocked_timeline;
signal_info.value = release_tick;
Require("GpuCommandLane", "blocked timeline signal",
m_runtime_context.device.signalSemaphore(&signal_info) ==
vk::Result::eSuccess,
"failed to release the blocked GPU timeline");
no_gpu_wait.join();
gpu.SendCommandSync([&] { gpu_scheduler.Finish(); });
m_runtime_context.device.destroySemaphore(blocked_timeline, nullptr);
Require("GpuCommandLane", "CPU-only unmap with pending host retirement",
parser_did_not_wait_gpu && cpu_unmap_did_not_submit &&
host_resource_retired,
"CPU-only unmap submitted or waited for unrelated native work");
Require("GpuCommandLane", "nonblocking GPU packets",
parser_did_not_wait_gpu && parser_kept_nonblocking_boundaries,
"a cache event, GL2 writeback, DE counter, or interrupt boundary "
"used the wrong "
"submission behavior");
alignas(uint64_t) uint64_t release_label = 0;
alignas(uint64_t) uint64_t gds_label = UINT64_MAX;
bool release_mem_submission_counts = false;
gpu.SendCommandSync([&] {
processor->BufferInit();
const auto clock_before = Sync::ReadReferenceClock();
processor->WriteAtEndOfPipe64(0, 0, 0x04, 0x38, 5, 4,
&release_label, UINT64_MAX, 2);
Require("GpuCommandLane", "EOP reference clock with interrupt",
release_label >= clock_before &&
release_label <= Sync::ReadReferenceClock(),
"clock write with writeback and interrupt lost its data");
for (const auto interrupt : {0u, 3u, 1u, 2u}) {
release_label = 0;
auto immediate = make_release_mem(1, interrupt, &release_label, 0x11223344u);
Pm4Execution immediate_execution;
const auto immediate_tick = gpu_scheduler.CurrentTick();
const auto immediate_result =
processor->Process(immediate_execution, immediate);
Require("GpuCommandLane", "32-bit release boundary",
immediate_result == Pm4ProcessResult::Complete &&
gpu_scheduler.CurrentTick() ==
immediate_tick + (interrupt == 1 || interrupt == 2) &&
release_label == (interrupt == 1 ? 0 : 0x11223344u),
"label-only release submitted work or interrupt release lost its boundary");
}
auto gds = make_release_mem(5, 0, &gds_label, 1ull << 16u);
Pm4Execution gds_execution;
const auto gds_tick = gpu_scheduler.CurrentTick();
const auto gds_result = processor->Process(gds_execution, gds);
const bool gds_waited_once =
gpu_scheduler.CurrentTick() == gds_tick + 1;
auto interrupt_only = make_release_mem(0, 4, nullptr, 0);
Pm4Execution interrupt_execution;
const auto interrupt_tick = gpu_scheduler.CurrentTick();
const auto interrupt_result =
processor->Process(interrupt_execution, interrupt_only);
const bool interrupt_split_once =
gpu_scheduler.CurrentTick() == interrupt_tick + 1;
auto gds_interrupt = make_release_mem(5, 2, &gds_label, 1ull << 16u);
Pm4Execution gds_interrupt_execution;
const auto gds_interrupt_tick = gpu_scheduler.CurrentTick();
const auto gds_interrupt_result =
processor->Process(gds_interrupt_execution, gds_interrupt);
const bool gds_interrupt_waited_once =
gpu_scheduler.CurrentTick() == gds_interrupt_tick + 1;
release_mem_submission_counts =
gds_result == Pm4ProcessResult::Complete &&
gds_waited_once && interrupt_result == Pm4ProcessResult::Complete &&
interrupt_split_once &&
gds_interrupt_result == Pm4ProcessResult::Complete &&
gds_interrupt_waited_once;
});
gpu.SendCommandSync([&] {
gpu_scheduler.Finish();
gpu_scheduler.WaitPriorityOperations(gpu_scheduler.CurrentTick() - 1);
});
Require("GpuCommandLane", "RELEASE_MEM submission counts",
release_mem_submission_counts &&
static_cast<uint32_t>(release_label) == 0x11223344u &&
static_cast<uint32_t>(gds_label) == 0,
"RELEASE_MEM lost its required split/readback or retained a "
"redundant GPU wait");
}
alignas(uint32_t) uint32_t packet_marker_a = 0;
alignas(uint32_t) uint32_t packet_marker_b = 0;
const auto write_packet = [](uint32_t *packet, uint32_t *destination,
uint32_t value) {
const auto destination_address = reinterpret_cast<uint64_t>(destination);
packet[0] = KYTY_PM4(5, Pm4::IT_WRITE_DATA, 0);
packet[1] = 0;
packet[2] = static_cast<uint32_t>(destination_address);
packet[3] = static_cast<uint32_t>(destination_address >> 32u);
packet[4] = value;
};
uint32_t live_graphics_value = 0;
std::array<uint32_t, 5> live_graphics_commands{};
write_packet(live_graphics_commands.data(), &live_graphics_value, 11);
std::binary_semaphore graphics_gate_entered{0};
std::binary_semaphore graphics_gate_release{0};
gpu.SendCommand([&] {
graphics_gate_entered.release();
graphics_gate_release.acquire();
});
gpu.Submit(live_graphics_commands, {});
graphics_gate_entered.acquire();
live_graphics_commands[4] = 22;
graphics_gate_release.release();
gpu.WaitForIdle();
Require("GpuCommandLane", "borrowed graphics commands",
live_graphics_value == 22,
"graphics submission executed a copied PM4 stream");
uint32_t live_compute_value = 0;
std::array<uint32_t, 5> live_compute_commands{};
write_packet(live_compute_commands.data(), &live_compute_value, 33);
std::binary_semaphore compute_gate_entered{0};
std::binary_semaphore compute_gate_release{0};
gpu.SendCommand([&] {
compute_gate_entered.release();
compute_gate_release.acquire();
});
gpu.SubmitCompute(0x20, live_compute_commands);
compute_gate_entered.acquire();
live_compute_commands[4] = 44;
compute_gate_release.release();
gpu.WaitForIdle();
Require("GpuCommandLane", "borrowed compute commands",
live_compute_value == 44,
"compute submission executed a copied PM4 stream");
LibKernel::EventQueue::KernelEqueue interrupt_queue =
LibKernel::EventQueue::KERNEL_EQUEUE_INVALID;
uint32_t graphics_interrupt_udata = 0;
uint32_t compute_interrupt_udata = 0;
uint32_t other_compute_interrupt_udata = 0;
const bool interrupt_queue_ready =
LibKernel::EventQueue::KernelCreateEqueue(&interrupt_queue,
"pm4-interrupts") == 0 &&
Sync::AddEqEvent(context, interrupt_queue, 0,
&graphics_interrupt_udata) == 0 &&
Sync::AddEqEvent(context, interrupt_queue, 0x20,
&compute_interrupt_udata) == 0 &&
Sync::AddEqEvent(context, interrupt_queue, 0x21,
&other_compute_interrupt_udata) == 0;
Require("GpuCommandLane", "interrupt queue", interrupt_queue_ready,
"failed to register the PM4 interrupt events");
const auto make_interrupt_packet =
[](uint32_t data_selector, uint32_t interrupt_selector,
void *destination, uint64_t value, uint32_t context_id) {
std::array<uint32_t, 8> packet{};
const auto address = reinterpret_cast<uint64_t>(destination);
packet[0] = KYTY_PM4(8, Pm4::IT_NOP, Pm4::R_RELEASE_MEM);
packet[1] = 0x28u | (5u << 8u);
packet[2] = (interrupt_selector << 24u) | (data_selector << 29u);
packet[3] = static_cast<uint32_t>(address);
packet[4] = static_cast<uint32_t>(address >> 32u);
packet[5] = static_cast<uint32_t>(value);
packet[6] = static_cast<uint32_t>(value >> 32u);
packet[7] = context_id & 0x07ffffffu;
return packet;
};
const auto finish_gpu = [&] {
gpu.SendCommandSync([&] {
auto &scheduler = context.GetCommandScheduler();
scheduler.Finish();
scheduler.WaitPriorityOperations(scheduler.CurrentTick() - 1);
});
};
const auto wait_for_interrupt =
[&](LibKernel::EventQueue::KernelEvent &event, int &count) {
LibKernel::KernelUseconds timeout = 0;
return LibKernel::EventQueue::KernelWaitEqueue(
interrupt_queue, &event, 1, &count, &timeout);
};
uint32_t no_interrupt_graphics_value = 0;
std::array<uint32_t, 5> no_interrupt_graphics_commands{};
write_packet(no_interrupt_graphics_commands.data(),
&no_interrupt_graphics_value, 55);
gpu.Submit(no_interrupt_graphics_commands, {});
gpu.WaitForIdle();
finish_gpu();
LibKernel::EventQueue::KernelEvent interrupt_event{};
int interrupt_count = 0;
const auto no_graphics_interrupt_wait =
wait_for_interrupt(interrupt_event, interrupt_count);
Require("GpuCommandLane", "no implicit graphics interrupt",
no_graphics_interrupt_wait == LibKernel::KERNEL_ERROR_ETIMEDOUT &&
interrupt_count == 0 && no_interrupt_graphics_value == 55,
"graphics submission completion synthesized an interrupt");
auto live_interrupt_commands =
make_interrupt_packet(0, 1, nullptr, 0, 0x234u);
std::binary_semaphore interrupt_gate_entered{0};
std::binary_semaphore interrupt_gate_release{0};
gpu.SendCommand([&] {
interrupt_gate_entered.release();
interrupt_gate_release.acquire();
});
gpu.SubmitCompute(0x20, live_interrupt_commands);
interrupt_gate_entered.acquire();
live_interrupt_commands[2] = 0;
interrupt_gate_release.release();
gpu.WaitForIdle();
finish_gpu();
interrupt_count = 0;
const auto no_interrupt_wait =
wait_for_interrupt(interrupt_event, interrupt_count);
Require("GpuCommandLane", "no implicit completion interrupt",
no_interrupt_wait == LibKernel::KERNEL_ERROR_ETIMEDOUT &&
interrupt_count == 0,
"submission completion synthesized an interrupt absent from the "
"executed PM4 stream");
uint32_t graphics_interrupt_label = 0xa5a5a5a5u;
auto graphics_interrupt_commands = make_interrupt_packet(
1, 1, &graphics_interrupt_label, 0x11223344u, 0);
gpu.Submit(graphics_interrupt_commands, {});
gpu.WaitForIdle();
finish_gpu();
interrupt_count = 0;
const auto graphics_interrupt_wait =
wait_for_interrupt(interrupt_event, interrupt_count);
Require("GpuCommandLane", "graphics interrupt routing",
graphics_interrupt_wait == 0 && interrupt_count == 1 &&
interrupt_event.ident == 0 && interrupt_event.data == 0 &&
interrupt_event.udata == &graphics_interrupt_udata &&
graphics_interrupt_label == 0xa5a5a5a5u,
"graphics kOnly interrupt was misrouted or wrote its label");
uint32_t compute_interrupt_label = 0x5a5a5a5au;
auto compute_interrupt_commands = make_interrupt_packet(
1, 0, &compute_interrupt_label, 0x55667788u, 0x456u);
std::binary_semaphore compute_interrupt_gate_entered{0};
std::binary_semaphore compute_interrupt_gate_release{0};
gpu.SendCommand([&] {
compute_interrupt_gate_entered.release();
compute_interrupt_gate_release.acquire();
});
gpu.SubmitCompute(0x20, compute_interrupt_commands);
compute_interrupt_gate_entered.acquire();
compute_interrupt_commands[2] |= 1u << 24u;
compute_interrupt_gate_release.release();
gpu.WaitForIdle();
finish_gpu();
interrupt_count = 0;
const auto compute_interrupt_wait =
wait_for_interrupt(interrupt_event, interrupt_count);
const bool compute_interrupt_matches =
compute_interrupt_wait == 0 && interrupt_count == 1 &&
interrupt_event.ident == 0x20 && interrupt_event.data == 0x456u &&
interrupt_event.udata == &compute_interrupt_udata &&
compute_interrupt_label == 0x55667788u;
uint64_t compute_clock_label = 0;
auto compute_clock_commands = make_interrupt_packet(
3, 1, &compute_clock_label, 0, 0x567u);
gpu.SubmitCompute(0x20, compute_clock_commands);
gpu.WaitForIdle();
finish_gpu();
interrupt_count = 0;
const auto compute_clock_wait =
wait_for_interrupt(interrupt_event, interrupt_count);
const bool compute_clock_matches =
compute_clock_wait == 0 && interrupt_count == 1 &&
interrupt_event.ident == 0x20 && interrupt_event.data == 0x567u &&
interrupt_event.udata == &compute_interrupt_udata &&
compute_clock_label != 0;
constexpr uint64_t compute_done_value = 0x1234567887654321ull;
uint64_t compute_done_label = 0;
auto compute_done_commands = make_interrupt_packet(
2, 2, &compute_done_label, compute_done_value, 0x678u);
compute_done_commands[1] = 0x62fu; // CS_DONE, shader-done index, no GCR action.
gpu.SubmitCompute(0x20, compute_done_commands);
gpu.WaitForIdle();
finish_gpu();
interrupt_count = 0;
const auto compute_done_wait =
wait_for_interrupt(interrupt_event, interrupt_count);
Require("GpuCommandLane", "CS_DONE write-confirm interrupt",
compute_done_wait == 0 && interrupt_count == 1 &&
interrupt_event.ident == 0x20 && interrupt_event.data == 0x678u &&
interrupt_event.udata == &compute_interrupt_udata &&
compute_done_label == compute_done_value,
"CS_DONE lost its full 64-bit label write or write-confirm interrupt");
interrupt_count = 0;
const auto extra_interrupt_wait =
wait_for_interrupt(interrupt_event, interrupt_count);
const auto delete_other_compute_event =
Sync::DeleteEqEvent(context, interrupt_queue, 0x21);
LibKernel::EventQueue::KernelDeleteEqueue(interrupt_queue);
context.TriggerInterrupt(0, 0);
context.TriggerInterrupt(0x20, 0);
context.TriggerInterrupt(0x21, 0);
Require("GpuCommandLane", "compute interrupt routing",
compute_interrupt_matches && compute_clock_matches &&
extra_interrupt_wait == LibKernel::KERNEL_ERROR_ETIMEDOUT &&
interrupt_count == 0 && delete_other_compute_event == 0,
"compute kOnly interrupt was misrouted, broadcast, or lost a "
"label write");
std::array<uint32_t, 1024> polling_loop{};
for (size_t i = 0; i < polling_loop.size() - 4; i += 2) {
polling_loop[i] = KYTY_PM4(2, Pm4::IT_NOP, Pm4::R_ZERO);
polling_loop[i + 1] = 0;
}
const auto loop_address = reinterpret_cast<uint64_t>(polling_loop.data());
polling_loop[1020] = KYTY_PM4(4, Pm4::IT_INDIRECT_BUFFER, 0);
polling_loop[1021] = static_cast<uint32_t>(loop_address);
polling_loop[1022] = static_cast<uint32_t>(loop_address >> 32u);
polling_loop[1023] =
0x0f200000u | static_cast<uint32_t>(polling_loop.size());
std::array<uint32_t, 14> polling_commands{};
write_packet(polling_commands.data(), &packet_marker_a, 11);
polling_commands[5] = KYTY_PM4(4, Pm4::IT_INDIRECT_BUFFER, 0);
polling_commands[6] = static_cast<uint32_t>(loop_address);
polling_commands[7] = static_cast<uint32_t>(loop_address >> 32u);
polling_commands[8] =
0x0f200000u | static_cast<uint32_t>(polling_loop.size());
write_packet(polling_commands.data() + 9, &packet_marker_b, 22);
std::binary_semaphore stream_gate_entered{0};
std::binary_semaphore stream_gate_release{0};
gpu.SendCommand([&] {
stream_gate_entered.release();
stream_gate_release.acquire();
});
gpu.Submit(polling_commands, {});
stream_gate_entered.acquire();
stream_gate_release.release();
uint32_t packet_marker_a_at_callback = UINT32_MAX;
uint32_t packet_marker_b_at_callback = UINT32_MAX;
for (uint32_t attempt = 0; attempt < 8 && packet_marker_a_at_callback != 11;
attempt++) {
gpu.SendCommandSync([&] {
if (packet_marker_a == 11 && packet_marker_b == 0) {
packet_marker_a_at_callback = packet_marker_a;
packet_marker_b_at_callback = packet_marker_b;
polling_commands[13] = 33;
polling_loop[1020] = KYTY_PM4(2, Pm4::IT_NOP, Pm4::R_ZERO);
polling_loop[1021] = 0;
polling_loop[1022] = KYTY_PM4(2, Pm4::IT_NOP, Pm4::R_ZERO);
polling_loop[1023] = 0;
}
});
std::this_thread::yield();
}
gpu.WaitForIdle();
Require("GpuCommandLane", "packet-boundary command polling",
packet_marker_a_at_callback == 11 &&
packet_marker_b_at_callback == 0 && packet_marker_a == 11 &&
packet_marker_b == 33,
"host command waited for an entire PM4 stream or resumed a copied "
"stream");
uint32_t label = 0;
uint32_t prefix = 0;
uint32_t suffix = 0;
const auto address = [](const void *value) {
return reinterpret_cast<uint64_t>(value);
};
std::array<uint32_t, 17> commands{};
commands[0] = KYTY_PM4(5, Pm4::IT_WRITE_DATA, 0);
commands[1] = 0;
commands[2] = static_cast<uint32_t>(address(&prefix));
commands[3] = static_cast<uint32_t>(address(&prefix) >> 32u);
commands[4] = 11;
commands[5] = KYTY_PM4(7, Pm4::IT_WAIT_REG_MEM, 0);
commands[6] = 0x10u | 3u;
commands[7] = static_cast<uint32_t>(address(&label));
commands[8] = static_cast<uint32_t>(address(&label) >> 32u);
commands[9] = 1;
commands[10] = UINT32_MAX;
commands[12] = KYTY_PM4(5, Pm4::IT_WRITE_DATA, 0);
commands[13] = 0;
commands[14] = static_cast<uint32_t>(address(&suffix));
commands[15] = static_cast<uint32_t>(address(&suffix) >> 32u);
commands[16] = 22;
gpu.Submit(commands, {});
std::binary_semaphore ordered_started{0};
std::atomic<bool> ordered_finished{false};
uint32_t ordered_suffix = 0;
std::jthread ordered([&] {
ordered_started.release();
gpu.WaitForIdle();
ordered_suffix = suffix;
ordered_finished = true;
});
ordered_started.acquire();
gpu.SendCommandSync([&] {
Require("GpuCommandLane", "guest queue idle fence", !ordered_finished.load(),
"idle fence returned before a queued submission");
label = 1;
});
ordered.join();
Require("GpuCommandLane", "ordered completion",
prefix == 11 && suffix == 22 && ordered_suffix == 22 &&
ordered_finished.load(),
"idle fence did not drain prior PM4 work");
auto &resources = context;
constexpr uint64_t empty_unmap_base = 0x0000000200400000ull;
constexpr uint64_t empty_unmap_size = 0x4000;
resources.MapMemory(empty_unmap_base, empty_unmap_size);
label = 0;
prefix = 0;
suffix = 0;
gpu.Submit(commands, {});
std::binary_semaphore unmap_complete{0};
std::jthread unmap_thread([&] {
resources.UnmapMemory(empty_unmap_base, empty_unmap_size);
unmap_complete.release();
});
const bool unmap_returned =
unmap_complete.try_acquire_for(std::chrono::seconds(2));
gpu.SendCommandSync([&] { label = 1; });
if (!unmap_returned) {
unmap_complete.acquire();
}
unmap_thread.join();
gpu.WaitForIdle();
Require("GpuCommandLane", "unmap queue progress",
unmap_returned &&
!resources.IsMapped(empty_unmap_base, empty_unmap_size) &&
prefix == 11 && suffix == 22,
"an unrelated unmap waited for a blocked PM4 submission");
auto &scheduler = context.GetCommandScheduler();
std::atomic<bool> completion_published{false};
gpu.SendCommandSync([&] {
scheduler.DeferPriorityOperation([&] { completion_published = true; });
});
resources.MapMemory(empty_unmap_base, empty_unmap_size);
resources.UnmapMemory(empty_unmap_base, empty_unmap_size);
Require("GpuCommandLane", "unmap native completion",
completion_published.load() &&
!resources.IsMapped(empty_unmap_base, empty_unmap_size),
"unmap returned before an earlier native guest-memory callback");
std::binary_semaphore priority_entered{0};
std::binary_semaphore release_priority{0};
gpu.SendCommandSync([&] {
scheduler.DeferPriorityOperation([&] {
priority_entered.release();
release_priority.acquire();
});
scheduler.Flush();
});
priority_entered.acquire();
resources.MapMemory(empty_unmap_base, empty_unmap_size);
std::binary_semaphore priority_unmap_entered{0};
std::binary_semaphore priority_unmap_complete{0};
std::jthread priority_unmap_thread([&] {
gpu.SendCommandSync([&] {
priority_unmap_entered.release();
resources.UnmapMemory(empty_unmap_base, empty_unmap_size);
});
priority_unmap_complete.release();
});
priority_unmap_entered.acquire();
const bool unmap_overtook_priority =
priority_unmap_complete.try_acquire_for(std::chrono::seconds(1));
release_priority.release();
if (!unmap_overtook_priority) {
priority_unmap_complete.acquire();
}
priority_unmap_thread.join();
Require("GpuCommandLane", "unmap priority ordering",
!unmap_overtook_priority &&
!resources.IsMapped(empty_unmap_base, empty_unmap_size),
"unmap returned before an earlier guest-memory callback");
constexpr uintptr_t fault_base = 0x0000000200500000ull;
constexpr uint64_t fault_size = 0x10000;
int64_t fault_direct_offset = -1;
Require("GpuCommandLane", "processor fault direct allocation",
Libs::LibKernel::Memory::KernelAllocateDirectMemory(
0, Libs::LibKernel::Memory::KernelGetDirectMemorySize(),
fault_size, fault_size, 0, &fault_direct_offset) == 0,
"processor-fault direct-memory allocation failed");
void *fault_memory = reinterpret_cast<void *>(fault_base);
Require("GpuCommandLane", "processor fault direct mapping",
Libs::LibKernel::Memory::KernelMapDirectMemory(
&fault_memory, fault_size, 0x3, 0x10, fault_direct_offset,
fault_size) == 0,
"processor-fault fixed direct-memory mapping failed");
Require("GpuCommandLane", "processor fault allocation",
fault_memory == reinterpret_cast<void *>(fault_base),
"fixed processor-fault allocation failed");
resources.MapMemory(fault_base, fault_size);
constexpr uint64_t immediate_dst = fault_base + 0x1000;
constexpr uint64_t immediate_memory_dst = fault_base + 0x2000;
constexpr uint64_t memory_src = fault_base + 0x4000;
constexpr uint64_t memory_copy_dst = fault_base + 0x6000;
constexpr uint64_t memory_dst = fault_base + 0x8000;
constexpr uint64_t l2_copy_dst = fault_base + 0xa000;
constexpr uint64_t nowhere_dst = fault_base + 0xc000;
constexpr uint64_t clean_cached_fill = fault_base + 0x3000;
constexpr uint64_t clean_cached_copy = fault_base + 0x3008;
constexpr uint64_t clean_cached_source = fault_base + 0x3010;
constexpr uint64_t clean_cached_readback = fault_base + 0xb000;
constexpr std::array<uint32_t, 2> source_words{0x11223344u, 0x55667788u};
constexpr std::array<uint32_t, 2> nowhere_words{0x89abcdefu, 0x01234567u};
constexpr std::array<uint32_t, 2> clean_copy_words{0x13579bdfu,
0x2468ace0u};
std::memcpy(reinterpret_cast<void *>(memory_src), source_words.data(),
sizeof(source_words));
std::memcpy(reinterpret_cast<void *>(nowhere_dst), nowhere_words.data(),
sizeof(nowhere_words));
std::memcpy(reinterpret_cast<void *>(clean_cached_source),
clean_copy_words.data(), sizeof(clean_copy_words));
std::array<uint32_t, 56> dma_commands{};
size_t dma_cursor = 0;
const auto append_dma = [&](uint8_t dst_sel, uint64_t dst, uint8_t src_sel,
uint64_t src, uint32_t bytes,
uint32_t command_flags = 0) {
auto *packet = dma_commands.data() + dma_cursor;
dma_cursor += 7;
packet[0] = KYTY_PM4(7, Pm4::IT_DMA_DATA, 0);
packet[1] = (static_cast<uint32_t>(dst_sel) << 20u) |
(static_cast<uint32_t>(src_sel) << 29u);
packet[2] = static_cast<uint32_t>(src);
packet[3] = static_cast<uint32_t>(src >> 32u);
packet[4] = static_cast<uint32_t>(dst);
packet[5] = static_cast<uint32_t>(dst >> 32u);
packet[6] = bytes | command_flags;
};
constexpr uint64_t gds_immediate_offset = 0;
constexpr uint64_t gds_memory_offset = 0x10;
constexpr uint32_t immediate_value = 0xa1b2c3d4u;
append_dma(1, gds_immediate_offset, 2, immediate_value,
sizeof(source_words));
append_dma(0, immediate_dst, 1, gds_immediate_offset, sizeof(source_words));
append_dma(1, gds_memory_offset, 0, memory_src, sizeof(source_words));
append_dma(0, memory_dst, 1, gds_memory_offset, sizeof(source_words));
append_dma(0, immediate_memory_dst, 2, immediate_value,
sizeof(source_words));
append_dma(0, memory_copy_dst, 0, memory_src, sizeof(source_words));
append_dma(3, l2_copy_dst, 3, memory_src, sizeof(source_words));
append_dma(2, nowhere_dst, 3, nowhere_dst, sizeof(nowhere_words),
1u << 31u);
Require("GpuCommandLane", "DMA_DATA packet assembly",
dma_cursor == dma_commands.size(),
"DMA_DATA packet stream has the wrong size");
gpu.Submit(dma_commands, {});
gpu.WaitForIdle();
constexpr uint32_t clean_fill_value = 0xdecafbad;
gpu.SendCommandSync([&] {
auto &buffer_cache = resources.GetBufferCache();
Require("GpuCommandLane", "clean cached setup",
buffer_cache.IsRegionRegistered(clean_cached_fill,
sizeof(source_words)) &&
!buffer_cache.HasGpuDirtyBytes(nowhere_dst,
sizeof(nowhere_words)) &&
buffer_cache.HasGpuDirtyBytes(immediate_dst,
sizeof(source_words)),
"DMA prefetch modified ownership or did not establish a dirty "
"cached page");
buffer_cache.FillBuffer(clean_cached_fill, sizeof(source_words),
clean_fill_value, false);
buffer_cache.CopyBuffer(clean_cached_copy, clean_cached_source,
sizeof(clean_copy_words), false, false);
Require("GpuCommandLane", "clean cached DMA ownership",
!buffer_cache.HasGpuDirtyBytes(clean_cached_fill,
sizeof(source_words)) &&
!buffer_cache.HasGpuDirtyBytes(clean_cached_copy,
sizeof(clean_copy_words)) &&
buffer_cache.HasGpuDirtyBytes(immediate_dst,
sizeof(source_words)),
"clean cached DMA acquired ownership or released a byte-disjoint "
"dirty sibling");
constexpr uint64_t gds_clean_mirror = 0x20;
buffer_cache.CopyBuffer(gds_clean_mirror, clean_cached_fill,
2 * sizeof(source_words), true, false);
buffer_cache.CopyBuffer(clean_cached_readback, gds_clean_mirror,
2 * sizeof(source_words), false, true);
});
Require("GpuCommandLane", "DMA_DATA immediate GDS readback",
resources.HandleFault(PageFaultAccess::Read, immediate_dst),
"immediate-to-GDS-to-memory copy did not publish GPU bytes");
Require("GpuCommandLane", "DMA_DATA memory GDS readback",
resources.HandleFault(PageFaultAccess::Read, memory_dst),
"memory-to-GDS-to-memory copy did not publish GPU bytes");
Require("GpuCommandLane", "DMA_DATA immediate memory readback",
resources.HandleFault(PageFaultAccess::Read, immediate_memory_dst),
"immediate-to-memory copy did not publish GPU bytes");
Require("GpuCommandLane", "DMA_DATA memory copy readback",
resources.HandleFault(PageFaultAccess::Read, memory_copy_dst),
"memory-to-memory copy did not publish GPU bytes");
Require("GpuCommandLane", "DMA_DATA L2 readback",
resources.HandleFault(PageFaultAccess::Read, l2_copy_dst),
"MemoryUsingL2 copy did not publish GPU bytes");
Require("GpuCommandLane", "clean cached mirror readback",
resources.HandleFault(PageFaultAccess::Read, clean_cached_readback),
"clean host DMA was not reflected in the cached buffer");
std::array<uint32_t, 2> immediate_words{};
std::array<uint32_t, 2> copied_words{};
std::array<uint32_t, 2> immediate_memory_words{};
std::array<uint32_t, 2> memory_copy_words{};
std::array<uint32_t, 2> l2_copy_words{};
std::array<uint32_t, 4> clean_cached_words{};
std::memcpy(immediate_words.data(),
reinterpret_cast<const void *>(immediate_dst),
sizeof(immediate_words));
std::memcpy(copied_words.data(), reinterpret_cast<const void *>(memory_dst),
sizeof(copied_words));
std::memcpy(immediate_memory_words.data(),
reinterpret_cast<const void *>(immediate_memory_dst),
sizeof(immediate_memory_words));
std::memcpy(memory_copy_words.data(),
reinterpret_cast<const void *>(memory_copy_dst),
sizeof(memory_copy_words));
std::memcpy(l2_copy_words.data(),
reinterpret_cast<const void *>(l2_copy_dst),
sizeof(l2_copy_words));
std::memcpy(clean_cached_words.data(),
reinterpret_cast<const void *>(clean_cached_readback),
sizeof(clean_cached_words));
Require("GpuCommandLane", "DMA_DATA immediate GDS contents",
immediate_words ==
std::array<uint32_t, 2>{immediate_value, immediate_value},
"immediate-to-GDS-to-memory bytes do not match");
Require("GpuCommandLane", "DMA_DATA memory GDS contents",
copied_words == source_words,
"memory-to-GDS-to-memory bytes do not match");
Require("GpuCommandLane", "DMA_DATA immediate memory contents",
immediate_memory_words ==
std::array<uint32_t, 2>{immediate_value, immediate_value},
"immediate-to-memory bytes do not match");
Require("GpuCommandLane", "DMA_DATA memory copy contents",
memory_copy_words == source_words,
"memory-to-memory bytes do not match");
Require("GpuCommandLane", "DMA_DATA L2 contents",
l2_copy_words == source_words, "MemoryUsingL2 bytes do not match");
Require("GpuCommandLane", "DMA_DATA nowhere contents",
std::memcmp(reinterpret_cast<const void *>(nowhere_dst),
nowhere_words.data(), sizeof(nowhere_words)) == 0,
"Nowhere destination modified guest memory");
Require("GpuCommandLane", "clean cached DMA contents",
clean_cached_words == std::array<uint32_t, 4>{clean_fill_value,
clean_fill_value,
clean_copy_words[0],
clean_copy_words[1]},
"host-memory fill/copy did not update the clean cached mirror");
gpu.SendCommandSync([&] {
Require("GpuCommandLane", "GPU memory invalidation",
resources.InvalidateMemory(fault_base, sizeof(uint32_t)),
"GPU memory invalidation did not find its mapped range");
});
auto &buffer_cache = resources.GetBufferCache();
Require("GpuCommandLane", "clean tracked fault setup",
buffer_cache.IsRegionRegistered(immediate_dst, sizeof(uint32_t)) &&
!buffer_cache.HasGpuDirtyBytes(immediate_dst, sizeof(uint32_t)),
"clean write-fault test address is not backed by a clean cached "
"Buffer");
constexpr uint64_t dirty_fault_address = fault_base + 0xd000;
constexpr uint32_t dirty_fault_value = 0x5a17c0deu;
constexpr uint32_t dirty_fault_stale = 0x0ddba11u;
std::memcpy(reinterpret_cast<void *>(dirty_fault_address),
&dirty_fault_stale, sizeof(dirty_fault_stale));
gpu.SendCommandSync([&] {
auto dirty = buffer_cache.ObtainBuffer(
dirty_fault_address, sizeof(dirty_fault_value), true, false);
Require("GpuCommandLane", "dirty tracked fault setup",
dirty.first != nullptr,
"dirty write-fault test could not create a cached Buffer");
buffer_cache.FillBuffer(dirty_fault_address, sizeof(dirty_fault_value),
dirty_fault_value, false);
});
Require("GpuCommandLane", "dirty tracked fault ownership",
buffer_cache.HasGpuDirtyBytes(dirty_fault_address,
sizeof(dirty_fault_value)),
"dirty write-fault test address did not acquire GPU ownership");
std::binary_semaphore block_entered{0};
std::binary_semaphore block_release{0};
std::binary_semaphore clean_fault_complete{0};
bool clean_fault_handled = false;
gpu.SendCommand([&] {
block_entered.release();
block_release.acquire();
});
block_entered.acquire();
std::jthread clean_fault_thread([&] {
clean_fault_handled =
resources.HandleFault(PageFaultAccess::Write, immediate_dst);
clean_fault_complete.release();
});
const bool clean_fault_was_direct =
clean_fault_complete.try_acquire_for(std::chrono::seconds(1));
if (!clean_fault_was_direct) {
block_release.release();
clean_fault_complete.acquire();
}
clean_fault_thread.join();
Require("GpuCommandLane", "clean fault direct path",
clean_fault_was_direct && clean_fault_handled,
"a clean CPU write fault waited for the blocked GPU command lane");
std::binary_semaphore dirty_fault_complete{0};
bool dirty_fault_handled = false;
std::jthread dirty_fault_thread([&] {
dirty_fault_handled =
resources.HandleFault(PageFaultAccess::Write, dirty_fault_address);
dirty_fault_complete.release();
});
const bool dirty_fault_bypassed_lane =
dirty_fault_complete.try_acquire_for(std::chrono::milliseconds(200));
block_release.release();
if (!dirty_fault_bypassed_lane) {
dirty_fault_complete.acquire();
}
dirty_fault_thread.join();
uint32_t dirty_fault_backing = 0;
std::memcpy(&dirty_fault_backing,
reinterpret_cast<const void *>(dirty_fault_address),
sizeof(dirty_fault_backing));
Require("GpuCommandLane", "dirty fault synchronization",
!dirty_fault_bypassed_lane && dirty_fault_handled &&
dirty_fault_backing == dirty_fault_value &&
!buffer_cache.HasGpuDirtyBytes(dirty_fault_address,
sizeof(dirty_fault_value)),
"a GPU-dirty write fault bypassed synchronization or lost the "
"published bytes");
constexpr uint64_t empty_copy_fault_address = fault_base + 0xe000;
constexpr uint32_t empty_copy_fault_value = 0x6b28d1efu;
gpu.SendCommandSync([&] {
auto dirty = buffer_cache.ObtainBuffer(empty_copy_fault_address,
sizeof(empty_copy_fault_value),
true, false);
Require("GpuCommandLane", "empty-copy write setup",
dirty.first != nullptr,
"empty-copy write test could not create a cached Buffer");
buffer_cache.FillBuffer(empty_copy_fault_address,
sizeof(empty_copy_fault_value),
empty_copy_fault_value, false);
buffer_cache.ReadMemory(empty_copy_fault_address,
sizeof(empty_copy_fault_value));
const bool gpu_owned = buffer_cache.IsRegionGpuModified(
empty_copy_fault_address, sizeof(empty_copy_fault_value));
const bool cpu_owned = buffer_cache.IsRegionCpuModified(
empty_copy_fault_address, sizeof(empty_copy_fault_value));
Require("GpuCommandLane", "empty-copy read ownership",
!gpu_owned && !cpu_owned,
"readback did not leave clean ownership for the queued write "
"invalidation");
buffer_cache.ReadMemory(empty_copy_fault_address,
sizeof(empty_copy_fault_value), true);
});
uint32_t empty_copy_fault_backing = 0;
std::memcpy(&empty_copy_fault_backing,
reinterpret_cast<const void *>(empty_copy_fault_address),
sizeof(empty_copy_fault_backing));
const bool empty_copy_cpu_owned = buffer_cache.IsRegionCpuModified(
empty_copy_fault_address, sizeof(empty_copy_fault_value));
const bool empty_copy_gpu_owned = buffer_cache.IsRegionGpuModified(
empty_copy_fault_address, sizeof(empty_copy_fault_value));
const bool empty_copy_dirty = buffer_cache.HasGpuDirtyBytes(
empty_copy_fault_address, sizeof(empty_copy_fault_value));
Require("GpuCommandLane", "empty-copy write ownership",
empty_copy_fault_backing == empty_copy_fault_value &&
empty_copy_cpu_owned && !empty_copy_gpu_owned &&
!empty_copy_dirty,
"write invalidation without a remaining copy did not establish CPU "
"ownership");
resources.UnmapMemory(fault_base, fault_size);
Require("GpuCommandLane", "processor fault unmap",
Libs::LibKernel::Memory::KernelMunmap(fault_base, fault_size) == 0,
"processor-fault direct-memory mapping release failed");
Require("GpuCommandLane", "processor fault free",
Libs::LibKernel::Memory::KernelReleaseDirectMemory(
fault_direct_offset, fault_size) == 0,
"processor-fault direct-memory allocation release failed");
std::binary_semaphore command_entered{0};
std::binary_semaphore release_command{0};
std::atomic<bool> shutdown_complete{false};
bool shutdown_owner_visible = false;
resources.MapMemory(empty_unmap_base, empty_unmap_size);
gpu.SendCommand([&] {
command_entered.release();
release_command.acquire();
shutdown_owner_visible = &context.GetGpu() == &gpu;
});
command_entered.acquire();
std::jthread shutdown_thread([&] {
context.ShutdownGpu();
shutdown_complete = true;
});
while (!gpu.IsStopping()) {
std::this_thread::yield();
}
Require("GpuCommandLane", "owned shutdown", !shutdown_complete.load(),
"GPU owner returned before an in-flight command completed");
release_command.release();
shutdown_thread.join();
Require(
"GpuCommandLane", "owned shutdown completion",
shutdown_complete.load() && shutdown_owner_visible,
"GPU owner did not remain available while draining queued work");
resources.UnmapMemory(empty_unmap_base, empty_unmap_size);
Require("GpuCommandLane", "unmap after shutdown",
!resources.IsMapped(empty_unmap_base, empty_unmap_size),
"renderer retained mapped coverage after the GPU command lane stopped");
context.ShutdownGpu();
std::printf("[host] %-32s ok\n", "GpuCommandLane");
}
void CheckUnifiedImageViewCache() {
EnsureRuntimeContext();
constexpr const char *name = "UnifiedImageViewCache";
CommandScheduler scheduler(Renderer(), m_runtime_context);
ImageInfo color_info{};
color_info.pixel_format = vk::Format::eR8G8B8A8Unorm;
color_info.guest_format = Prospero::BufferFormat::k8_8_8_8UNorm;
color_info.type = Prospero::ImageType::kColor2D;
color_info.extent = {8, 8, 1};
color_info.resources = {2, 2};
color_info.pitch = 8;
color_info.bytes_per_block = 4;
color_info.samples = 1;
color_info.tile_mode = Prospero::TileMode::kLinear;
color_info.mip_layout[0] = {0, 512, 8, 8};
color_info.mip_layout[1] = {512, 128, 4, 4};
Libs::Graphics::Image color(m_runtime_context, scheduler, color_info);
const vk::ComponentMapping identity{};
const vk::ComponentMapping bgra{
vk::ComponentSwizzle::eB, vk::ComponentSwizzle::eG,
vk::ComponentSwizzle::eR, vk::ComponentSwizzle::eA};
ImageViewInfo sampled{};
sampled.format = color_info.pixel_format;
sampled.type = vk::ImageViewType::e2D;
sampled.aspect = vk::ImageAspectFlagBits::eColor;
sampled.base_level = 0;
sampled.level_count = 1;
sampled.base_layer = 0;
sampled.layer_count = 1;
sampled.mapping = identity;
sampled.usage = vk::ImageUsageFlagBits::eSampled;
const auto first = color.FindView(sampled);
const auto first_again = color.FindView(sampled);
auto swizzled_info = sampled;
swizzled_info.mapping = bgra;
const auto swizzled = color.FindView(swizzled_info);
auto mip_info = sampled;
mip_info.base_level = 1;
const auto mip = color.FindView(mip_info);
auto layer_info = sampled;
layer_info.base_layer = 1;
const auto layer = color.FindView(layer_info);
auto array_info = sampled;
array_info.type = vk::ImageViewType::e2DArray;
array_info.layer_count = 2;
const auto array = color.FindView(array_info);
auto reinterpreted_info = sampled;
reinterpreted_info.format = vk::Format::eR8G8B8A8Uint;
const auto reinterpreted = color.FindView(reinterpreted_info);
auto storage_info = sampled;
storage_info.usage = vk::ImageUsageFlagBits::eStorage;
const auto storage = color.FindView(storage_info);
auto attachment_info = sampled;
attachment_info.usage = vk::ImageUsageFlagBits::eColorAttachment;
const auto attachment = color.FindView(attachment_info);
Require(name, "color views",
first != nullptr && first_again == first && swizzled != nullptr &&
swizzled != first && mip != nullptr && mip != first &&
layer != nullptr && layer != first && array != nullptr &&
array != first && reinterpreted != nullptr &&
reinterpreted != first && storage != nullptr &&
storage != first && attachment == first &&
color.views.size() == 7,
"dynamic view identity omitted mapping, format, mip, layer, type, "
"or storage usage");
auto video_out_info = color_info;
video_out_info.pixel_format = vk::Format::eA2R10G10B10UnormPack32;
video_out_info.guest_format = Prospero::BufferFormat::k10_10_10_2UNorm;
Libs::Graphics::Image video_out(m_runtime_context, scheduler,
video_out_info);
auto video_out_view_info = sampled;
video_out_view_info.format = video_out_info.pixel_format;
video_out_view_info.usage = vk::ImageUsageFlagBits::eTransferSrc;
const auto video_out_view = video_out.FindView(video_out_view_info);
auto storage_alias_info = video_out_view_info;
storage_alias_info.format = vk::Format::eA2B10G10R10UnormPack32;
storage_alias_info.usage = vk::ImageUsageFlagBits::eStorage;
const auto storage_alias_view = video_out.FindView(storage_alias_info);
Require(
name, "video-out storage alias usage",
ImageViewOps::FormatsCompatible(video_out_info.pixel_format,
storage_alias_info.format) &&
static_cast<bool>(video_out.backing.usage &
vk::ImageUsageFlagBits::eStorage) &&
video_out_view != nullptr && storage_alias_view != nullptr,
"compatible video-out backing was created without storage usage");
ImageInfo depth_info{};
depth_info.pixel_format = vk::Format::eD32Sfloat;
depth_info.guest_format = Prospero::BufferFormat::k32Float;
depth_info.type = Prospero::ImageType::kColor2D;
depth_info.extent = {8, 8, 1};
depth_info.resources = {1, 2};
depth_info.pitch = 8;
depth_info.bytes_per_block = 4;
depth_info.samples = 1;
depth_info.tile_mode = Prospero::TileMode::kDepth;
depth_info.mip_layout[0] = {0, 512, 8, 8};
Libs::Graphics::Image depth(m_runtime_context, scheduler, depth_info);
ImageViewInfo depth_sampled{};
depth_sampled.format = depth_info.pixel_format;
depth_sampled.type = vk::ImageViewType::e2D;
depth_sampled.aspect = vk::ImageAspectFlagBits::eDepth;
depth_sampled.base_level = 0;
depth_sampled.level_count = 1;
depth_sampled.base_layer = 0;
depth_sampled.layer_count = 1;
depth_sampled.mapping = {vk::ComponentSwizzle::eR, vk::ComponentSwizzle::eR,
vk::ComponentSwizzle::eR,
vk::ComponentSwizzle::eR};
depth_sampled.usage = vk::ImageUsageFlagBits::eSampled;
const auto depth_first = depth.FindView(depth_sampled);
auto depth_alias_info = depth_sampled;
depth_alias_info.format = vk::Format::eR32Uint;
const auto depth_alias = depth.FindView(depth_alias_info);
const auto depth_again = depth.FindView(depth_sampled);
auto depth_array_info = depth_sampled;
depth_array_info.type = vk::ImageViewType::e2DArray;
depth_array_info.layer_count = 2;
const auto depth_array = depth.FindView(depth_array_info);
auto depth_attachment_info = depth_array_info;
depth_attachment_info.mapping = {};
depth_attachment_info.usage =
vk::ImageUsageFlagBits::eDepthStencilAttachment;
const auto depth_attachment = depth.FindView(depth_attachment_info);
Require(name, "depth views",
depth_first != nullptr && depth_again == depth_first &&
depth_alias == depth_first && depth_array != nullptr &&
depth_array != depth_first && depth_attachment != nullptr &&
depth_attachment != depth_array &&
depth.views.size() == 3,
"unified depth view cache lost sampled/attachment identity");
auto depth_stencil_info = depth_info;
depth_stencil_info.pixel_format = vk::Format::eD24UnormS8Uint;
depth_stencil_info.guest_format = Prospero::BufferFormat::k16UNorm;
depth_stencil_info.bytes_per_block = 2;
Libs::Graphics::Image depth_stencil(m_runtime_context, scheduler,
depth_stencil_info);
auto depth_only_info = depth_sampled;
depth_only_info.format = depth_stencil_info.pixel_format;
const auto depth_only = depth_stencil.FindView(depth_only_info);
auto combined_info = depth_only_info;
combined_info.aspect =
vk::ImageAspectFlagBits::eDepth | vk::ImageAspectFlagBits::eStencil;
combined_info.mapping = {};
combined_info.usage = vk::ImageUsageFlagBits::eDepthStencilAttachment;
const auto combined = depth_stencil.FindView(combined_info);
Require(name, "depth/stencil aspects",
depth_only != nullptr && combined != nullptr &&
depth_only != combined &&
depth_stencil.views.size() == 2 &&
depth_stencil.views[0].info.aspect ==
vk::ImageAspectFlagBits::eDepth &&
depth_stencil.views[1].info.aspect ==
(vk::ImageAspectFlagBits::eDepth |
vk::ImageAspectFlagBits::eStencil),
"explicit depth-only and combined aspects shared one image view");
Require(name, "role-free backing",
color.backing.image_type == vk::ImageType::e2D &&
color.backing.layers == 2 &&
depth.backing.image_type == vk::ImageType::e2D &&
ImageViewOps::FormatsCompatible(vk::Format::eR8G8B8A8Unorm,
vk::Format::eR8G8B8A8Uint) &&
!ImageViewOps::FormatsCompatible(vk::Format::eD32Sfloat,
vk::Format::eR32Sfloat),
"role-free image backing or host format compatibility diverged");
auto line_info = color_info;
line_info.type = Prospero::ImageType::kColor1D;
line_info.extent = {8, 1, 1};
line_info.resources = {2, 2};
line_info.mip_layout[0] = {0, 64, 8, 1};
line_info.mip_layout[1] = {64, 16, 4, 1};
Libs::Graphics::Image line(m_runtime_context, scheduler, line_info);
auto line_view_info = sampled;
line_view_info.type = vk::ImageViewType::e1D;
const auto line_view = line.FindView(line_view_info);
auto line_array_info = line_view_info;
line_array_info.type = vk::ImageViewType::e1DArray;
line_array_info.layer_count = 2;
const auto line_array = line.FindView(line_array_info);
Require(name, "1D views",
line_view != nullptr && line_array != nullptr &&
line_array != line_view &&
line.backing.image_type == vk::ImageType::e1D &&
line.backing.layers == 2,
"1D and 1D-array views did not use a first-class 1D backing");
auto volume_info = color_info;
volume_info.type = Prospero::ImageType::kColor3D;
volume_info.extent = {8, 8, 4};
volume_info.resources = {2, 1};
Libs::Graphics::Image volume(m_runtime_context, scheduler, volume_info);
auto slice_info = sampled;
slice_info.type = vk::ImageViewType::e2D;
slice_info.base_level = 1;
slice_info.base_layer = 1;
const auto slice = volume.FindView(slice_info);
auto slice_array_info = slice_info;
slice_array_info.type = vk::ImageViewType::e2DArray;
slice_array_info.base_layer = 0;
slice_array_info.layer_count = 2;
const auto slice_array = volume.FindView(slice_array_info);
Require(name, "3D slice views",
slice != nullptr && slice_array != nullptr &&
slice_array != slice &&
static_cast<bool>(volume.backing.flags &
vk::ImageCreateFlagBits::e2DArrayCompatible),
"2D slice views of a compatible 3D backing were rejected");
std::printf("[host] %-32s ok\n", name);
}
void CheckBufferCacheDirtyGarbageCollection() {
constexpr const char *name = "BufferCacheDirtyGarbageCollection";
constexpr uintptr_t base = 0x0000000200700000ull;
constexpr uint64_t allocation_size = 0x5000000;
constexpr uint64_t allocation_alignment = 0x10000;
constexpr uint64_t first_offset = 0x100;
constexpr uint64_t second_offset = 0x200;
constexpr uint64_t clean_offset = 0x300;
constexpr uint64_t unmap_offset = 0x4100;
constexpr uint64_t partial_unmap_offset = 0x8100;
constexpr uint64_t partial_unmap_survivor_offset = 0xc100;
constexpr uint64_t ring_fault_first_offset = 0x2100;
constexpr uint64_t ring_fault_second_offset = 0x2110;
constexpr uint32_t first_value = 0x10203040u;
constexpr uint32_t second_value = 0x50607080u;
constexpr uint32_t first_stale = 0x0badf00du;
constexpr uint32_t second_stale = 0xdeadbeefu;
constexpr uint32_t clean_value = 0xaabbccddu;
constexpr uint32_t unmap_value = 0x91a2b3c4u;
constexpr uint32_t remap_value = 0xd5e6f708u;
constexpr uint32_t partial_unmap_value = 0x62738495u;
constexpr uint32_t partial_unmap_survivor_value = 0xa6b7c8d9u;
constexpr uint32_t ring_fault_first_value = 0x0a1b2c3du;
constexpr uint32_t ring_fault_second_value = 0x4e5f6071u;
EnsureRuntimeContext();
RenderContext context(m_runtime_context);
auto &scheduler = context.GetCommandScheduler();
HW::Context registers{};
HW::UserConfig user_config{};
HW::Shader shaders{};
scheduler.Begin(registers, user_config, shaders);
context.InitializeGpu(nullptr);
int64_t direct_offset = -1;
Require(name, "direct allocation",
Libs::LibKernel::Memory::KernelAllocateDirectMemory(
0, Libs::LibKernel::Memory::KernelGetDirectMemorySize(),
allocation_size, allocation_alignment, 0, &direct_offset) == 0,
"dirty-GC direct-memory allocation failed");
void *mapped = reinterpret_cast<void *>(base);
Require(name, "direct mapping",
Libs::LibKernel::Memory::KernelMapDirectMemory(
&mapped, allocation_size, 0x3, 0x10, direct_offset,
allocation_alignment) == 0 &&
mapped == reinterpret_cast<void *>(base),
"dirty-GC fixed direct-memory mapping failed");
auto *memory = static_cast<uint8_t *>(mapped);
std::memcpy(memory + first_offset, &first_stale, sizeof(first_stale));
std::memcpy(memory + second_offset, &second_stale, sizeof(second_stale));
std::memcpy(memory + clean_offset, &clean_value, sizeof(clean_value));
{
auto &resources = context;
auto &cache = resources.GetBufferCache();
resources.MapMemory(base, allocation_size);
const auto MarkGpuWrite = [&](uint64_t address, uint64_t size) {
auto allocation = cache.ObtainBuffer(address, size, true, false);
Require(name, "dirty allocation", allocation.first != nullptr,
"dirty-GC buffer allocation failed");
};
const auto ReadNativeValue =
[&](const Libs::Graphics::Buffer &buffer, uint64_t offset) {
auto readback =
CreateHostBuffer(name, sizeof(uint32_t),
vk::BufferUsageFlagBits::eTransferDst, {0});
const vk::BufferCopy copy{offset, 0, sizeof(uint32_t)};
scheduler.Current().Handle().copyBuffer(buffer.Handle(), readback.buffer,
1, &copy);
vk::BufferMemoryBarrier barrier{};
barrier.sType = vk::StructureType::eBufferMemoryBarrier;
barrier.srcAccessMask = vk::AccessFlagBits::eTransferWrite;
barrier.dstAccessMask = vk::AccessFlagBits::eHostRead;
barrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
barrier.buffer = readback.buffer;
barrier.size = readback.size;
scheduler.Current().Handle().pipelineBarrier(
vk::PipelineStageFlagBits::eTransfer,
vk::PipelineStageFlagBits::eHost, {}, 0, nullptr, 1, &barrier, 0,
nullptr);
scheduler.Finish();
const auto value = ReadBuffer(name, readback, 1)[0];
DestroyBuffer(&readback);
return value;
};
constexpr uint64_t stream_pages = base + 0x18000;
constexpr uint64_t stream_address = stream_pages + 0x4000 - sizeof(uint32_t);
const std::array<uint32_t, 2> stream_values{0x13579bdfu, 0x2468ace0u};
std::memcpy(reinterpret_cast<void *>(stream_address), stream_values.data(),
sizeof(stream_values));
Require(name, "GPU-readable stream source",
Libs::LibKernel::Memory::KernelMprotect(
reinterpret_cast<void *>(stream_pages), 0x8000, 0x10) == 0 &&
cache.IsRegionCpuModified(stream_address, sizeof(stream_values)) &&
!cache.IsRegionGpuModified(stream_address, sizeof(stream_values)),
"cross-page stream source is not CPU-dirty and GPU-clean");
const auto stream_tick = scheduler.CurrentTick();
const auto [streamed, streamed_offset] =
cache.ObtainBuffer(stream_address, sizeof(stream_values), false);
Require(name, "cross-page read-only stream upload",
streamed == &cache.GetUtilityBuffer(MemoryUsage::Stream) &&
scheduler.CurrentTick() == stream_tick &&
std::memcmp(streamed->Mapped().data() + streamed_offset,
stream_values.data(), sizeof(stream_values)) == 0,
"GPU-readable guest bytes were not streamed without submission");
Require(name, "stream source protection restore",
Libs::LibKernel::Memory::KernelMprotect(
reinterpret_cast<void *>(stream_pages), 0x8000, 0x3) == 0,
"stream source protection was not restored");
Require(name, "streamed GPU bytes",
ReadNativeValue(*streamed, streamed_offset) == stream_values[0] &&
ReadNativeValue(*streamed, streamed_offset + sizeof(uint32_t)) == stream_values[1],
"stream commit did not publish both cross-page words to the GPU");
constexpr uint64_t large_copy_source_offset = 0x40000;
constexpr uint64_t large_copy_destination_offset = 0x80000;
constexpr uint64_t large_copy_size = 96 * 1024;
std::vector<uint8_t> large_copy_source(large_copy_size);
for (uint64_t index = 0; index < large_copy_size; ++index) {
large_copy_source[index] =
static_cast<uint8_t>((index * 37 + index / 251) & 0xffu);
}
std::memcpy(memory + large_copy_source_offset, large_copy_source.data(),
large_copy_size);
std::memset(memory + large_copy_destination_offset, 0, large_copy_size);
cache.CopyBuffer(base + large_copy_destination_offset,
base + large_copy_source_offset, large_copy_size, false,
false);
Require(name, "large host copy",
std::memcmp(memory + large_copy_destination_offset,
large_copy_source.data(), large_copy_size) == 0,
"chunked host copy lost bytes across its scratch boundary");
constexpr uint64_t index_offset = 0x180000;
constexpr uint64_t index_page = BufferCache::CACHING_PAGESIZE;
constexpr uint64_t index_span = 3 * index_page;
const uint64_t index_begin = base + index_offset;
Require(name, "registered-range empty lookup",
!cache.IsRegionRegistered(index_begin, index_span) &&
!BufferCacheTestAccess::PageOwner(cache, index_begin),
"empty Buffer cache reported a registered range");
const auto index_left =
cache.FindBuffer(index_begin + 0x100, sizeof(uint32_t));
const auto index_right = cache.FindBuffer(
index_begin + 2 * index_page + 0x100, sizeof(uint32_t));
Require(name, "registered-range owners",
index_left && index_right &&
cache.FindBuffer(index_begin + 0x200, sizeof(uint32_t)) ==
index_left &&
BufferCacheTestAccess::PageOwner(cache, index_begin) ==
index_left &&
!BufferCacheTestAccess::PageOwner(cache,
index_begin + index_page) &&
BufferCacheTestAccess::PageOwner(
cache, index_begin + 2 * index_page) == index_right,
"failed to create disjoint Buffer index owners");
Require(
name, "registered-range boundaries",
cache.IsRegionRegistered(index_begin, 1) &&
cache.IsRegionRegistered(index_begin + index_page - 1, 1) &&
!cache.IsRegionRegistered(index_begin - 1, 1) &&
!cache.IsRegionRegistered(index_begin + index_page, index_page) &&
cache.IsRegionRegistered(index_begin + index_page,
index_page + 1) &&
!cache.IsRegionRegistered(index_begin + index_span, 1) &&
cache.IsRegionRegistered(index_begin - 1, index_span + 2),
"indexed lookup mishandled a half-open boundary, gap, or broad "
"overlap");
const auto clean_unmap_tick = scheduler.CurrentTick();
resources.UnmapMemory(index_begin, index_page);
Require(name, "clean cached buffer unmap drain",
scheduler.CurrentTick() == clean_unmap_tick + 1 &&
!resources.IsMapped(index_begin, index_page),
"unmapping a clean cached buffer did not drain native work");
resources.MapMemory(index_begin, index_page);
const auto index_bridge =
cache.FindBuffer(index_begin + index_page - 1, index_page + 2);
Require(name, "registered-range merge",
index_bridge && cache.IsRegionRegistered(index_begin + index_page,
index_page) &&
BufferCacheTestAccess::PageOwner(cache, index_begin) ==
index_bridge &&
BufferCacheTestAccess::PageOwner(
cache, index_begin + index_page) == index_bridge &&
BufferCacheTestAccess::PageOwner(
cache, index_begin + 2 * index_page) == index_bridge,
"bridging acquisition did not publish its merged Buffer range");
scheduler.Finish();
constexpr uint64_t recycled_offset = index_offset + 4 * index_page;
const auto recycled = cache.FindBuffer(base + recycled_offset + 0x100,
sizeof(uint32_t));
Require(name, "slot generation reuse",
(recycled.index == index_left.index && recycled != index_left) ||
(recycled.index == index_right.index &&
recycled != index_right),
"retired Buffer slot was not reused with a new generation");
const auto [resolved_left, resolved_left_offset] = cache.ObtainBuffer(
index_begin + 0x100, sizeof(uint32_t), true, false, index_left);
const auto [resolved_right, resolved_right_offset] =
cache.ObtainBuffer(index_begin + 2 * index_page + 0x100,
sizeof(uint32_t), true, false, index_right);
Require(name, "two-pass stale id resolution",
resolved_left == &cache.GetBuffer(index_bridge) &&
resolved_right == resolved_left &&
resolved_left_offset != resolved_right_offset,
"saved descriptor IDs did not resolve through the merged owner");
constexpr uint64_t stream_page = BufferCache::CACHING_PAGESIZE;
constexpr uint64_t stream_leap = stream_page * 128;
for (const bool forward : {true, false}) {
const char *stream_name =
forward ? "ForwardBufferStream" : "BackwardBufferStream";
const uint64_t stream_begin = base + (forward ? 0x1800000 : 0x2000000);
const uint64_t caught_address = forward
? stream_begin + stream_leap / 2
: stream_begin - stream_leap / 2;
const uint64_t untouched_address = forward
? stream_begin - stream_leap / 2
: stream_begin + stream_leap / 2;
const auto caught = cache.FindBuffer(caught_address, stream_page);
const auto untouched = cache.FindBuffer(untouched_address, stream_page);
auto grown = cache.FindBuffer(stream_begin, stream_page * 2);
constexpr uint32_t stream_value = 0x6a3b92d1u;
for (const auto address : {stream_begin, caught_address}) {
MarkGpuWrite(address, sizeof(stream_value));
cache.FillBuffer(address, sizeof(stream_value), stream_value, false);
}
uint64_t requested_begin = stream_begin;
uint64_t requested_end = stream_begin + stream_page * 2;
for (uint32_t step = 1; step <= 18; ++step) {
const auto address = (forward ? requested_end : requested_begin) - 4;
grown = cache.FindBuffer(address, 8);
if (forward) {
requested_end += stream_page;
} else {
requested_begin -= stream_page;
}
if (step < 18) {
const auto &owner = cache.GetBuffer(grown);
Require(stream_name, "growth before threshold",
owner.CpuAddress() == requested_begin &&
owner.Size() == requested_end - requested_begin &&
owner.StreamScore() == static_cast<int>(step),
"sequential requests grew beyond their consumed pages too "
"early");
}
}
const auto &grown_buffer = cache.GetBuffer(grown);
Require(stream_name, "growth reserve",
grown_buffer.CpuAddress() ==
(forward ? requested_begin
: requested_begin - stream_leap) &&
grown_buffer.Size() ==
requested_end - requested_begin + stream_leap &&
grown_buffer.StreamScore() == 0 &&
cache.GetBuffer(caught).is_deleted &&
!cache.GetBuffer(untouched).is_deleted &&
BufferCacheTestAccess::PageOwner(cache, caught_address) ==
grown &&
BufferCacheTestAccess::PageOwner(
cache, untouched_address) == untouched,
"streaming did not reserve the next pages or retain correct "
"neighboring owners");
for (uint32_t step = 0; step < 128; ++step) {
const auto address = forward
? requested_end + step * stream_page - 4
: requested_begin - step * stream_page - 4;
Require(stream_name, "reuse reserved pages",
cache.FindBuffer(address, 8) == grown,
"continued sequential requests replaced the buffer within "
"its growth reserve");
}
for (const auto address : {stream_begin, caught_address}) {
Require(
stream_name, "preserve GPU contents",
ReadNativeValue(grown_buffer, grown_buffer.Offset(address)) ==
stream_value,
"stream growth lost GPU-written contents from a merged owner");
}
}
resources.UnmapMemory(index_begin, index_span);
resources.UnmapMemory(base + recycled_offset, index_page);
Require(name, "invalidation retains owners",
!resources.IsMapped(index_begin, index_span) &&
!resources.IsMapped(base + recycled_offset, index_page) &&
cache.IsRegionRegistered(index_begin, index_span) &&
BufferCacheTestAccess::PageOwner(cache, index_begin) ==
index_bridge &&
BufferCacheTestAccess::PageOwner(
cache, index_begin + index_page) == index_bridge &&
BufferCacheTestAccess::PageOwner(
cache, index_begin + 2 * index_page) == index_bridge &&
BufferCacheTestAccess::PageOwner(
cache, base + recycled_offset) == recycled &&
cache.IsRegionCpuModified(index_begin, index_span) &&
cache.IsRegionCpuModified(base + recycled_offset,
index_page),
"CPU invalidation retired a reusable Buffer owner");
resources.MapMemory(index_begin, index_span);
resources.MapMemory(base + recycled_offset, index_page);
MarkGpuWrite(base + first_offset, sizeof(first_value));
MarkGpuWrite(base + second_offset, sizeof(second_value));
cache.FillBuffer(base + first_offset, sizeof(first_value), first_value,
false);
cache.FillBuffer(base + second_offset, sizeof(second_value), second_value,
false);
auto &download = BufferCacheTestAccess::DownloadBuffer(cache);
auto *fixed_download = &download;
const auto fixed_download_handle = download.Handle();
auto [ring_tail, ring_tail_offset] =
download.Map(download.Size() - sizeof(uint32_t), 4);
Require(name, "fault-ring tail reservation",
ring_tail != nullptr && ring_tail_offset == 0,
"failed to position the reusable download ring near wrap");
download.Commit();
MarkGpuWrite(base + ring_fault_first_offset,
sizeof(ring_fault_first_value));
MarkGpuWrite(base + ring_fault_second_offset,
sizeof(ring_fault_second_value));
cache.FillBuffer(base + ring_fault_first_offset,
sizeof(ring_fault_first_value), ring_fault_first_value,
false);
cache.FillBuffer(base + ring_fault_second_offset,
sizeof(ring_fault_second_value), ring_fault_second_value,
false);
Require(name, "fault-ring wrapped batch",
resources.HandleFault(PageFaultAccess::Read,
base + ring_fault_first_offset),
"fault readback could not wrap a live download-ring tick");
// Both disjoint writes on the faulting page share one wrapped batch.
uint64_t expected_packing_offset = 2 * 64;
uint64_t expected_packing_alignment = 1;
if (!download.IsCoherent()) {
Require(name, "fault-ring atom policy",
StreamBufferTestAccess::NormalizeReservation(
false,
m_runtime_context.physical_device_properties.limits
.nonCoherentAtomSize,
expected_packing_offset, expected_packing_alignment),
"failed to calculate non-coherent fault-ring stride");
}
auto [packing_probe, packing_offset] = download.Map(1, 1, false);
Require(name, "adjacent download reservation stride",
packing_probe != nullptr &&
packing_offset == expected_packing_offset,
"adjacent fault downloads did not reserve an atom-safe stride");
download.Commit();
uint32_t first_page_backing = 0;
uint32_t second_page_backing = 0;
uint32_t ring_fault_first_backing = 0;
uint32_t ring_fault_second_backing = 0;
Libs::LibKernel::Memory::TryReadBacking(base + first_offset,
&first_page_backing, sizeof(first_page_backing));
Libs::LibKernel::Memory::TryReadBacking(base + second_offset,
&second_page_backing, sizeof(second_page_backing));
std::memcpy(&ring_fault_first_backing, memory + ring_fault_first_offset,
sizeof(ring_fault_first_backing));
std::memcpy(&ring_fault_second_backing, memory + ring_fault_second_offset,
sizeof(ring_fault_second_backing));
Require(name, "fault-ring wrapped contents",
first_page_backing == first_stale &&
second_page_backing == second_stale &&
cache.HasGpuDirtyBytes(base + first_offset, sizeof(first_value)) &&
cache.HasGpuDirtyBytes(base + second_offset, sizeof(second_value)) &&
ring_fault_first_backing == ring_fault_first_value &&
ring_fault_second_backing == ring_fault_second_value &&
&BufferCacheTestAccess::DownloadBuffer(cache) ==
fixed_download &&
download.Handle() == fixed_download_handle &&
download.Size() == (64ull << 20),
"wrapped fault batch lost its page contents or published an unrelated page");
cache.ReadMemory(base + first_offset, sizeof(first_value));
Libs::LibKernel::Memory::TryReadBacking(base + first_offset,
&first_page_backing, sizeof(first_page_backing));
Libs::LibKernel::Memory::TryReadBacking(base + second_offset,
&second_page_backing, sizeof(second_page_backing));
Require(name, "deferred neighboring page read",
first_page_backing == first_value && second_page_backing == second_value &&
!cache.HasGpuDirtyBytes(base + first_offset, sizeof(first_value)) &&
!cache.HasGpuDirtyBytes(base + second_offset, sizeof(second_value)),
"the earlier page did not publish its dirty ranges on its own read");
constexpr uint64_t page_size = TRACKER_PAGE_SIZE;
constexpr uint64_t page_owner_offset = 0x240000;
constexpr uint64_t page_owner_size = 0xc0000;
constexpr uint64_t page_fault_offset = page_owner_offset + 0x100;
constexpr uint64_t page_inside_offset =
page_owner_offset + page_size - sizeof(uint32_t);
constexpr uint64_t page_outside_offset =
page_owner_offset + page_size;
constexpr uint32_t page_stale = 0x13579bdfu;
constexpr uint32_t page_value = 0x2468ace0u;
constexpr std::array page_offsets{
page_fault_offset, page_inside_offset, page_outside_offset};
static_assert((base + page_owner_offset) % page_size == 0);
for (const auto offset : page_offsets) {
Libs::LibKernel::Memory::WriteBacking(base + offset, &page_stale,
sizeof(page_stale));
}
(void)cache.FindBuffer(base + page_owner_offset, page_owner_size);
for (const auto offset : page_offsets) {
MarkGpuWrite(base + offset, sizeof(page_value));
cache.FillBuffer(base + offset, sizeof(page_value), page_value,
false);
}
const auto page_publication_tick = scheduler.CurrentTick();
std::binary_semaphore page_publication_entered{0};
std::binary_semaphore release_page_publication{0};
std::atomic<bool> page_readback_returned{false};
scheduler.DeferPriorityOperation([&] {
page_publication_entered.release();
release_page_publication.acquire();
Libs::LibKernel::Memory::WriteBacking(
base + page_inside_offset, &page_stale, sizeof(page_stale));
});
std::jthread release_page_callback([&] {
page_publication_entered.acquire();
const auto deadline = std::chrono::steady_clock::now() +
std::chrono::milliseconds(100);
while (!page_readback_returned.load() &&
std::chrono::steady_clock::now() < deadline) {
std::this_thread::yield();
}
release_page_publication.release();
});
cache.ReadMemory(base + page_fault_offset, sizeof(page_value));
page_readback_returned = true;
release_page_callback.join();
scheduler.WaitPriorityOperations(page_publication_tick);
uint32_t page_inside_backing = 0;
uint32_t page_outside_backing = 0;
Libs::LibKernel::Memory::TryReadBacking(base + page_inside_offset,
&page_inside_backing,
sizeof(page_inside_backing));
Libs::LibKernel::Memory::TryReadBacking(base + page_outside_offset,
&page_outside_backing,
sizeof(page_outside_backing));
Require(name, "page publication order and boundary",
page_inside_backing == page_value &&
page_outside_backing == page_stale &&
!cache.HasGpuDirtyBytes(base + page_inside_offset,
sizeof(page_value)) &&
cache.HasGpuDirtyBytes(base + page_outside_offset,
sizeof(page_value)) &&
!cache.IsRegionGpuModified(base + page_inside_offset,
sizeof(page_value)) &&
cache.IsRegionGpuModified(base + page_outside_offset,
sizeof(page_value)),
"readback did not publish after the older overlapping callback "
"or retained ownership of the wrong page");
cache.ReadMemory(base + page_outside_offset, sizeof(page_value));
Libs::LibKernel::Memory::TryReadBacking(base + page_outside_offset,
&page_outside_backing, sizeof(page_outside_backing));
Require(name, "next page publication",
page_outside_backing == page_value &&
!cache.HasGpuDirtyBytes(base + page_outside_offset,
sizeof(page_value)) &&
!cache.IsRegionGpuModified(base + page_outside_offset,
sizeof(page_value)),
"the next page did not publish and release ownership on its own read");
Libs::LibKernel::Memory::WriteBacking(base + first_offset, &first_stale,
sizeof(first_stale));
Libs::LibKernel::Memory::WriteBacking(base + second_offset, &second_stale,
sizeof(second_stale));
MarkGpuWrite(base + first_offset, sizeof(first_value));
MarkGpuWrite(base + second_offset, sizeof(second_value));
cache.FillBuffer(base + first_offset, sizeof(first_value), first_value,
false);
cache.FillBuffer(base + second_offset, sizeof(second_value), second_value,
false);
for (uint32_t tick = 0; tick < 160; tick++) {
cache.RunGarbageCollector();
}
Require(name, "age before pressure",
cache.IsRegionRegistered(base, allocation_size),
"buffer was reclaimed without memory pressure");
BufferCacheTestAccess::SetGarbageCollectionThresholds(
cache, 0, std::numeric_limits<uint64_t>::max());
const auto gc_submission_tick = scheduler.CurrentTick();
cache.RunGarbageCollector();
uint32_t first_before_completion = 0;
uint32_t second_before_completion = 0;
Libs::LibKernel::Memory::TryReadBacking(base + first_offset,
&first_before_completion,
sizeof(first_before_completion));
Libs::LibKernel::Memory::TryReadBacking(base + second_offset,
&second_before_completion,
sizeof(second_before_completion));
Require(name, "normal dirty retention",
first_before_completion == first_stale &&
second_before_completion == second_stale &&
cache.IsRegionRegistered(base, allocation_size) &&
cache.IsRegionGpuModified(base + first_offset,
sizeof(first_value)) &&
cache.IsRegionGpuModified(base + second_offset,
sizeof(second_value)) &&
cache.HasGpuDirtyBytes(base + first_offset,
sizeof(first_value)) &&
cache.HasGpuDirtyBytes(base + second_offset,
sizeof(second_value)) &&
scheduler.CurrentTick() == gc_submission_tick,
"normal GC retired or synchronized a dirty Buffer");
BufferCacheTestAccess::SetGarbageCollectionThresholds(cache, 0, 0);
const auto older_publication_tick = scheduler.CurrentTick();
std::binary_semaphore older_publication_entered{0};
std::binary_semaphore release_older_publication{0};
std::atomic<bool> gc_returned{false};
scheduler.DeferPriorityOperation([&] {
older_publication_entered.release();
release_older_publication.acquire();
Libs::LibKernel::Memory::WriteBacking(
base + first_offset, &first_stale, sizeof(first_stale));
});
std::jthread release_publication([&] {
older_publication_entered.acquire();
const auto deadline = std::chrono::steady_clock::now() +
std::chrono::milliseconds(100);
while (!gc_returned.load() &&
std::chrono::steady_clock::now() < deadline) {
std::this_thread::yield();
}
release_older_publication.release();
});
cache.RunGarbageCollector();
gc_returned = true;
release_publication.join();
scheduler.WaitPriorityOperations(older_publication_tick);
uint32_t first_backing = 0;
uint32_t second_backing = 0;
uint32_t clean_backing = 0;
std::memcpy(&first_backing, memory + first_offset, sizeof(first_backing));
std::memcpy(&second_backing, memory + second_offset,
sizeof(second_backing));
std::memcpy(&clean_backing, memory + clean_offset, sizeof(clean_backing));
Require(name, "downloaded contents",
first_backing == first_value && second_backing == second_value &&
clean_backing == clean_value &&
!cache.IsRegionRegistered(base, allocation_size) &&
!cache.IsRegionGpuModified(base + first_offset,
sizeof(first_value)) &&
!cache.IsRegionGpuModified(base + second_offset,
sizeof(second_value)) &&
!cache.HasGpuDirtyBytes(base + first_offset,
sizeof(first_value)) &&
!cache.HasGpuDirtyBytes(base + second_offset,
sizeof(second_value)) &&
scheduler.CurrentTick() > gc_submission_tick,
"critical GC did not publish exact ranges before clearing "
"Buffer ownership");
constexpr uint64_t starvation_offset = 0x100000;
constexpr uint64_t starvation_stride = 2 * BufferCache::CACHING_PAGESIZE;
constexpr uint32_t starvation_count = 33;
constexpr uint32_t starvation_value = 0x31415926u;
BufferCacheTestAccess::SetGarbageCollectionThresholds(
cache, std::numeric_limits<uint64_t>::max(),
std::numeric_limits<uint64_t>::max());
for (uint32_t index = 0; index < starvation_count; index++) {
const auto address = base + starvation_offset + index * starvation_stride;
(void)cache.ObtainBuffer(address, sizeof(starvation_value), true);
cache.FillBuffer(address, sizeof(starvation_value), starvation_value,
false);
}
for (uint32_t tick = 0; tick < 160; tick++) {
cache.RunGarbageCollector();
}
constexpr uint64_t starvation_clean_offset =
starvation_offset + starvation_count * starvation_stride;
const auto starvation_clean = cache.FindBuffer(
base + starvation_clean_offset, sizeof(starvation_value));
Require(name, "normal-GC clean candidate",
static_cast<bool>(starvation_clean),
"failed to create the clean starvation candidate");
for (uint32_t tick = 0; tick <= 160; tick++) {
cache.RunGarbageCollector();
}
BufferCacheTestAccess::SetGarbageCollectionThresholds(
cache, 0, std::numeric_limits<uint64_t>::max());
cache.RunGarbageCollector();
Require(
name, "normal-GC dirty bypass",
cache.IsRegionRegistered(base + starvation_offset,
sizeof(starvation_value)) &&
cache.IsRegionRegistered(
base + starvation_offset +
(starvation_count - 1) * starvation_stride,
sizeof(starvation_value)) &&
!cache.IsRegionRegistered(base + starvation_clean_offset,
sizeof(starvation_value)),
"old dirty owners hid an eligible clean Buffer from normal GC");
BufferCacheTestAccess::SetGarbageCollectionThresholds(cache, 0, 0);
const auto starvation_retired =
BufferCacheTestAccess::PageOwner(cache, base + starvation_offset);
cache.RunGarbageCollector();
Require(name, "critical-GC starvation cleanup",
!cache.IsRegionRegistered(base + starvation_offset,
sizeof(starvation_value)) &&
!cache.IsRegionRegistered(
base + starvation_offset +
(starvation_count - 1) * starvation_stride,
sizeof(starvation_value)) &&
!BufferCacheTestAccess::IsBufferAllocated(
cache, starvation_retired),
"critical GC did not immediately free the skipped dirty owners");
constexpr uint64_t lookup_only_offset = 0x218000;
constexpr uint64_t obtained_offset = 0x220000;
constexpr uint64_t residency_size =
2 * BufferCache::CACHING_PAGESIZE;
BufferCacheTestAccess::SetGarbageCollectionThresholds(
cache, std::numeric_limits<uint64_t>::max(),
std::numeric_limits<uint64_t>::max());
const auto lookup_only = cache.FindBuffer(
base + lookup_only_offset, residency_size);
const auto obtained =
cache.FindBuffer(base + obtained_offset, residency_size);
for (uint32_t tick = 0; tick <= 160; tick++) {
cache.RunGarbageCollector();
}
Require(name, "lookup-only owner identity",
cache.FindBuffer(base + lookup_only_offset, residency_size) ==
lookup_only,
"lookup-only discovery replaced its valid Buffer owner");
const auto [obtained_buffer, obtained_buffer_offset] = cache.ObtainBuffer(
base + obtained_offset, residency_size, false, false, obtained);
BufferCacheTestAccess::SetGarbageCollectionThresholds(
cache, 0, std::numeric_limits<uint64_t>::max());
cache.RunGarbageCollector();
Require(name, "lookup versus acquisition residency",
!cache.IsRegionRegistered(base + lookup_only_offset,
residency_size) &&
!BufferCacheTestAccess::PageOwner(
cache, base + lookup_only_offset) &&
obtained_buffer == &cache.GetBuffer(obtained) &&
obtained_buffer_offset == 0 &&
cache.IsRegionRegistered(base + obtained_offset,
residency_size) &&
BufferCacheTestAccess::PageOwner(
cache, base + obtained_offset) == obtained,
"lookup refreshed residency or actual acquisition did not");
BufferCacheTestAccess::SetGarbageCollectionThresholds(cache, 0, 0);
auto unmap_allocation = cache.ObtainBuffer(
base + unmap_offset, sizeof(unmap_value), true, false);
Require(name, "direct-invalidation allocation",
unmap_allocation.first != nullptr,
"direct-invalidation buffer allocation failed");
cache.FillBuffer(base + unmap_offset, sizeof(unmap_value), unmap_value,
false);
resources.UnmapMemory(base + 0x4000, 0x4000);
uint32_t unmap_backing = 0;
std::memcpy(&unmap_backing, memory + unmap_offset, sizeof(unmap_backing));
Require(name, "direct-invalidation contents",
unmap_backing == unmap_value &&
!resources.IsMapped(base + 0x4000, 0x4000) &&
cache.IsRegionRegistered(base + 0x4000, 0x4000) &&
cache.IsRegionCpuModified(base + 0x4000, 0x4000) &&
!cache.IsRegionGpuModified(base + 0x4000, 0x4000) &&
!cache.HasGpuDirtyBytes(base + 0x4000, 0x4000),
"dirty invalidation did not publish bytes and retain its owner");
std::memcpy(memory + unmap_offset, &remap_value, sizeof(remap_value));
resources.MapMemory(base + 0x4000, 0x4000);
const auto remapped =
cache.ObtainBuffer(base + unmap_offset, sizeof(remap_value), true);
Require(name, "same-address remap allocation",
remapped.first == unmap_allocation.first,
"same-address remap did not synchronize the retained Buffer");
Require(name, "same-address remap contents",
ReadNativeValue(*remapped.first, remapped.second) == remap_value,
"same-address remap reused stale native Buffer bytes");
auto partial_unmap_allocation =
cache.ObtainBuffer(base + 0x8000, 0x8000, true, false);
Require(name, "partial-invalidation allocation",
partial_unmap_allocation.first != nullptr,
"cross-page cached buffer allocation failed");
cache.FillBuffer(base + partial_unmap_offset, sizeof(partial_unmap_value),
partial_unmap_value, false);
cache.FillBuffer(base + partial_unmap_survivor_offset,
sizeof(partial_unmap_survivor_value),
partial_unmap_survivor_value, false);
resources.UnmapMemory(base + 0x8000, 0x4000);
Require(
name, "partial-invalidation keeps disjoint bytes dirty",
resources.IsMapped(base + partial_unmap_survivor_offset,
sizeof(partial_unmap_survivor_value)) &&
!cache.IsRegionCpuModified(
base + partial_unmap_survivor_offset,
sizeof(partial_unmap_survivor_value)) &&
cache.IsRegionGpuModified(base + partial_unmap_survivor_offset,
sizeof(partial_unmap_survivor_value)) &&
cache.HasGpuDirtyBytes(base + partial_unmap_survivor_offset,
sizeof(partial_unmap_survivor_value)) &&
!cache.HasGpuDirtyBytes(base + partial_unmap_offset,
sizeof(partial_unmap_value)),
"partial invalidation read back an unrelated mapped page");
Require(name, "partial-invalidation survivor read fault",
resources.HandleFault(PageFaultAccess::Read,
base + partial_unmap_survivor_offset),
"CPU read did not publish the retained GPU-dirty page");
uint32_t partial_unmap_survivor_backing = 0;
std::memcpy(&partial_unmap_survivor_backing,
memory + partial_unmap_survivor_offset,
sizeof(partial_unmap_survivor_backing));
Require(
name, "partial-invalidation ownership",
!resources.IsMapped(base + 0x8000, 0x4000) &&
cache.IsRegionRegistered(base + 0x8000, 0x8000) &&
cache.IsRegionCpuModified(base + 0x8000, 0x4000) &&
!cache.IsRegionCpuModified(
base + partial_unmap_survivor_offset,
sizeof(partial_unmap_survivor_value)) &&
!cache.HasGpuDirtyBytes(base + partial_unmap_offset,
sizeof(partial_unmap_value)) &&
!cache.HasGpuDirtyBytes(base + partial_unmap_survivor_offset,
sizeof(partial_unmap_survivor_value)) &&
partial_unmap_survivor_backing == partial_unmap_survivor_value,
"partial invalidation or subsequent CPU read mishandled ownership or "
"backing");
resources.MapMemory(base + 0x8000, 0x4000);
auto survivor =
cache.ObtainBuffer(base + partial_unmap_survivor_offset,
sizeof(partial_unmap_survivor_value), false);
Require(name, "partial-invalidation survivor", survivor.first != nullptr,
"cached-buffer remainder was not retained");
Require(name, "partial-invalidation survivor contents",
ReadNativeValue(*survivor.first, survivor.second) ==
partial_unmap_survivor_value,
"partial invalidation lost disjoint native bytes");
for (const uint64_t large_size : {64ull << 20, 68ull << 20}) {
constexpr uint64_t large_offset = 0x10000;
const uint32_t large_value =
large_size == (64ull << 20) ? 0x5aa55aa5u : 0xa55aa55au;
constexpr uint32_t large_stale = 0x12345678u;
std::memcpy(memory + large_offset, &large_stale, sizeof(large_stale));
std::memcpy(memory + large_offset + large_size - sizeof(large_stale),
&large_stale, sizeof(large_stale));
auto large_allocation =
cache.ObtainBuffer(base + large_offset, large_size, true, false);
Require(name, "large dirty allocation",
large_allocation.first != nullptr,
"failed to allocate the large dirty native buffer");
cache.FillBuffer(base + large_offset, large_size, large_value, false);
const auto large_submission_tick = scheduler.CurrentTick();
for (uint32_t tick = 0; tick <= 160; tick++) {
cache.RunGarbageCollector();
}
// The capacity-sized case can wrap the ring; the oversized case must
// keep its separate staging allocation alive through publication.
Require(name, "large synchronized retirement",
!cache.IsRegionRegistered(base + large_offset, large_size) &&
scheduler.CurrentTick() > large_submission_tick &&
scheduler.CurrentTick() <= large_submission_tick + 2,
"large dirty Buffer exceeded the GC synchronization budget");
uint32_t large_before_completion = 0;
Libs::LibKernel::Memory::TryReadBacking(base + large_offset,
&large_before_completion,
sizeof(large_before_completion));
Require(name, "large backing publication",
large_before_completion == large_value,
"large Buffer GC retired ownership before publication");
const auto large_image_source =
cache.ObtainBufferForImage(base + large_offset, sizeof(large_value));
Require(name, "fixed download after Buffer retirement",
large_image_source.first != nullptr &&
&BufferCacheTestAccess::DownloadBuffer(cache) ==
fixed_download &&
fixed_download->Handle() == fixed_download_handle &&
fixed_download->Size() == (64ull << 20),
"image acquisition replaced the shared Buffer download stream");
std::vector<uint32_t> large_published(large_size / sizeof(uint32_t));
Require(name, "large Buffer publication contents",
Libs::LibKernel::Memory::TryReadBacking(
base + large_offset, large_published.data(), large_size) &&
std::ranges::all_of(large_published, [large_value](uint32_t value) {
return value == large_value;
}),
"large Buffer GC did not publish its complete transfer");
}
constexpr uint64_t grouped_first_offset = 0x10000;
constexpr uint64_t grouped_second_offset = 0x2300000;
constexpr uint64_t grouped_owner_size = 33ull * 1024 * 1024;
constexpr uint32_t grouped_first_value = 0x1122aabbu;
constexpr uint32_t grouped_second_value = 0x3344ccddu;
constexpr uint32_t grouped_stale = 0;
static_assert(grouped_owner_size < (64ull << 20) &&
grouped_owner_size * 2 > (64ull << 20));
Libs::LibKernel::Memory::WriteBacking(
base + grouped_first_offset, &grouped_stale, sizeof(grouped_stale));
Libs::LibKernel::Memory::WriteBacking(
base + grouped_second_offset, &grouped_stale, sizeof(grouped_stale));
auto grouped_first = cache.ObtainBuffer(base + grouped_first_offset,
grouped_owner_size, true, false);
auto grouped_second = cache.ObtainBuffer(base + grouped_second_offset,
grouped_owner_size, true, false);
Require(name, "per-owner GC allocation",
grouped_first.first != nullptr &&
grouped_second.first != nullptr &&
grouped_first.first != grouped_second.first,
"disjoint GC candidates merged into one Buffer owner");
cache.FillBuffer(base + grouped_first_offset, grouped_owner_size,
grouped_first_value, false);
cache.FillBuffer(base + grouped_second_offset, grouped_owner_size,
grouped_second_value, false);
for (uint32_t tick = 0; tick <= 160; tick++) {
cache.RunGarbageCollector();
}
Require(name, "per-owner fixed-ring retirement",
!cache.IsRegionRegistered(base + grouped_first_offset,
grouped_owner_size) &&
!cache.IsRegionRegistered(base + grouped_second_offset,
grouped_owner_size) &&
&BufferCacheTestAccess::DownloadBuffer(cache) ==
fixed_download &&
fixed_download->Handle() == fixed_download_handle,
"GC aggregated disjoint retirees or replaced the download ring");
uint32_t grouped_first_backing = 0;
uint32_t grouped_second_backing = 0;
Libs::LibKernel::Memory::TryReadBacking(base + grouped_first_offset,
&grouped_first_backing,
sizeof(grouped_first_backing));
Libs::LibKernel::Memory::TryReadBacking(base + grouped_second_offset,
&grouped_second_backing,
sizeof(grouped_second_backing));
Require(name, "per-owner fixed-ring contents",
grouped_first_backing == grouped_first_value &&
grouped_second_backing == grouped_second_value,
"per-owner GC transfers lost data while wrapping the fixed ring");
constexpr uint64_t sparse_offset = 0x2180000;
constexpr uint64_t sparse_owner_stride = 2 * BufferCache::CACHING_PAGESIZE;
constexpr std::array<uint64_t, 3> sparse_starts{1, 7, 13};
constexpr std::array<uint64_t, 3> sparse_sizes{3, 2, 1};
constexpr std::array<uint32_t, 2> sparse_values{0x41372b19u, 0x957f6953u};
std::array<uint8_t, 16> sparse_clean{};
sparse_clean.fill(0xcdu);
const auto FillSparseOwner = [&](uint32_t owner_index) {
const auto address = base + sparse_offset +
owner_index * sparse_owner_stride;
Libs::LibKernel::Memory::WriteBacking(address, sparse_clean.data(),
sparse_clean.size());
Libs::Graphics::Buffer *owner = nullptr;
for (size_t index = 0; index < sparse_starts.size(); ++index) {
const auto allocation = cache.ObtainBuffer(
address + sparse_starts[index], sparse_sizes[index], true, false);
Require(name, "sparse byte-range owner",
allocation.first != nullptr &&
(owner == nullptr || owner == allocation.first),
"disjoint byte ranges did not resolve to one cached owner");
owner = allocation.first;
}
// Deliberately change native clean neighbors too: only the exact dirty
// bytes may be published to guest memory.
owner->Fill(owner->Offset(address), sparse_clean.size(),
sparse_values[owner_index]);
};
const auto CheckSparseOwner = [&](uint32_t owner_index) {
const auto address = base + sparse_offset +
owner_index * sparse_owner_stride;
auto expected = sparse_clean;
std::array<uint8_t, sizeof(uint32_t)> value_bytes{};
std::memcpy(value_bytes.data(), &sparse_values[owner_index],
value_bytes.size());
for (size_t index = 0; index < sparse_starts.size(); ++index) {
for (uint64_t byte = sparse_starts[index];
byte < sparse_starts[index] + sparse_sizes[index]; ++byte) {
expected[byte] = value_bytes[byte % value_bytes.size()];
}
}
std::array<uint8_t, 16> published{};
Require(name, "sparse byte-range publication",
Libs::LibKernel::Memory::TryReadBacking(
address, published.data(), published.size()) &&
published == expected &&
!cache.HasGpuDirtyBytes(address, published.size()) &&
!cache.IsRegionGpuModified(address, published.size()),
"odd-byte readback lost dirty bytes or overwrote clean neighbors");
};
FillSparseOwner(0);
bool sparse_callback_ran = false;
bool sparse_callback_clean = false;
scheduler.DeferOperation([&] {
sparse_callback_clean =
!cache.HasGpuDirtyBytes(base + sparse_offset, sparse_clean.size()) &&
!cache.IsRegionGpuModified(base + sparse_offset, sparse_clean.size());
sparse_callback_ran = true;
});
const auto sparse_read_tick = scheduler.CurrentTick();
cache.ReadMemory(base + sparse_offset + sparse_starts[0], sparse_sizes[0]);
Require(name, "sparse byte-range submission",
scheduler.CurrentTick() == sparse_read_tick + 1,
"sparse source ranges were downloaded in separate submissions");
scheduler.PopPendingOperations();
Require(name, "readback callback ownership consistency",
sparse_callback_ran && sparse_callback_clean,
"readback drained a deferred callback before clearing page ownership");
CheckSparseOwner(0);
FillSparseOwner(0);
FillSparseOwner(1);
Require(name, "sparse disjoint GC owners",
BufferCacheTestAccess::PageOwner(cache, base + sparse_offset) !=
BufferCacheTestAccess::PageOwner(
cache, base + sparse_offset + sparse_owner_stride),
"sparse GC fixtures merged into one source owner");
const auto sparse_gc_tick = scheduler.CurrentTick();
for (uint32_t tick = 0; tick <= 160; ++tick) {
cache.RunGarbageCollector();
}
Require(name, "sparse multi-owner GC submission",
scheduler.CurrentTick() == sparse_gc_tick + 1 &&
!cache.IsRegionRegistered(base + sparse_offset,
sparse_owner_stride + 16),
"sparse GC owners did not share one completion drain");
CheckSparseOwner(0);
CheckSparseOwner(1);
constexpr uint64_t disjoint_owner_offset = 0x2140000;
constexpr uint64_t disjoint_owner_size = 0x8000;
constexpr uint64_t disjoint_dirty_offset = disjoint_owner_offset + 0x100;
constexpr uint64_t disjoint_new_owner_offset =
disjoint_owner_offset + 0x5000;
constexpr uint32_t disjoint_value = 0x89abcdefu;
constexpr uint32_t disjoint_stale = 0x76543210u;
std::memcpy(memory + disjoint_dirty_offset, &disjoint_stale,
sizeof(disjoint_stale));
auto wide_owner = cache.ObtainBuffer(base + disjoint_owner_offset,
disjoint_owner_size, false);
Require(name, "disjoint retirement owner", wide_owner.first != nullptr,
"failed to create the clean cross-page retirement owner");
(void)cache.ObtainBuffer(base + disjoint_dirty_offset,
sizeof(disjoint_value), true, false);
cache.FillBuffer(base + disjoint_dirty_offset, sizeof(disjoint_value),
disjoint_value, false);
for (uint32_t tick = 0; tick <= 160; tick++) {
cache.RunGarbageCollector();
}
Require(name, "disjoint synchronized retirement",
!cache.IsRegionRegistered(base + disjoint_owner_offset,
disjoint_owner_size),
"cross-page owner survived pressured collection");
uint32_t disjoint_before_unmap = 0;
Libs::LibKernel::Memory::TryReadBacking(base + disjoint_dirty_offset,
&disjoint_before_unmap,
sizeof(disjoint_before_unmap));
Require(
name, "disjoint retirement publication",
disjoint_before_unmap == disjoint_value,
"whole-owner Buffer retired before publishing its dirty bytes");
resources.UnmapMemory(base + disjoint_owner_offset + 0x4000, 0x4000);
uint32_t disjoint_after_unmap = 0;
Libs::LibKernel::Memory::TryReadBacking(base + disjoint_dirty_offset,
&disjoint_after_unmap,
sizeof(disjoint_after_unmap));
Require(name, "disjoint synchronized invalidation",
disjoint_after_unmap == disjoint_value &&
!cache.HasGpuDirtyBytes(base + disjoint_dirty_offset,
sizeof(disjoint_value)),
"disjoint invalidation lost the completed whole-owner publication");
resources.MapMemory(base + disjoint_owner_offset + 0x4000, 0x4000);
auto disjoint_new_owner =
cache.ObtainBuffer(base + disjoint_new_owner_offset,
sizeof(disjoint_value), true, false);
Require(name, "disjoint post-publication reacquire",
disjoint_new_owner.first != nullptr,
"disjoint acquisition failed after whole-owner publication");
uint32_t disjoint_backing = 0;
Libs::LibKernel::Memory::TryReadBacking(base + disjoint_dirty_offset,
&disjoint_backing,
sizeof(disjoint_backing));
Require(name, "disjoint retirement contents",
disjoint_backing == disjoint_value,
"old retirement callback lost dirty bytes or retained ownership "
"after a disjoint reacquire");
cache.ReadMemory(base + disjoint_new_owner_offset,
sizeof(disjoint_value));
constexpr uint64_t reacquire_owner_offset = 0x2200000;
constexpr uint64_t reacquire_owner_size = 0x8000;
constexpr uint64_t reacquire_dirty_offset =
reacquire_owner_offset + 0x100;
constexpr uint64_t reacquire_disjoint_offset =
reacquire_owner_offset + 0x5000;
constexpr uint32_t reacquire_value = 0xc001d00du;
constexpr uint32_t reacquire_stale = 0x0badf00du;
std::memcpy(memory + reacquire_dirty_offset, &reacquire_stale,
sizeof(reacquire_stale));
auto reacquire_owner = cache.ObtainBuffer(base + reacquire_owner_offset,
reacquire_owner_size, false);
Require(name, "reacquire retirement owner",
reacquire_owner.first != nullptr,
"failed to create a fresh cross-page retirement owner");
(void)cache.ObtainBuffer(base + reacquire_dirty_offset,
sizeof(reacquire_value), true, false);
cache.FillBuffer(base + reacquire_dirty_offset, sizeof(reacquire_value),
reacquire_value, false);
for (uint32_t tick = 0; tick <= 160; tick++) {
cache.RunGarbageCollector();
}
Require(name, "reacquire synchronized retirement",
!cache.IsRegionRegistered(base + reacquire_owner_offset,
reacquire_owner_size),
"fresh cross-page owner survived pressured collection");
uint32_t reacquire_before = 0;
Libs::LibKernel::Memory::TryReadBacking(base + reacquire_dirty_offset,
&reacquire_before,
sizeof(reacquire_before));
Require(name, "reacquire publication completed",
reacquire_before == reacquire_value,
"fresh whole-owner ownership retired before publication");
auto reacquired =
cache.ObtainBuffer(base + reacquire_disjoint_offset,
sizeof(reacquire_value), true, false);
Require(name, "independent disjoint post-publication reacquire",
reacquired.first != nullptr,
"clean disjoint-half acquisition failed after publication");
uint32_t reacquire_after = 0;
Libs::LibKernel::Memory::TryReadBacking(base + reacquire_dirty_offset,
&reacquire_after,
sizeof(reacquire_after));
Require(name, "independent disjoint reacquire contents",
reacquire_after == reacquire_value &&
!cache.HasGpuDirtyBytes(base + reacquire_dirty_offset,
sizeof(reacquire_value)),
"disjoint-half acquisition failed to publish the retired "
"owner's dirty prefix");
cache.ReadMemory(base + reacquire_disjoint_offset,
sizeof(reacquire_value));
resources.UnmapMemory(base, allocation_size);
scheduler.Finish();
}
context.ShutdownGpu();
Require(name, "unmap direct backing",
Libs::LibKernel::Memory::KernelMunmap(base, allocation_size) == 0,
"dirty-GC direct-memory mapping release failed");
Require(name, "release direct backing",
Libs::LibKernel::Memory::KernelReleaseDirectMemory(
direct_offset, allocation_size) == 0,
"dirty-GC direct-memory allocation release failed");
std::printf("[host] %-32s ok\n", name);
}
void CheckComputeMetaClearClassification() {
constexpr const char *name = "ComputeMetaClearClassification";
constexpr uintptr_t base = 0x0000000204200000ull;
constexpr uint64_t allocation_size = 0x10000;
constexpr uint64_t read_only_meta = 0x0000000204201f00ull;
constexpr uint64_t read_write_meta = 0x0000000204202000ull;
constexpr uint64_t write_only_meta = 0x0000000204202100ull;
EnsureRuntimeContext();
int64_t direct_offset = -1;
Require(name, "scratch allocation",
Libs::LibKernel::Memory::KernelAllocateDirectMemory(
0, Libs::LibKernel::Memory::KernelGetDirectMemorySize(),
allocation_size, allocation_size, 0, &direct_offset) == 0,
"metadata scratch allocation failed");
void *mapped = reinterpret_cast<void *>(base);
Require(name, "scratch mapping",
Libs::LibKernel::Memory::KernelMapDirectMemory(
&mapped, allocation_size, 0x3, 0x10, direct_offset,
allocation_size) == 0 && mapped == reinterpret_cast<void *>(base),
"metadata scratch mapping failed");
std::memset(mapped, 0xa5, allocation_size);
RenderContext context(m_runtime_context);
context.InitializeGpu(nullptr);
LibKernel::Memory::InstallGpuResources(&context);
auto &scheduler = context.GetCommandScheduler();
HW::Context registers{};
HW::UserConfig user_config{};
HW::Shader shaders{};
scheduler.Begin(registers, user_config, shaders);
context.MapMemory(base, allocation_size);
auto &texture_cache = context.GetTextureCache();
TextureCacheTestAccess::RegisterHtileMeta(texture_cache, read_only_meta);
TextureCacheTestAccess::RegisterHtileMeta(texture_cache, read_write_meta);
TextureCacheTestAccess::RegisterHtileMeta(texture_cache, write_only_meta);
// An empty dispatch must not inspect even an unreadable shader address.
shaders.SetCsShader({.data_addr = 1});
constexpr std::array<std::array<uint32_t, 3>, 3> empty_dispatches{{
{0, 1, 1}, {1, 0, 1}, {1, 1, 0}}};
for (const auto mode : {0x41u, 0x61u}) {
for (const auto &groups : empty_dispatches) {
context.GetRenderExecutor().DispatchDirect(
0, scheduler.Current(), groups[0], groups[1], groups[2], mode, false);
}
}
Require(name, "HTile fixture",
texture_cache.IsMeta(read_only_meta) &&
texture_cache.IsMeta(read_write_meta) &&
texture_cache.IsMeta(write_only_meta) &&
!texture_cache.IsMetaCleared(read_only_meta, 0) &&
!texture_cache.IsMetaCleared(read_write_meta, 0) &&
!texture_cache.IsMetaCleared(write_only_meta, 0),
"test HTile entries were not registered in their initial state");
const auto MakeInput = [](uint64_t address, bool read, bool written,
ShaderRecompiler::IR::CompiledShaderInfo &program,
ShaderRecompiler::IR::ResourceSnapshot &snapshot) {
program.stage = ShaderType::Compute;
ShaderRecompiler::IR::BufferResource resource{};
resource.read = read;
resource.written = written;
resource.formatted = true;
program.info.buffers.push_back(resource);
ShaderBufferResource descriptor{};
descriptor.UpdateAddress48(address);
descriptor.fields[1] |= 16u << 16u;
descriptor.fields[2] = 8;
descriptor.fields[3] =
DstSel(4, 5, 6, 7) |
(static_cast<uint32_t>(Prospero::BufferFormat::k32_32_32_32UInt)
<< 12u);
ShaderRecompiler::IR::DescriptorValue value{};
value.dword_count = 4;
std::copy_n(descriptor.fields, value.dword_count, value.dwords.begin());
ShaderComputeInputInfo input{};
input.stage.program = &program;
snapshot.buffers.push_back(value);
input.stage.resources = &snapshot;
return input;
};
auto &executor = context.GetRenderExecutor();
auto &command = scheduler.Current();
ShaderRecompiler::IR::CompiledShaderInfo read_only_program{};
ShaderRecompiler::IR::ResourceSnapshot read_only_snapshot;
const auto read_only_input =
MakeInput(read_only_meta, true, false, read_only_program,
read_only_snapshot);
RenderExecutorTestAccess::UpdateComputeMetaClears(
executor, read_only_input, command);
Require(name, "read-only metadata preserved",
!texture_cache.IsMetaCleared(read_only_meta, 0),
"a metadata read-only dispatch changed logical clear state");
ShaderRecompiler::IR::CompiledShaderInfo read_write_program{};
ShaderRecompiler::IR::ResourceSnapshot read_write_snapshot;
const auto read_write_input =
MakeInput(read_write_meta, true, true, read_write_program,
read_write_snapshot);
RenderExecutorTestAccess::UpdateComputeMetaClears(
executor, read_write_input, command);
Require(name, "read/write metadata preserved",
!texture_cache.IsMetaCleared(read_write_meta, 0),
"a metadata read/modify/write dispatch changed logical clear state");
ShaderRecompiler::IR::CompiledShaderInfo write_only_program{};
ShaderRecompiler::IR::ResourceSnapshot write_only_snapshot;
const auto write_only_input =
MakeInput(write_only_meta, false, true, write_only_program,
write_only_snapshot);
RenderExecutorTestAccess::UpdateComputeMetaClears(
executor, write_only_input, command);
Require(name, "write-only metadata clear state",
texture_cache.IsMetaCleared(write_only_meta, 0),
"a metadata write-only dispatch did not update logical clear state");
context.GetGpu().SendCommandSync([&] {
// Scratch reuse retains an old HTile registration, but both native stores
// still have to execute without clearing the unwritten adjacent words.
TextureCacheTestAccess::RegisterHtileMeta(texture_cache, write_only_meta);
std::vector<u32> code;
AppendVMovU32(&code, 1, 0);
for (const auto offset : {0u, 8u}) {
code.push_back(EncodeMubuf0(0x1cu, offset, false, false));
code.push_back(EncodeMubuf1(1, 0, 0));
}
AppendEnd(&code);
ShaderMapUserData(reinterpret_cast<uint64_t>(code.data()),
{.type = Prospero::ShaderBinaryType::kCs,
.code_size_bytes = static_cast<uint32_t>(code.size() * sizeof(u32))});
shaders.SetCsShader({.data_addr = reinterpret_cast<uint64_t>(code.data()),
.num_thread_x = 1, .num_thread_y = 1, .num_thread_z = 1,
.wave_size = 64, .user_sgpr = 4});
ShaderBufferResource descriptor{};
descriptor.UpdateAddress48(write_only_meta);
descriptor.fields[2] = 12;
descriptor.fields[3] = DstSel(4, 5, 6, 7) |
(static_cast<u32>(Prospero::BufferFormat::k32UInt) << 12u);
for (u32 i = 0; i < 4; i++) {
shaders.SetCsUserSgpr(i, descriptor.fields[i], HW::UserSgprType::Unknown);
}
executor.DispatchDirect(0, scheduler.Current(), 1, 1, 1, 0x41u, false);
std::array<u32, 4> actual{};
context.GetBufferCache().ReadMemory(write_only_meta, sizeof(actual));
Require(name, "metadata scratch stores and guards",
Libs::LibKernel::Memory::TryReadBacking(
write_only_meta, actual.data(), sizeof(actual)) &&
actual == std::array<u32, 4>{0, 0xa5a5a5a5u, 0, 0xa5a5a5a5u},
"metadata registration suppressed guest stores or erased adjacent data");
context.UnmapMemory(base, allocation_size);
scheduler.Finish();
});
LibKernel::Memory::InstallGpuResources(nullptr);
context.ShutdownGpu();
Require(name, "scratch unmap",
Libs::LibKernel::Memory::KernelMunmap(base, allocation_size) == 0,
"metadata scratch unmap failed");
Require(name, "scratch release",
Libs::LibKernel::Memory::KernelReleaseDirectMemory(
direct_offset, allocation_size) == 0,
"metadata scratch release failed");
std::printf("[gpu] %-32s ok\n", name);
}
void CheckBdaSyncUploads() {
constexpr const char *name = "BdaSyncUploads";
constexpr auto page = BufferCache::CACHING_PAGESIZE;
EnsureRuntimeContext();
const auto base = LibKernel::Memory::AllocateRuntimeMemory(
0, 3 * page, Common::VirtualMemory::Mode::ReadWrite, "bda_sync_uploads");
Require(name, "allocation", base != 0, "guest allocation failed");
auto *words = reinterpret_cast<u32 *>(base);
std::fill_n(words, 3 * page / sizeof(u32), 0x12345678u);
RenderContext context(m_runtime_context);
auto &scheduler = context.GetCommandScheduler();
auto &cache = context.GetBufferCache();
HW::Context registers{};
HW::UserConfig user_config{};
HW::Shader shaders{};
scheduler.Begin(registers, user_config, shaders);
context.MapMemory(base, page);
context.MapMemory(base + 2 * page, page);
const auto id = cache.FindBuffer(base, 3 * page);
auto &buffer = cache.GetBuffer(id);
auto readback = CreateHostBuffer(name, 6 * sizeof(u32),
vk::BufferUsageFlagBits::eTransferDst, {});
const auto snapshot = [&](uint64_t offset, uint32_t index) {
const vk::BufferCopy copy{offset, index * sizeof(u32), sizeof(u32)};
scheduler.Current().Handle().copyBuffer(buffer.Handle(), readback.buffer, 1, &copy);
};
context.PrepareBda();
Require(name, "mapped intersection",
!cache.IsRegionCpuModified(base, page) &&
cache.IsRegionCpuModified(base + page, page) &&
!cache.IsRegionCpuModified(base + 2 * page, page),
"BDA upload missed a mapped range or uploaded an unmapped hole");
snapshot(0, 0);
snapshot(2 * page, 1);
cache.InvalidateMemory(base + 17, 1);
words[4] = 0x87654321u;
words[TRACKER_PAGE_SIZE / sizeof(u32) - 1] = 0xabcdef01u;
context.PrepareBda();
snapshot(16, 2);
snapshot(TRACKER_PAGE_SIZE - sizeof(u32), 3);
Require(name, "whole-page upload", !cache.IsRegionCpuModified(base, TRACKER_PAGE_SIZE),
"one-byte invalidation did not upload and consume its whole page");
context.MapMemory(base + page, page);
context.PrepareBda();
snapshot(page, 4);
Require(name, "new mapping", !cache.IsRegionCpuModified(base + page, page),
"mapping into an existing buffer did not queue its dirty pages");
context.UnmapMemory(base + 2 * page, page);
words[2 * page / sizeof(u32)] = 0xfedcba98u;
context.PrepareBda();
Require(name, "unmapped dirty page", cache.IsRegionCpuModified(base + 2 * page, page),
"BDA synchronization consumed dirtiness outside mapped memory");
context.MapMemory(base + 2 * page, page);
context.PrepareBda();
snapshot(2 * page, 5);
vk::BufferMemoryBarrier barrier{};
barrier.srcAccessMask = vk::AccessFlagBits::eTransferWrite;
barrier.dstAccessMask = vk::AccessFlagBits::eHostRead;
barrier.srcQueueFamilyIndex = barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
barrier.buffer = readback.buffer;
barrier.size = readback.size;
scheduler.Current().Handle().pipelineBarrier(vk::PipelineStageFlagBits::eTransfer,
vk::PipelineStageFlagBits::eHost, {}, 0, nullptr, 1, &barrier, 0, nullptr);
scheduler.Finish();
Require(name, "GPU contents", ReadBuffer(name, readback, 6) ==
std::vector<u32>{0x12345678u, 0x12345678u, 0x87654321u, 0xabcdef01u,
0x12345678u, 0xfedcba98u},
"initial, re-dirtied, newly mapped or remapped bytes were stale on the GPU");
context.UnmapMemory(base, 3 * page);
scheduler.Finish();
DestroyBuffer(&readback);
Require(name, "release", LibKernel::Memory::FreeGuestMemory(base, 3 * page),
"guest allocation release failed");
std::printf("[gpu] %-32s ok\n", name);
}
void CheckBdaPageTableUploads() {
constexpr const char *name = "BdaPageTableUploads";
constexpr uintptr_t base = 0x0000000207b00000ull;
constexpr uint64_t page = BufferCache::CACHING_PAGESIZE;
constexpr std::array<uint64_t, 3> page_counts{2, 521, 525};
constexpr uint64_t result_entries = page_counts[0] + page_counts[1] + page_counts[2];
EnsureRuntimeContext();
RenderContext context(m_runtime_context);
auto &scheduler = context.GetCommandScheduler();
HW::Context registers{};
HW::UserConfig user_config{};
HW::Shader shaders{};
scheduler.Begin(registers, user_config, shaders);
auto &cache = context.GetBufferCache();
// A small real ring exercises chunking and waits for same-command-buffer reuse.
BufferCacheTestAccess::SetUnusedStagingBufferSize(cache, 4096);
auto &table = *cache.GetBdaPageTableBuffer();
const auto table_offset = BufferCache::PageIndex(base) * sizeof(vk::DeviceAddress);
auto readback = CreateHostBuffer(name, result_entries * sizeof(vk::DeviceAddress),
vk::BufferUsageFlagBits::eTransferDst, {});
std::vector<uint64_t> expected;
expected.reserve(result_entries);
for (const auto pages : page_counts) {
// The destination has a preceding GPU write and, after the first iteration,
// a preceding read. Registration must preserve both ordering requirements.
table.Fill(table_offset, pages * sizeof(vk::DeviceAddress), 0xa5a5a5a5u);
const auto previous_tick = scheduler.CurrentTick();
const auto id = cache.FindBuffer(base, pages * page);
const auto &buffer = cache.GetBuffer(id);
const auto destination_offset = expected.size() * sizeof(vk::DeviceAddress);
for (uint64_t index = 0; index < pages; ++index) {
expected.push_back(buffer.BufferDeviceAddress() + index * page);
}
const vk::BufferCopy copy{table_offset, destination_offset,
pages * sizeof(vk::DeviceAddress)};
scheduler.Current().Handle().copyBuffer(table.Handle(), readback.buffer, 1, &copy);
Require(name, "chunked registration",
buffer.CpuAddress() == base && buffer.Size() == pages * page &&
(pages == page_counts.front() || scheduler.CurrentTick() > previous_tick),
"BDA registration failed to replace its owner or wait for staging-ring reuse");
}
vk::BufferMemoryBarrier barrier{};
barrier.srcAccessMask = vk::AccessFlagBits::eTransferWrite;
barrier.dstAccessMask = vk::AccessFlagBits::eHostRead;
barrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
barrier.buffer = readback.buffer;
barrier.size = readback.size;
scheduler.Current().Handle().pipelineBarrier(vk::PipelineStageFlagBits::eTransfer,
vk::PipelineStageFlagBits::eHost, {}, 0, nullptr, 1, &barrier, 0, nullptr);
scheduler.Finish();
const auto words = ReadBuffer(name, readback, result_entries * 2);
for (size_t index = 0; index < expected.size(); ++index) {
const auto actual = uint64_t{words[index * 2]} | (uint64_t{words[index * 2 + 1]} << 32);
Require(name, "GPU page-table snapshot", actual == expected[index],
"staged BDA updates were stale, overwritten before their read, or incompletely flushed");
}
DestroyBuffer(&readback);
std::printf("[gpu] %-32s ok\n", name);
}
void CheckLargeSparseImageUpload() {
constexpr const char *name = "LargeSparseImageUpload";
constexpr uint64_t size = 576ull << 20, ring_size = 512ull << 20, page = 0x10000;
constexpr uint64_t base = 0x0000000250000000ull;
constexpr std::array<uint64_t, 3> mapped_offsets{0, ring_size, size - page};
EnsureRuntimeContext();
RenderContext context(m_runtime_context);
auto &scheduler = context.GetCommandScheduler();
HW::Context registers{};
HW::UserConfig user_config{};
HW::Shader shaders{};
scheduler.Begin(registers, user_config, shaders);
context.InitializeGpu(nullptr);
void *reservation = reinterpret_cast<void *>(base);
Require(name, "sparse reservation", LibKernel::Memory::KernelReserveVirtualRange(
&reservation, size, 0x10, page) == 0, "large sparse reservation failed");
int64_t direct = -1;
Require(name, "physical pages", LibKernel::Memory::KernelAllocateDirectMemory(
0, LibKernel::Memory::KernelGetDirectMemorySize(), 3u * page, page, 0, &direct) == 0,
"sparse physical allocation failed");
for (u32 index = 0; index < mapped_offsets.size(); ++index) {
void *mapped = reinterpret_cast<void *>(base + mapped_offsets[index]);
Require(name, "map sparse page", LibKernel::Memory::KernelMapDirectMemory(
&mapped, page, 0x3, 0x10, direct + index * page, page) == 0,
"sparse page mapping failed");
auto *words = static_cast<u32 *>(mapped);
std::fill_n(words, page / sizeof(u32), 0u);
words[0] = 0x00020001u + index * 0x100u;
words[1] = 0x00040003u + index * 0x100u;
words[page / sizeof(u32) - 1u] = 0xabcdef01u;
context.MapMemory(base + mapped_offsets[index], page);
}
auto &cache = context.GetBufferCache();
// Exercise the ordinary ring before requesting the captured 576-MiB footprint.
const auto ring_slice = cache.ObtainBufferForImage(base, page);
auto readback = CreateHostBuffer(name, 8u * sizeof(u32),
vk::BufferUsageFlagBits::eTransferDst, {});
const vk::BufferCopy small_copy{ring_slice.second, 0, sizeof(u32)};
scheduler.Current().Handle().copyBuffer(ring_slice.first->Handle(), readback.buffer, 1, &small_copy);
const auto large = cache.ObtainBufferForImage(base, size);
Require(name, "temporary image source", large.first != nullptr && large.first != ring_slice.first &&
large.first->Size() == size && ring_slice.first->Size() == ring_size,
"oversized image upload resized the ring or truncated its source");
constexpr std::array<uint64_t, 5> probes{0, page, ring_size - 4u, ring_size, size - 4u};
std::array<vk::BufferCopy, probes.size()> copies{};
for (u32 index = 0; index < probes.size(); ++index)
copies[index] = {large.second + probes[index], (index + 1u) * sizeof(u32), sizeof(u32)};
scheduler.Current().Handle().copyBuffer(large.first->Handle(), readback.buffer,
copies.size(), copies.data());
// A compute consumer requires the same storage-buffer capability as the detiler.
const auto converted = TextureCacheTestAccess::Tiler(context.GetTextureCache()).TransformColor(
{large.first->Handle(), large.second + ring_size, 8u}, ColorTransform::SwapBgra16, true);
const vk::BufferCopy converted_copy{converted.offset, 6u * sizeof(u32), 8u};
scheduler.Current().Handle().copyBuffer(converted.buffer, readback.buffer, 1, &converted_copy);
vk::BufferMemoryBarrier barrier{};
barrier.srcAccessMask = vk::AccessFlagBits::eTransferWrite;
barrier.dstAccessMask = vk::AccessFlagBits::eHostRead;
barrier.srcQueueFamilyIndex = barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
barrier.buffer = readback.buffer;
barrier.size = readback.size;
scheduler.Current().Handle().pipelineBarrier(vk::PipelineStageFlagBits::eTransfer,
vk::PipelineStageFlagBits::eHost, {}, 0, nullptr, 1, &barrier, 0, nullptr);
scheduler.Finish();
Require(name, "sparse upload and deferred lifetime",
ReadBuffer(name, readback, 8) == std::vector<u32>{0x00020001u, 0x00020001u, 0u, 0u,
0x00020101u, 0xabcdef01u, 0x00020103u, 0x00040101u},
"upload lost sparse holes, boundary bytes, compute access, or in-flight lifetime");
// Cached uploads must read reserved holes without faulting back into the tracker,
// while retaining the private-memory sources accepted by ordinary buffer reads.
const auto private_base = LibKernel::Memory::AllocateRuntimeMemory(
0, page, Common::VirtualMemory::Mode::ReadWrite, "private_buffer_upload");
Require(name, "private source allocation", private_base != 0, "private allocation failed");
auto *private_words = reinterpret_cast<u32 *>(private_base);
private_words[0] = 0x55667788u;
private_words[page / sizeof(u32) - 1u] = 0x99aabbccu;
context.MapMemory(private_base, page);
context.MapMemory(base + page, 2u * page);
const auto sparse_buffer = cache.ObtainBuffer(base, 2u * page, false);
const auto private_buffer = cache.ObtainBuffer(private_base, page, false);
const auto sparse_stream = cache.ObtainBuffer(base + 2u * page, sizeof(u32), false);
Require(name, "sparse stream source",
sparse_stream.first == &cache.GetUtilityBuffer(MemoryUsage::Stream),
"small sparse read did not exercise the stream path");
const std::array<vk::BufferCopy, 2> sparse_copies{{
{sparse_buffer.second, 0, sizeof(u32)},
{sparse_buffer.second + page, sizeof(u32), sizeof(u32)}}};
const std::array<vk::BufferCopy, 2> private_copies{{
{private_buffer.second, 2u * sizeof(u32), sizeof(u32)},
{private_buffer.second + page - sizeof(u32), 3u * sizeof(u32), sizeof(u32)}}};
scheduler.Current().Handle().copyBuffer(sparse_buffer.first->Handle(), readback.buffer,
sparse_copies.size(), sparse_copies.data());
scheduler.Current().Handle().copyBuffer(private_buffer.first->Handle(), readback.buffer,
private_copies.size(), private_copies.data());
const vk::BufferCopy stream_copy{sparse_stream.second, 4u * sizeof(u32), sizeof(u32)};
scheduler.Current().Handle().copyBuffer(sparse_stream.first->Handle(), readback.buffer,
1, &stream_copy);
scheduler.Current().Handle().pipelineBarrier(vk::PipelineStageFlagBits::eTransfer,
vk::PipelineStageFlagBits::eHost, {}, 0, nullptr, 1, &barrier, 0, nullptr);
scheduler.Finish();
Require(name, "cached sparse and private upload",
ReadBuffer(name, readback, 5) == std::vector<u32>{0x00020001u, 0u, 0x55667788u, 0x99aabbccu, 0u} &&
!cache.IsRegionGpuModified(base, 2u * page) &&
!cache.IsRegionGpuModified(private_base, page),
"cached upload lost sparse/private data or changed read-only GPU ownership");
context.UnmapMemory(private_base, page);
context.UnmapMemory(base + page, 2u * page);
Require(name, "private source release", LibKernel::Memory::FreeGuestMemory(private_base, page),
"private allocation release failed");
DestroyBuffer(&readback);
for (const auto offset : mapped_offsets) context.UnmapMemory(base + offset, page);
scheduler.Finish();
context.ShutdownGpu();
Require(name, "release mappings", LibKernel::Memory::KernelMunmap(base, size) == 0,
"sparse mappings release failed");
Require(name, "release physical pages",
LibKernel::Memory::KernelReleaseDirectMemory(direct, 3u * page) == 0,
"sparse physical release failed");
std::printf("[gpu] %-32s ok\n", name);
}
void CheckUnifiedTextureCacheFlow() {
constexpr const char *name = "UnifiedTextureCacheFlow";
constexpr uintptr_t base = 0x0000000200600000ull;
constexpr uint64_t allocation_size = 0x3000000;
constexpr uint64_t allocation_alignment = 0x200000;
EnsureRuntimeContext();
RenderContext context(m_runtime_context);
auto &scheduler = context.GetCommandScheduler();
HW::Context registers{};
HW::UserConfig user_config{};
HW::Shader shaders{};
scheduler.Begin(registers, user_config, shaders);
context.InitializeGpu(nullptr);
auto &gpu = context.GetGpu();
int64_t direct_offset = -1;
Require(name, "direct allocation",
Libs::LibKernel::Memory::KernelAllocateDirectMemory(
0, Libs::LibKernel::Memory::KernelGetDirectMemorySize(),
allocation_size, allocation_alignment, 0, &direct_offset) == 0,
"cache test direct-memory allocation failed");
void *mapped = reinterpret_cast<void *>(base);
Require(name, "direct mapping",
Libs::LibKernel::Memory::KernelMapDirectMemory(
&mapped, allocation_size, 0x3, 0x10, direct_offset,
allocation_alignment) == 0 &&
mapped == reinterpret_cast<void *>(base),
"fixed cache direct-memory mapping failed");
auto *memory = static_cast<uint8_t *>(mapped);
const uint32_t initial = 0x44332211u;
std::memcpy(memory, &initial, sizeof(initial));
{
auto &resources = context;
LibKernel::Memory::InstallGpuResources(&resources);
auto &texture_cache = resources.GetTextureCache();
const auto [narrow_download, narrow_download_offset] =
TextureCacheTestAccess::MapDownload(texture_cache, 4, 4);
const auto [wide_download, wide_download_offset] =
TextureCacheTestAccess::MapDownload(texture_cache, 1, 16);
Require(name, "image download alignment",
narrow_download != nullptr && narrow_download_offset % 4 == 0 &&
wide_download != nullptr && wide_download_offset % 16 == 0,
"wide/block image readback was not aligned to its texel block");
resources.MapMemory(base, allocation_size);
auto sampled =
MakeLinearDesc(base, sizeof(initial), vk::Format::eR8G8B8A8Srgb,
Prospero::BufferFormat::k8_8_8_8Srgb,
Prospero::ImageType::kColor2D, {1, 1, 1}, 1, 4, 1);
auto &command = scheduler.Current();
auto first_desc = sampled;
const auto first = texture_cache.FindImage(first_desc);
auto repeated_desc = sampled;
const auto repeated = texture_cache.FindImage(repeated_desc);
auto compatible_desc = sampled;
compatible_desc.info.pixel_format = vk::Format::eR8G8B8A8Uint;
compatible_desc.info.guest_format = Prospero::BufferFormat::k8_8_8_8UInt;
compatible_desc.view_info.format = compatible_desc.info.pixel_format;
const auto compatible = texture_cache.FindImage(compatible_desc);
Require(name, "normalized FindImage",
first && repeated == first && compatible == first &&
texture_cache.GetImage(first).info.pixel_format ==
vk::Format::eR8G8B8A8Srgb,
"registered compatible backing did not reuse one ImageId");
const auto IsOnlyImage = [](const std::vector<ImageId> &ids,
ImageId expected) {
return ids.size() == 1 && ids.front() == expected;
};
const auto exact_byte_miss =
TextureCacheTestAccess::FindImages(texture_cache, base + 8, 1, false);
const auto touched_page_hit =
TextureCacheTestAccess::FindImages(texture_cache, base + 8, 1, true);
const auto next_page_miss = TextureCacheTestAccess::FindImages(
texture_cache, base + 0x1000, 1, true);
Require(name, "exact and 4-KiB page filtering",
exact_byte_miss.empty() && IsOnlyImage(touched_page_hit, first) &&
next_page_miss.empty(),
"coarse candidates did not preserve exact-byte and touched-page "
"semantics");
const auto MakeOwnershipInfo = [](uint64_t address, uint64_t size) {
ImageInfo info{};
info.data = {address, size};
info.extent = {1, 1, 1};
info.resources = {1, 1};
info.samples = 1;
return info;
};
const auto spanning_info = MakeOwnershipInfo(base + 0x1ff000, 0x2000);
const auto spanning =
TextureCacheTestAccess::InsertImage(texture_cache, spanning_info);
const auto spanning_results = TextureCacheTestAccess::FindImages(
texture_cache, spanning_info.data.address, spanning_info.data.size,
false);
Require(name, "one-MiB cross-page deduplication",
spanning && IsOnlyImage(spanning_results, spanning) &&
TextureCacheTestAccess::PageOwnerCount(
texture_cache, spanning_info.data.address) == 1 &&
TextureCacheTestAccess::PageOwnerCount(
texture_cache, spanning_info.data.End() - 1) == 1,
"an image spanning two coarse pages was missing or returned more "
"than once");
TextureCacheTestAccess::DeleteImage(texture_cache, spanning);
Require(name, "cross-page owner cleanup",
TextureCacheTestAccess::PageOwnerCount(
texture_cache, spanning_info.data.address) == 0 &&
TextureCacheTestAccess::PageOwnerCount(
texture_cache, spanning_info.data.End() - 1) == 0 &&
TextureCacheTestAccess::FindImages(
texture_cache, spanning_info.data.address,
spanning_info.data.size, false)
.empty(),
"cross-page unregister left a stale coarse-page membership");
const auto shared_info = MakeOwnershipInfo(base + 0x500100, 0x100);
const auto shared_first =
TextureCacheTestAccess::InsertImage(texture_cache, shared_info);
const auto shared_second =
TextureCacheTestAccess::InsertImage(texture_cache, shared_info);
const auto shared_results = TextureCacheTestAccess::FindImages(
texture_cache, shared_info.data.address, shared_info.data.size,
false);
Require(name, "shared coarse-page registration",
shared_results.size() == 2 &&
std::find(shared_results.begin(), shared_results.end(),
shared_first) != shared_results.end() &&
std::find(shared_results.begin(), shared_results.end(),
shared_second) != shared_results.end(),
"two registered owners were not retained in one coarse page");
TextureCacheTestAccess::DeleteImage(texture_cache, shared_first);
const auto shared_survivor = TextureCacheTestAccess::FindImages(
texture_cache, shared_info.data.address, shared_info.data.size,
false);
Require(name, "shared coarse-page unregister",
IsOnlyImage(shared_survivor, shared_second) &&
TextureCacheTestAccess::PageOwnerCount(
texture_cache, shared_info.data.address) == 1,
"unregistering one owner removed its coarse-page neighbor");
TextureCacheTestAccess::DeleteImage(texture_cache, shared_second);
Require(name, "final shared coarse-page unregister",
TextureCacheTestAccess::PageOwnerCount(
texture_cache, shared_info.data.address) == 0,
"the final shared owner remained registered");
constexpr uint64_t large_owner_size = 64ull * 1024 * 1024;
const auto large_info =
MakeOwnershipInfo(base + 0x800000, large_owner_size);
const auto large_owner =
TextureCacheTestAccess::InsertImage(texture_cache, large_info);
Require(name, "production one-MiB registration granularity",
TextureCacheTestAccess::OwnedPageCount(
texture_cache, large_info.data.address, large_info.data.size,
large_owner) == 64,
"64 MiB image registration did not create exactly 64 coarse "
"memberships");
TextureCacheTestAccess::DeleteImage(texture_cache, large_owner);
Require(name, "production one-MiB unregister granularity",
TextureCacheTestAccess::OwnedPageCount(
texture_cache, large_info.data.address, large_info.data.size,
large_owner) == 0,
"large image unregister left coarse memberships behind");
const ImageId stale{first.index, first.generation + 1};
TextureCacheTestAccess::AddPageOwner(texture_cache, base, stale);
const auto stale_filtered = TextureCacheTestAccess::FindImages(
texture_cache, base, sizeof(initial), false);
Require(name, "stale page owner filtering",
IsOnlyImage(stale_filtered, first) &&
TextureCacheTestAccess::RemovePageOwner(texture_cache, base,
stale),
"a stale generation escaped the direct page-owner lookup");
TextureCacheTestAccess::SetQueryEpoch(texture_cache, UINT32_MAX);
const auto wrap_results = TextureCacheTestAccess::FindImages(
texture_cache, base, sizeof(initial), false);
Require(name, "page-owner query epoch wrap",
IsOnlyImage(wrap_results, first) &&
TextureCacheTestAccess::QueryEpoch(texture_cache) == 1,
"query deduplication failed while wrapping its epoch");
auto &image = texture_cache.GetImage(first);
constexpr uint32_t final_sampled_value = 0x88776655u;
Require(name, "sampled write between discovery and acquisition",
resources.HandleFault(PageFaultAccess::Write,
sampled.info.data.address),
"sampled image did not accept a CPU update after discovery");
std::memcpy(memory, &final_sampled_value, sizeof(final_sampled_value));
Require(name, "sampled final-acquisition precondition",
image.IsCpuDirty(),
"the between-phase CPU write did not dirty the discovered image");
const auto first_view = texture_cache.FindTexture(first, first_desc);
const auto repeated_view =
texture_cache.FindTexture(repeated, repeated_desc);
auto mapped_view_info = sampled.view_info;
mapped_view_info.mapping.r = vk::ComponentSwizzle::eB;
mapped_view_info.mapping.b = vk::ComponentSwizzle::eR;
const auto mapped_view = image.FindView(mapped_view_info);
Require(
name, "dynamic views",
first_view != nullptr && repeated_view == first_view &&
mapped_view != nullptr && mapped_view != first_view &&
image.views.size() == 2 &&
!image.IsGpuModified() && !image.IsCpuDirty() &&
!TextureCacheTestAccess::PendingDownload(texture_cache, first),
"TextureCache::FindTexture did not preserve sampled view "
"identity, final refresh, and cache-only ownership");
constexpr uint64_t resource_replacement_offset = 0x700000;
const std::array<uint32_t, 3> resource_replacement_data{
0x01020304u, 0x11121314u, 0x21222324u};
std::memcpy(memory + resource_replacement_offset,
resource_replacement_data.data(),
sizeof(resource_replacement_data));
auto underspecified = sampled;
underspecified.info.data = {base + resource_replacement_offset,
sizeof(resource_replacement_data)};
underspecified.info.pixel_format = vk::Format::eR32Uint;
underspecified.info.guest_format = Prospero::BufferFormat::k32UInt;
underspecified.info.extent = {2, 1, 1};
underspecified.info.resources = {1, 1};
underspecified.info.pitch = 2;
underspecified.info.mip_layout[0] = {0, 8, 2, 1, 0, 8, 2};
underspecified.info.mip_layout[1] = {8, 4, 1, 1, 8, 4, 1};
underspecified.info.linear_slice_stride = 0;
underspecified.view_info.format = underspecified.info.pixel_format;
underspecified.view_info.level_count = 1;
const auto underspecified_id = texture_cache.FindImage(underspecified);
(void)texture_cache.FindTexture(underspecified_id, underspecified);
texture_cache.MarkGpuWritten(underspecified_id);
auto complete = underspecified;
complete.info.resources = {2, 1};
complete.view_info.level_count = 2;
const auto complete_id = texture_cache.FindImage(complete);
const auto complete_owner =
TextureCacheTestAccess::Owner(texture_cache, complete_id);
Require(name, "insufficient-resource replacement",
complete_id && complete_id != underspecified_id &&
!TextureCacheTestAccess::Contains(texture_cache,
underspecified_id) &&
complete_owner != nullptr && complete_owner->registered &&
complete_owner->info.resources == complete.info.resources &&
!complete_owner->IsGpuModified() &&
complete_owner->IsDefinitelyCpuDirty(),
"an exact backing with insufficient resources was expanded "
"instead of being discarded and recreated");
ImageInfo chain = sampled.info;
chain.data = {0x2394c1000, 0x384000};
chain.extent = {64, 64, 1};
chain.resources = {7, 150};
chain.pitch = 64;
chain.tile_mode = Prospero::TileMode::kStandard4KB;
chain.mip_layout[0] = {0x2000, 0x258000, 64, 64};
chain.mip_layout[1] = {0x1000, 0x96000, 32, 32};
for (uint32_t level = 2; level < chain.resources.levels; ++level) {
chain.mip_layout[level] = {0, 0x96000, 32, 32};
}
ImageInfo subresource = chain;
subresource.data = {0x2394c3000, 0x4000};
subresource.resources = {1, 1};
subresource.mip_layout = {};
subresource.mip_layout[0] = {0, 0x4000, 64, 64};
const auto mip = subresource.MipOf(chain);
Require(name, "overlap resolution",
mip == 0 && subresource.SliceOf(chain, mip) == 0 &&
ImageRangeOverlaps(chain.data, subresource.data) &&
ImagePageRangesOverlap(chain.data, subresource.data),
"captured PS5 atlas mip/slice overlap did not resolve");
auto exact_desc = compatible_desc;
const auto exact = texture_cache.FindImage(exact_desc, true);
Require(name, "exact-format coexistence",
exact && exact != first &&
TextureCacheTestAccess::Contains(texture_cache, first) &&
texture_cache.GetImage(first).info.pixel_format ==
sampled.info.pixel_format &&
texture_cache.GetImage(exact).info.pixel_format ==
compatible_desc.info.pixel_format,
"exact-format lookup replaced or synchronized its compatible "
"cache record");
auto compatible_after_exact_desc = compatible_desc;
const auto compatible_after_exact =
texture_cache.FindImage(compatible_after_exact_desc);
Require(name, "last compatible format winner",
compatible_after_exact == exact,
"non-exact lookup did not retain the last compatible "
"registration");
ShaderRecompiler::IR::ImageResource null_resource{};
null_resource.resource_class = ShaderRecompiler::IR::ImageResourceClass::Sampled;
null_resource.dimension = ShaderRecompiler::Decoder::ImageDimension::Dim2D;
null_resource.read = true;
ShaderRecompiler::IR::DescriptorValue null_descriptor{};
null_descriptor.dword_count = 8;
for (u32 attempt = 0; attempt < 2; ++attempt) {
const auto binding = RenderExecutorTestAccess::ResolveTexture(
context.GetRenderExecutor(), null_resource, null_descriptor);
const auto native = MakeImageInfo(binding);
Require(name, "null image binding",
!binding.image_id && binding.image_view == nullptr &&
native.imageView == nullptr &&
native.imageLayout == vk::ImageLayout::eGeneral &&
TextureCacheTestAccess::Contains(texture_cache, exact),
"a null descriptor acquired a cache image or disturbed a real image");
}
constexpr uint64_t extent_alias_offset = 0x2400000;
auto narrow_target = sampled;
narrow_target.type = BindingType::RenderTarget;
narrow_target.info.data = {base + extent_alias_offset, 0x10000};
narrow_target.info.pixel_format = vk::Format::eR16G16B16A16Sfloat;
narrow_target.info.guest_format =
Prospero::BufferFormat::k16_16_16_16Float;
narrow_target.info.extent = {1, 1, 1};
narrow_target.info.pitch = 128;
narrow_target.info.bytes_per_block = 8;
narrow_target.info.tile_mode = Prospero::TileMode::kRenderTarget;
narrow_target.info.UpdateSize();
narrow_target.view_info.format = narrow_target.info.pixel_format;
narrow_target.view_info.usage = vk::ImageUsageFlagBits::eColorAttachment;
const auto narrow_target_id = texture_cache.FindImage(narrow_target);
const auto narrow_target_view =
texture_cache.FindRenderTarget(narrow_target_id, narrow_target);
auto wide_target = narrow_target;
wide_target.info.extent.width = 9;
wide_target.info.UpdateSize();
const auto wide_target_id = texture_cache.FindImage(wide_target);
const auto wide_target_view =
texture_cache.FindRenderTarget(wide_target_id, wide_target);
const auto wide_owner =
TextureCacheTestAccess::Owner(texture_cache, wide_target_id);
Require(name, "equal-allocation render-target growth",
narrow_target_view != nullptr && wide_target_view != nullptr &&
wide_target_id != narrow_target_id &&
!TextureCacheTestAccess::Contains(texture_cache,
narrow_target_id) &&
wide_owner != nullptr &&
wide_owner->info.extent == vk::Extent3D{9, 1, 1} &&
wide_owner->backing.extent == vk::Extent3D{9, 1, 1},
"a 9x1 render target reused its equal-size 1x1 native backing");
auto narrow_again = narrow_target;
const auto narrow_again_id = texture_cache.FindImage(narrow_again);
Require(name, "larger render-target backing reuse",
narrow_again_id == wide_target_id &&
texture_cache.GetImage(narrow_again_id).backing.extent ==
vk::Extent3D{9, 1, 1},
"a smaller render area replaced its compatible larger backing");
const auto HostReadBarrier = [&](vk::Buffer buffer, uint64_t size,
vk::PipelineStageFlags source_stage,
vk::AccessFlags source_access) {
vk::BufferMemoryBarrier barrier{};
barrier.sType = vk::StructureType::eBufferMemoryBarrier;
barrier.srcAccessMask = source_access;
barrier.dstAccessMask = vk::AccessFlagBits::eHostRead;
barrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
barrier.buffer = buffer;
barrier.offset = 0;
barrier.size = size;
scheduler.Current().Handle().pipelineBarrier(
source_stage, vk::PipelineStageFlagBits::eHost, {}, 0, nullptr, 1,
&barrier, 0, nullptr);
};
const auto TransferReadBarrier = [&](vk::Buffer buffer, uint64_t size) {
vk::BufferMemoryBarrier barrier{};
barrier.sType = vk::StructureType::eBufferMemoryBarrier;
barrier.srcAccessMask = vk::AccessFlagBits::eTransferWrite;
barrier.dstAccessMask = vk::AccessFlagBits::eTransferRead;
barrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
barrier.buffer = buffer;
barrier.offset = 0;
barrier.size = size;
scheduler.Current().Handle().pipelineBarrier(
vk::PipelineStageFlagBits::eTransfer,
vk::PipelineStageFlagBits::eTransfer, {}, 0, nullptr, 1, &barrier,
0, nullptr);
};
constexpr uint64_t mip_layout_offset = 0x27f0000;
std::vector<u32> mip_guest(0x2000 / sizeof(u32));
std::iota(mip_guest.begin(), mip_guest.end(), 0x51000000u);
ImageId previous{};
for (const uint32_t width : {32u, 64u, 32u}) {
if (previous) {
Require(name, "mip-layout CPU overwrite",
resources.HandleFault(PageFaultAccess::Write,
base + mip_layout_offset),
"mip-layout guest overwrite did not invalidate the old texture");
}
std::memcpy(memory + mip_layout_offset, mip_guest.data(),
mip_guest.size() * sizeof(u32));
TileSurfaceLayout surface{};
Require(name, "mip-tail layout",
TileGetTiledTextureLayout(
{.format = Prospero::BufferFormat::kBc3Srgb,
.tile_mode = Prospero::TileMode::kStandard4KB,
.width = width, .height = 64, .levels = 7}, surface) &&
surface.first_tail_level == (width == 32 ? 0u : 1u) &&
surface.total_size == (width == 32 ? 0x1000u : 0x2000u),
"BC3 width change did not move mip zero into or out of the tail");
ImageDesc desc{};
desc.type = BindingType::Texture;
desc.info.data = {base + mip_layout_offset, surface.total_size};
desc.info.pixel_format = vk::Format::eBc3SrgbBlock;
desc.info.guest_format = Prospero::BufferFormat::kBc3Srgb;
desc.info.extent = {width, 64, 1};
desc.info.resources = {7, 1};
desc.info.pitch = 64;
desc.info.bytes_per_block = 16;
desc.info.tile_mode = Prospero::TileMode::kStandard4KB;
desc.view_info.format = desc.info.pixel_format;
desc.view_info.type = vk::ImageViewType::e2D;
desc.view_info.level_count = 7;
desc.view_info.aspect = vk::ImageAspectFlagBits::eColor;
desc.view_info.usage = vk::ImageUsageFlagBits::eSampled;
std::vector<vk::BufferImageCopy> copies;
std::vector<u32> expected;
for (uint32_t level = 0; level < 7; level++) {
const auto &layout = surface.mips[level];
desc.info.mip_layout[level] = {
layout.offset, layout.size, layout.padded_width,
layout.padded_height};
vk::BufferImageCopy copy{};
copy.bufferOffset = expected.size() * sizeof(u32);
copy.imageSubresource = {vk::ImageAspectFlagBits::eColor, level, 0, 1};
copy.imageExtent = {std::max(width >> level, 1u),
std::max(64u >> level, 1u), 1};
copies.push_back(copy);
for (uint32_t y = 0; y < layout.height; y++) {
for (uint32_t x = 0; x < layout.width; x++) {
uint32_t offset = 0;
Require(name, "mip-tail CPU address",
TileGetBlockOffset(
surface.texture.block, x + layout.tail_x,
y + layout.tail_y, 0, offset),
"BC3 mip-tail block address could not be computed");
const auto word = (layout.offset + offset) / sizeof(u32);
expected.insert(expected.end(), mip_guest.begin() + word,
mip_guest.begin() + word + 4);
}
}
}
desc.info.UpdateSize();
const auto id = texture_cache.FindImage(desc);
Require(name, "distinct mip-layout backing", id && id != previous,
"different guest mip layouts reused the same native image");
(void)texture_cache.FindTexture(id, desc);
auto readback = CreateHostBuffer(
name, expected.size() * sizeof(u32),
vk::BufferUsageFlagBits::eTransferDst,
std::vector<u32>(expected.size()));
texture_cache.GetImage(id).Download(copies, readback.buffer, 0,
readback.size);
HostReadBarrier(readback.buffer, readback.size,
vk::PipelineStageFlagBits::eTransfer,
vk::AccessFlagBits::eTransferWrite);
scheduler.Finish();
Require(name, "mip-layout GPU contents",
ReadBuffer(name, readback, expected.size()) == expected,
"cached native copying contaminated the requested guest mip layout");
DestroyBuffer(&readback);
if (width == 64) {
// Leave only the larger backing so shrinking must reject its layout.
TextureCacheTestAccess::DeleteImage(texture_cache, previous);
}
previous = id;
}
// The game reuses a raw render target allocation for a larger BC5 mip
// chain. Equal block extents do not make their guest tiling compatible.
constexpr uint64_t bc5_alias_offset = 0x1800000;
TileSurfaceLayout bc5_surface{};
Require(name, "BC5 alias layout",
TileGetTiledTextureLayout(
{.format = Prospero::BufferFormat::kBc5UNorm,
.tile_mode = Prospero::TileMode::kStandard64KB,
.width = 1024, .height = 1024, .levels = 11}, bc5_surface) &&
bc5_surface.total_size == 0x160000,
"captured BC5 allocation no longer has its 0x160000-byte footprint");
std::memset(memory + bc5_alias_offset, 0, bc5_surface.total_size);
auto raw_alias = MakeLinearDesc(
base + bc5_alias_offset, 0x100000, vk::Format::eR32G32B32A32Sfloat,
Prospero::BufferFormat::k32_32_32_32Float, Prospero::ImageType::kColor2D,
{256, 256, 1}, 1, 16, 1);
raw_alias.type = BindingType::RenderTarget;
raw_alias.info.tile_mode = Prospero::TileMode::kRenderTarget;
raw_alias.info.UpdateSize();
raw_alias.view_info.usage = vk::ImageUsageFlagBits::eColorAttachment;
const auto raw_alias_id = texture_cache.FindImage(raw_alias);
(void)texture_cache.FindRenderTarget(raw_alias_id, raw_alias);
vk::ClearValue raw_alias_clear{};
std::fill_n(raw_alias_clear.color.float32.data(), 4, 1.0f);
TextureCacheTestAccess::ClearImage(
texture_cache, scheduler.Current(), raw_alias_id,
{vk::ImageAspectFlagBits::eColor, 0, 1, 0, 1}, raw_alias_clear);
Require(name, "BC5 alias CPU overwrite",
resources.InvalidateMemory(base + bc5_alias_offset,
bc5_surface.total_size),
"new BC5 data could not invalidate the old render target");
std::vector<u32> bc5_guest(bc5_surface.total_size / sizeof(u32));
std::iota(bc5_guest.begin(), bc5_guest.end(), 0x41000000u);
std::memcpy(memory + bc5_alias_offset, bc5_guest.data(),
bc5_surface.total_size);
auto bc5_alias = MakeLinearDesc(
base + bc5_alias_offset, bc5_surface.total_size, vk::Format::eBc5UnormBlock,
Prospero::BufferFormat::kBc5UNorm, Prospero::ImageType::kColor2D,
{1024, 1024, 1}, 1, 16, 1);
bc5_alias.info.resources.levels = 11;
bc5_alias.info.tile_mode = Prospero::TileMode::kStandard64KB;
bc5_alias.view_info.level_count = 11;
std::vector<vk::BufferImageCopy> bc5_probes;
std::vector<u32> bc5_expected;
for (u32 level = 0; level < 11; ++level) {
const auto &layout = bc5_surface.mips[level];
bc5_alias.info.mip_layout[level] = {
layout.offset, layout.size, layout.padded_width, layout.padded_height};
u32 offset = 0;
Require(name, "BC5 alias block address",
TileGetBlockOffset(bc5_surface.texture.block, layout.tail_x,
layout.tail_y, 0, offset),
"BC5 mip origin could not be addressed in guest memory");
vk::BufferImageCopy copy{};
copy.bufferOffset = bc5_expected.size() * sizeof(u32);
copy.imageSubresource = {vk::ImageAspectFlagBits::eColor, level, 0, 1};
copy.imageExtent = {std::min(1024u >> level, 4u),
std::min(1024u >> level, 4u), 1};
bc5_probes.push_back(copy);
const auto word = (layout.offset + offset) / sizeof(u32);
bc5_expected.insert(bc5_expected.end(), bc5_guest.begin() + word,
bc5_guest.begin() + word + 4);
}
bc5_alias.info.UpdateSize();
const auto bc5_alias_id = texture_cache.FindImage(bc5_alias);
Require(name, "BC5 alias owner separation",
bc5_alias_id != raw_alias_id &&
TextureCacheTestAccess::Contains(texture_cache, raw_alias_id) &&
!texture_cache.GetImage(bc5_alias_id).usage.render_target &&
texture_cache.GetImage(bc5_alias_id).info.tile_mode ==
Prospero::TileMode::kStandard64KB,
"BC5 image expanded the unrelated raw render target or inherited its usage");
(void)texture_cache.FindTexture(bc5_alias_id, bc5_alias);
auto bc5_readback = CreateHostBuffer(
name, bc5_expected.size() * sizeof(u32),
vk::BufferUsageFlagBits::eTransferDst, std::vector<u32>(bc5_expected.size()));
texture_cache.GetImage(bc5_alias_id).Download(
bc5_probes, bc5_readback.buffer, 0, bc5_readback.size);
HostReadBarrier(bc5_readback.buffer, bc5_readback.size,
vk::PipelineStageFlagBits::eTransfer,
vk::AccessFlagBits::eTransferWrite);
scheduler.Finish();
Require(name, "BC5 alias mip contents",
ReadBuffer(name, bc5_readback, bc5_expected.size()) == bc5_expected,
"raw render-target copying overwrote the BC5 guest mip layout");
DestroyBuffer(&bc5_readback);
{
constexpr uint64_t tiled_array_offset = 0x2c00000;
constexpr uint32_t tiled_levels = 5, tiled_layers = 6;
TileSurfaceLayout surface{};
Require(name, "tiled array alias layout",
TileGetTiledTextureLayout(
{Prospero::BufferFormat::k32UInt, Prospero::TileMode::kStandard64KB,
TileSurfaceDimension::Dim2D, 256, 256, 1, tiled_levels, tiled_layers},
surface) && surface.total_size == 0x240000 &&
surface.block_slice_size == 0x60000 &&
surface.mips[0].offset + surface.mips[0].size * tiled_layers == 0x1a0000,
"fixture lost the gap between an array mip span and its full backing");
auto tiled_array = MakeLinearDesc(
base + tiled_array_offset, surface.total_size, vk::Format::eR32Uint,
Prospero::BufferFormat::k32UInt, Prospero::ImageType::kColor2D,
{256, 256, 1}, tiled_layers, sizeof(uint32_t), 1);
tiled_array.info.tile_mode = Prospero::TileMode::kStandard64KB;
tiled_array.info.resources.levels = tiled_levels;
tiled_array.view_info.level_count = tiled_levels;
for (uint32_t level = 0; level < tiled_levels; ++level) {
const auto &mip = surface.mips[level];
tiled_array.info.mip_layout[level] = {
mip.offset, mip.size * tiled_layers, mip.padded_width, mip.padded_height};
}
tiled_array.info.UpdateSize();
std::memset(memory + tiled_array_offset, 0, surface.total_size);
const auto tiled_array_id = texture_cache.FindImage(tiled_array);
(void)texture_cache.FindTexture(tiled_array_id, tiled_array);
constexpr uint32_t all_mips = 0x12345678u, last_mip = 0x89abcdefu;
vk::ClearValue clear{};
clear.color.uint32[0] = all_mips;
TextureCacheTestAccess::ClearImage(
texture_cache, scheduler.Current(), tiled_array_id,
{vk::ImageAspectFlagBits::eColor, 0, tiled_levels, 0, tiled_layers}, clear);
clear.color.uint32[0] = last_mip;
TextureCacheTestAccess::ClearImage(
texture_cache, scheduler.Current(), tiled_array_id,
{vk::ImageAspectFlagBits::eColor, tiled_levels - 1, 1, tiled_layers - 1, 1}, clear);
Libs::Graphics::Buffer full_array(
m_runtime_context, scheduler, MemoryUsage::DeviceLocal,
tiled_array.info.data.address, AllFlags, surface.total_size);
Libs::Graphics::Buffer partial_array(
m_runtime_context, scheduler, MemoryUsage::DeviceLocal,
tiled_array.info.data.address, AllFlags, 0x1a0000);
Require(name, "tiled array alias containment",
BufferCacheTestAccess::SynchronizeBufferFromImage(
resources.GetBufferCache(), full_array,
tiled_array.info.data.address, surface.total_size) &&
!BufferCacheTestAccess::SynchronizeBufferFromImage(
resources.GetBufferCache(), partial_array,
tiled_array.info.data.address, 0x1a0000),
"full array synchronization failed or accepted an incomplete slice chain");
auto readback = CreateHostBuffer(name, 2 * sizeof(uint32_t),
vk::BufferUsageFlagBits::eTransferDst, {0, 0});
TransferReadBarrier(full_array.Handle(), surface.total_size);
for (uint32_t probe = 0; probe < 2; ++probe) {
const auto &mip = surface.mips[probe == 0 ? 0 : tiled_levels - 1];
uint32_t within_block = 0;
Require(name, "tiled array probe address",
TileGetBlockOffset(surface.texture.block, mip.tail_x, mip.tail_y,
0, within_block),
"last-layer mip origin has no guest address");
const vk::BufferCopy copy{
(tiled_layers - 1) * surface.block_slice_size + mip.offset + within_block,
probe * sizeof(uint32_t), sizeof(uint32_t)};
command.Handle().copyBuffer(full_array.Handle(), readback.buffer, 1, &copy);
}
HostReadBarrier(readback.buffer, readback.size, vk::PipelineStageFlagBits::eTransfer,
vk::AccessFlagBits::eTransferWrite);
scheduler.Finish();
Require(name, "tiled array last-layer contents",
ReadBuffer(name, readback, 2) == std::vector<u32>{all_mips, last_mip},
"array synchronization lost GPU contents beyond the first mip span");
DestroyBuffer(&readback);
}
// A formatted Buffer read must use the private
// Exercise the cache-native image-copy path. Use a request larger
// than the stream shortcut and poison guest backing
// after upload so stale CPU staging cannot accidentally
// satisfy the content check.
constexpr uint64_t mip_prefix_offset = 0x2740000;
constexpr auto mip_format = Prospero::BufferFormat::k32UInt;
constexpr auto mip_tile = Prospero::TileMode::kLinear;
constexpr uint32_t mip_width = 4097;
constexpr uint32_t mip_height = 1;
constexpr uint32_t mip_levels = 1;
const uint32_t mip_pitch =
TileGetTexturePitch(mip_format, mip_width, mip_tile);
TileSizeAlign mip_total{};
std::array<TileSizeOffset, 16> mip_sizes{};
std::array<TilePaddedSize, 16> mip_padded{};
TileGetTextureSize(mip_format, mip_width, mip_height, mip_levels,
mip_tile, &mip_total, mip_sizes.data(),
mip_padded.data());
const uint64_t mip_prefix_size = mip_sizes[0].offset + mip_sizes[0].size;
const uint64_t mip_guest_size = mip_total.size + 256;
Require(name, "formatted mip fixture",
mip_sizes[0].offset == 0 &&
mip_prefix_size > BufferCache::CACHING_PAGESIZE &&
mip_prefix_size < mip_guest_size &&
mip_guest_size % sizeof(uint32_t) == 0,
"single-mip fixture did not expose a cache-sized fitting backing "
"prefix");
std::vector<uint32_t> mip_native(mip_guest_size / sizeof(uint32_t));
std::iota(mip_native.begin(), mip_native.end(), 0x61000000u);
std::memcpy(memory + mip_prefix_offset, mip_native.data(),
mip_guest_size);
auto mip_desc = MakeLinearDesc(
base + mip_prefix_offset, mip_guest_size, vk::Format::eR32Uint,
mip_format, Prospero::ImageType::kColor2D, {mip_width, mip_height, 1},
1, sizeof(uint32_t), 1);
mip_desc.info.resources.levels = mip_levels;
mip_desc.info.pitch = mip_pitch;
mip_desc.view_info.level_count = mip_levels;
for (uint32_t level = 0; level < mip_levels; level++) {
mip_desc.info.mip_layout[level] = {
mip_sizes[level].offset, mip_sizes[level].size,
mip_padded[level].width, mip_padded[level].height};
}
mip_desc.info.UpdateSize();
const auto mip_image = texture_cache.FindImage(mip_desc);
(void)texture_cache.FindTexture(mip_image, mip_desc);
texture_cache.MarkGpuWritten(mip_image);
std::vector<uint32_t> mip_stale(mip_native.size(), 0xdeadbeefu);
Libs::LibKernel::Memory::WriteBacking(base + mip_prefix_offset,
mip_stale.data(), mip_guest_size);
Libs::Graphics::Buffer mip_insufficient(
m_runtime_context, scheduler, MemoryUsage::DeviceLocal,
mip_desc.info.data.address, AllFlags, mip_prefix_size - 1);
Libs::Graphics::Buffer mip_prefix(
m_runtime_context, scheduler, MemoryUsage::DeviceLocal,
mip_desc.info.data.address, AllFlags, mip_prefix_size);
Require(name, "formatted mip containment",
!BufferCacheTestAccess::SynchronizeBufferFromImage(
resources.GetBufferCache(), mip_insufficient,
mip_desc.info.data.address, mip_prefix_size - 1) &&
BufferCacheTestAccess::SynchronizeBufferFromImage(
resources.GetBufferCache(), mip_prefix,
mip_desc.info.data.address, mip_prefix_size) &&
texture_cache.GetImage(mip_image).IsGpuModified(),
"image synchronization accepted a partial first "
"mip, rejected a fitting mip "
"prefix, or transferred ownership");
auto &buffer_cache = resources.GetBufferCache();
std::pair<Libs::Graphics::Buffer *, uint64_t> mip_formatted;
bool formatted_owner_was_absent = false;
gpu.SendCommandSync([&] {
formatted_owner_was_absent = !buffer_cache.IsRegionRegistered(
mip_desc.info.data.address, mip_desc.info.data.size);
mip_formatted = buffer_cache.ObtainBuffer(
mip_desc.info.data.address, mip_desc.info.data.size, false, true);
});
Require(name, "formatted cache fixture", formatted_owner_was_absent,
"formatted image already had a cached Buffer owner");
Require(name, "formatted Buffer path",
mip_formatted.first != nullptr &&
texture_cache.GetImage(mip_image).IsGpuModified(),
"formatted read bypassed the cached image-copy "
"path or transferred ownership");
auto mip_prefix_readback = CreateHostBuffer(
name, mip_prefix_size, vk::BufferUsageFlagBits::eTransferDst,
std::vector<u32>(mip_prefix_size / sizeof(uint32_t), 0));
auto mip_formatted_readback = CreateHostBuffer(
name, mip_guest_size, vk::BufferUsageFlagBits::eTransferDst,
std::vector<u32>(mip_guest_size / sizeof(uint32_t), 0));
TransferReadBarrier(mip_prefix.Handle(), mip_prefix_size);
const vk::BufferCopy mip_prefix_copy{0, 0, mip_prefix_size};
command.Handle().copyBuffer(
mip_prefix.Handle(), mip_prefix_readback.buffer, 1, &mip_prefix_copy);
TransferReadBarrier(mip_formatted.first->Handle(), mip_guest_size);
const vk::BufferCopy mip_formatted_copy{mip_formatted.second, 0,
mip_guest_size};
command.Handle().copyBuffer(mip_formatted.first->Handle(),
mip_formatted_readback.buffer, 1,
&mip_formatted_copy);
HostReadBarrier(mip_prefix_readback.buffer, mip_prefix_readback.size,
vk::PipelineStageFlagBits::eTransfer,
vk::AccessFlagBits::eTransferWrite);
HostReadBarrier(mip_formatted_readback.buffer,
mip_formatted_readback.size,
vk::PipelineStageFlagBits::eTransfer,
vk::AccessFlagBits::eTransferWrite);
const std::array<uint32_t, 2> volume_values{0x10203040u, 0x50607080u};
TileSizeAlign volume_size{};
TileGetTextureTotalSize(Prospero::BufferFormat::k32UInt, 1, 1, 2, 1,
Prospero::TileMode::kLinear, true, volume_size);
const auto volume_pitch = TileGetTexturePitch(
Prospero::BufferFormat::k32UInt, 1, Prospero::TileMode::kLinear);
const auto volume_slice_words = volume_size.size / 2 / sizeof(uint32_t);
std::vector<uint32_t> volume_guest(volume_size.size / sizeof(uint32_t));
volume_guest[0] = volume_values[0];
volume_guest[volume_slice_words] = volume_values[1];
std::memcpy(memory + 0x1000, volume_guest.data(), volume_size.size);
auto array_desc =
MakeLinearDesc(base + 0x1000, volume_size.size,
vk::Format::eR32Uint, Prospero::BufferFormat::k32UInt,
Prospero::ImageType::kColor2D, {1, 1, 1}, 2, 4, 1);
array_desc.info.pitch = volume_pitch;
array_desc.info.UpdateSize();
const auto array_image = texture_cache.FindImage(array_desc);
(void)texture_cache.FindTexture(array_image, array_desc);
texture_cache.MarkGpuWritten(array_image);
auto volume_desc = array_desc;
volume_desc.info.type = Prospero::ImageType::kColor3D;
volume_desc.info.extent = {1, 1, 2};
volume_desc.info.resources.layers = 1;
volume_desc.info.UpdateSize();
volume_desc.view_info.type = vk::ImageViewType::e3D;
volume_desc.view_info.layer_count = 1;
const auto volume_image = texture_cache.FindImage(volume_desc);
Require(name, "2D-array/3D overlap copy",
volume_image && volume_image != array_image &&
texture_cache.GetImage(volume_image).backing.image_type ==
vk::ImageType::e3D &&
texture_cache.GetImage(volume_image).IsGpuModified(),
"array-to-volume expansion lost slices or GPU "
"ownership");
constexpr uint64_t unique_volume_offset = 0x27b0000;
std::memcpy(memory + unique_volume_offset, volume_guest.data(),
volume_size.size);
auto unique_volume_desc = volume_desc;
unique_volume_desc.info.data.address = base + unique_volume_offset;
const auto unique_volume_image =
texture_cache.FindImage(unique_volume_desc);
(void)texture_cache.FindTexture(unique_volume_image, unique_volume_desc);
texture_cache.MarkGpuWritten(unique_volume_image);
Libs::Graphics::Buffer partial_volume(
m_runtime_context, scheduler, MemoryUsage::DeviceLocal,
unique_volume_desc.info.data.address, AllFlags,
unique_volume_desc.info.data.size - sizeof(uint32_t));
Libs::Graphics::Buffer full_volume(
m_runtime_context, scheduler, MemoryUsage::DeviceLocal,
unique_volume_desc.info.data.address, AllFlags,
unique_volume_desc.info.data.size);
Require(name, "formatted volume containment",
!BufferCacheTestAccess::SynchronizeBufferFromImage(
resources.GetBufferCache(), partial_volume,
unique_volume_desc.info.data.address,
unique_volume_desc.info.data.size - sizeof(uint32_t)) &&
BufferCacheTestAccess::SynchronizeBufferFromImage(
resources.GetBufferCache(), full_volume,
unique_volume_desc.info.data.address,
unique_volume_desc.info.data.size) &&
texture_cache.GetImage(unique_volume_image).IsGpuModified(),
"partial 3D synchronization was accepted, "
"full-volume synchronization was "
"rejected, or ownership changed");
auto volume_readback = CreateHostBuffer(
name, volume_size.size, vk::BufferUsageFlagBits::eTransferDst,
std::vector<u32>(volume_guest.size(), 0));
TransferReadBarrier(full_volume.Handle(), volume_size.size);
const vk::BufferCopy volume_copy{0, 0, volume_size.size};
command.Handle().copyBuffer(full_volume.Handle(), volume_readback.buffer,
1, &volume_copy);
HostReadBarrier(volume_readback.buffer, volume_readback.size,
vk::PipelineStageFlagBits::eTransfer,
vk::AccessFlagBits::eTransferWrite);
constexpr uint64_t depth_containment_offset = 0x27c0000;
constexpr std::array<float, 3> depth_containment_values{0.25f, 0.75f,
0.5f};
std::memcpy(memory + depth_containment_offset,
depth_containment_values.data(),
sizeof(depth_containment_values));
auto depth_containment = MakeLinearDesc(
base + depth_containment_offset, sizeof(depth_containment_values),
vk::Format::eD32Sfloat, Prospero::BufferFormat::k32Float,
Prospero::ImageType::kColor2D, {2, 1, 1}, 1, sizeof(float), 1);
depth_containment.type = BindingType::DepthTarget;
depth_containment.info.resources.levels = 2;
depth_containment.info.mip_layout[0] = {
0, 2 * sizeof(float), 2, 1, 0, 2 * sizeof(float), 2};
depth_containment.info.mip_layout[1] = {
2 * sizeof(float), sizeof(float), 1, 1, 2 * sizeof(float), sizeof(float), 1};
depth_containment.info.linear_slice_stride = 0;
depth_containment.view_info.format = vk::Format::eD32Sfloat;
depth_containment.view_info.aspect = vk::ImageAspectFlagBits::eDepth;
depth_containment.view_info.usage =
vk::ImageUsageFlagBits::eDepthStencilAttachment;
depth_containment.view_info.level_count = 2;
const auto depth_containment_image =
texture_cache.FindImage(depth_containment);
(void)texture_cache.FindDepthTarget(depth_containment_image,
depth_containment);
texture_cache.MarkGpuWritten(depth_containment_image);
Libs::Graphics::Buffer partial_depth(
m_runtime_context, scheduler, MemoryUsage::DeviceLocal,
depth_containment.info.data.address, AllFlags,
depth_containment.info.mip_layout[0].size);
Require(
name, "formatted depth containment",
!BufferCacheTestAccess::SynchronizeBufferFromImage(
resources.GetBufferCache(), partial_depth,
depth_containment.info.data.address,
depth_containment.info.mip_layout[0].size) &&
texture_cache.GetImage(depth_containment_image).IsGpuModified(),
"partial depth synchronization was accepted or "
"transferred ownership");
constexpr uint64_t raw_d16_offset = 0x27d0000;
constexpr std::array<uint16_t, 2> raw_d16_values{0x2000u, 0xc000u};
std::memcpy(memory + raw_d16_offset, raw_d16_values.data(),
sizeof(raw_d16_values));
auto raw_d16 = MakeLinearDesc(
base + raw_d16_offset, sizeof(raw_d16_values), vk::Format::eD16Unorm,
Prospero::BufferFormat::k16UNorm, Prospero::ImageType::kColor2D,
{2, 1, 1}, 1, sizeof(uint16_t), 1);
raw_d16.type = BindingType::DepthTarget;
raw_d16.view_info.format = vk::Format::eD16Unorm;
raw_d16.view_info.aspect = vk::ImageAspectFlagBits::eDepth;
raw_d16.view_info.usage = vk::ImageUsageFlagBits::eDepthStencilAttachment;
const auto raw_d16_image = texture_cache.FindImage(raw_d16);
(void)texture_cache.FindDepthTarget(raw_d16_image, raw_d16);
auto raw_d16_uint = raw_d16;
raw_d16_uint.type = BindingType::Texture;
raw_d16_uint.info.pixel_format = vk::Format::eR16Uint;
raw_d16_uint.info.guest_format = Prospero::BufferFormat::k16UInt;
raw_d16_uint.view_info.format = vk::Format::eR16Uint;
raw_d16_uint.view_info.aspect = vk::ImageAspectFlagBits::eColor;
raw_d16_uint.view_info.usage = vk::ImageUsageFlagBits::eSampled;
const auto raw_d16_uint_image = texture_cache.FindImage(raw_d16_uint);
Require(
name, "raw D16 uint texture backing",
raw_d16_uint_image && raw_d16_uint_image != raw_d16_image &&
!TextureCacheTestAccess::Contains(texture_cache, raw_d16_image) &&
texture_cache.GetImage(raw_d16_uint_image).backing.format ==
vk::Format::eR16Uint,
"D16 uint sampling did not recreate a true integer color backing");
Require(name, "raw D16 uint texture view",
texture_cache.FindTexture(raw_d16_uint_image, raw_d16_uint) !=
nullptr,
"D16 uint sampling did not create an integer sampled view");
constexpr uint64_t layered_raw_d16_offset = raw_d16_offset + 0x1000;
constexpr std::array<uint16_t, 12> layered_raw_d16_values{
0x0102u, 0x0304u, 0x1112u, 0x1314u, 0x2122u, 0x2324u,
0x3132u, 0x3334u, 0x4142u, 0x4344u, 0x5152u, 0x5354u};
std::memcpy(memory + layered_raw_d16_offset,
layered_raw_d16_values.data(),
sizeof(layered_raw_d16_values));
auto layered_d16_color = MakeLinearDesc(
base + layered_raw_d16_offset, sizeof(layered_raw_d16_values),
vk::Format::eR16Unorm, Prospero::BufferFormat::k16UNorm,
Prospero::ImageType::kColor2D, {2, 1, 1}, 6, sizeof(uint16_t), 1);
const auto layered_d16_color_image =
texture_cache.FindImage(layered_d16_color);
(void)texture_cache.FindTexture(layered_d16_color_image,
layered_d16_color);
auto &layered_d16_color_native =
texture_cache.GetImage(layered_d16_color_image);
layered_d16_color_native.Transit(vk::ImageLayout::eTransferDstOptimal,
vk::AccessFlagBits2::eTransferWrite, {},
scheduler.Current().Handle());
vk::ClearColorValue layered_raw_d16_clear_value{};
layered_raw_d16_clear_value.float32[0] = 1.0f;
const vk::ImageSubresourceRange layered_raw_d16_clear_range{
vk::ImageAspectFlagBits::eColor, 0, 1, 0, 6};
scheduler.Current().Handle().clearColorImage(
layered_d16_color_native.backing.image,
vk::ImageLayout::eTransferDstOptimal, &layered_raw_d16_clear_value, 1,
&layered_raw_d16_clear_range);
texture_cache.MarkGpuWritten(layered_d16_color_image);
auto layered_raw_d16_depth = layered_d16_color;
layered_raw_d16_depth.type = BindingType::DepthTarget;
layered_raw_d16_depth.info.data.size = sizeof(uint16_t) * 2 * 5;
layered_raw_d16_depth.info.pixel_format = vk::Format::eD16Unorm;
layered_raw_d16_depth.info.guest_format =
Prospero::BufferFormat::k16UNorm;
layered_raw_d16_depth.info.resources.layers = 5;
layered_raw_d16_depth.info.mip_layout[0].size =
layered_raw_d16_depth.info.data.size;
layered_raw_d16_depth.view_info.format = vk::Format::eD16Unorm;
layered_raw_d16_depth.view_info.aspect = vk::ImageAspectFlagBits::eDepth;
layered_raw_d16_depth.view_info.layer_count = 5;
layered_raw_d16_depth.view_info.usage =
vk::ImageUsageFlagBits::eDepthStencilAttachment;
const auto layered_raw_d16_depth_image =
texture_cache.FindImage(layered_raw_d16_depth);
const auto &layered_depth_owner =
texture_cache.GetImage(layered_raw_d16_depth_image);
Require(name, "layered raw D16 owner layout",
layered_raw_d16_depth_image != layered_d16_color_image &&
!TextureCacheTestAccess::Contains(texture_cache,
layered_d16_color_image) &&
layered_depth_owner.info.resources.layers == 6 &&
layered_depth_owner.info.data.size ==
sizeof(layered_raw_d16_values) &&
layered_depth_owner.info.mip_layout[0].size ==
sizeof(layered_raw_d16_values),
"a partial depth view detached the retained six-layer owner from "
"its full guest range");
(void)texture_cache.FindDepthTarget(layered_raw_d16_depth_image,
layered_raw_d16_depth);
auto reacquired_layered_raw_d16_uint = layered_d16_color;
reacquired_layered_raw_d16_uint.info.pixel_format = vk::Format::eR16Uint;
reacquired_layered_raw_d16_uint.info.guest_format =
Prospero::BufferFormat::k16UInt;
reacquired_layered_raw_d16_uint.view_info.format = vk::Format::eR16Uint;
const auto reacquired_layered_raw_d16_uint_image =
texture_cache.FindImage(reacquired_layered_raw_d16_uint);
const auto reacquired_layered_raw_d16_uint_view =
texture_cache.FindTexture(reacquired_layered_raw_d16_uint_image,
reacquired_layered_raw_d16_uint);
auto layered_raw_d16_readback = CreateHostBuffer(
name, sizeof(layered_raw_d16_values),
vk::BufferUsageFlagBits::eTransferDst, std::vector<u32>(6, 0));
auto &reacquired_layered_raw_d16_native =
texture_cache.GetImage(reacquired_layered_raw_d16_uint_image);
reacquired_layered_raw_d16_native.Transit(
vk::ImageLayout::eTransferSrcOptimal,
vk::AccessFlagBits2::eTransferRead, {}, scheduler.Current().Handle());
vk::BufferImageCopy layered_raw_d16_copy{};
layered_raw_d16_copy.bufferRowLength = 2;
layered_raw_d16_copy.imageSubresource = {vk::ImageAspectFlagBits::eColor,
0, 0, 6};
layered_raw_d16_copy.imageExtent = {2, 1, 1};
scheduler.Current().Handle().copyImageToBuffer(
reacquired_layered_raw_d16_native.backing.image,
vk::ImageLayout::eTransferSrcOptimal, layered_raw_d16_readback.buffer,
1, &layered_raw_d16_copy);
HostReadBarrier(layered_raw_d16_readback.buffer,
layered_raw_d16_readback.size,
vk::PipelineStageFlagBits::eTransfer,
vk::AccessFlagBits::eTransferWrite);
scheduler.Finish();
const auto layered_raw_d16_words =
ReadBuffer(name, layered_raw_d16_readback, 6);
std::array<uint16_t, 12> layered_raw_d16_after{};
std::memcpy(layered_raw_d16_after.data(), layered_raw_d16_words.data(),
sizeof(layered_raw_d16_after));
std::array<uint16_t, 12> layered_raw_d16_expected{};
layered_raw_d16_expected.fill(UINT16_MAX);
Require(
name, "layered raw D16 round trip",
reacquired_layered_raw_d16_uint_image &&
reacquired_layered_raw_d16_uint_image !=
layered_raw_d16_depth_image &&
!TextureCacheTestAccess::Contains(texture_cache,
layered_raw_d16_depth_image) &&
reacquired_layered_raw_d16_native.backing.format ==
vk::Format::eR16Uint &&
reacquired_layered_raw_d16_uint_view != nullptr &&
layered_raw_d16_after == layered_raw_d16_expected,
"the partial depth view lost a layer or failed to restore raw UINT "
"sampling");
DestroyBuffer(&layered_raw_d16_readback);
// RE2 converts a prefix of a 32-layer HTile depth array to R32F storage.
constexpr uint32_t htile_layers = 32;
constexpr uint64_t htile_slice_size = 0x10000;
auto htile_depth = MakeLinearDesc(
base + 0x2800000, htile_layers * htile_slice_size,
vk::Format::eD32Sfloat, Prospero::BufferFormat::k32Float,
Prospero::ImageType::kColor2D, {128, 128, 1}, htile_layers, 4, 1);
htile_depth.type = BindingType::DepthTarget;
htile_depth.info.tile_mode = Prospero::TileMode::kDepth;
htile_depth.info.UpdateSize();
htile_depth.info.metadata.kind = ImageMetadataKind::Htile;
htile_depth.info.metadata.range = {base + 0x2a00000, htile_layers * 0x8000};
htile_depth.view_info.aspect = vk::ImageAspectFlagBits::eDepth;
htile_depth.view_info.usage = vk::ImageUsageFlagBits::eDepthStencilAttachment;
const auto htile_depth_image = texture_cache.FindImage(htile_depth);
vk::ClearValue htile_clear{};
htile_clear.depthStencil.depth = 0.25f;
TextureCacheTestAccess::ClearImage(
texture_cache, scheduler.Current(), htile_depth_image,
{vk::ImageAspectFlagBits::eDepth, 0, 1, 0, htile_layers}, htile_clear);
auto htile_storage = htile_depth;
htile_storage.type = BindingType::Storage;
htile_storage.info.pixel_format = vk::Format::eR32Sfloat;
htile_storage.info.metadata = {};
htile_storage.view_info.format = vk::Format::eR32Sfloat;
htile_storage.view_info.aspect = vk::ImageAspectFlagBits::eColor;
htile_storage.view_info.usage = vk::ImageUsageFlagBits::eStorage;
ImageId htile_storage_image{};
for (uint32_t layers : {1u, 2u}) {
htile_storage.info.data.size = layers * htile_slice_size;
htile_storage.info.resources.layers = layers;
htile_storage.info.mip_layout[0].size = htile_storage.info.data.size;
htile_storage.view_info.layer_count = layers;
const auto id = texture_cache.FindImage(htile_storage);
const auto &owner = texture_cache.GetImage(id);
Require(name, "HTile depth storage prefix layout",
id != htile_depth_image &&
(!htile_storage_image || id == htile_storage_image) &&
owner.info.data == htile_depth.info.data &&
owner.info.resources == htile_depth.info.resources &&
owner.info.mip_layout == htile_depth.info.mip_layout &&
owner.backing.layers == htile_layers &&
!owner.info.HasMetadata(),
"partial storage conversion separated array layers from their guest layout");
Require(name, "HTile depth storage prefix view",
texture_cache.FindTexture(id, htile_storage) != nullptr,
"converted depth prefix has no storage view");
htile_storage_image = id;
}
auto htile_readback = CreateHostBuffer(
name, htile_layers * sizeof(float), vk::BufferUsageFlagBits::eTransferDst,
std::vector<u32>(htile_layers));
vk::BufferImageCopy htile_copy{};
htile_copy.imageSubresource = {vk::ImageAspectFlagBits::eColor, 0, 0, htile_layers};
htile_copy.imageExtent = {1, 1, 1};
texture_cache.GetImage(htile_storage_image).Download(
std::span<const vk::BufferImageCopy>(&htile_copy, 1), htile_readback.buffer,
0, htile_readback.size);
HostReadBarrier(htile_readback.buffer, htile_readback.size,
vk::PipelineStageFlagBits::eTransfer,
vk::AccessFlagBits::eTransferWrite);
scheduler.Finish();
Require(name, "HTile depth storage array contents",
ReadBuffer(name, htile_readback, htile_layers) ==
std::vector<u32>(htile_layers, std::bit_cast<u32>(0.25f)),
"partial storage conversion lost native depth data outside its view");
DestroyBuffer(&htile_readback);
auto layered_stencil_desc = MakeLinearDesc(
base + 0x27d4000, 6 * 4 * sizeof(float),
vk::Format::eD32SfloatS8Uint, Prospero::BufferFormat::k32Float,
Prospero::ImageType::kColor2D, {4, 1, 1}, 6, 4, 1);
layered_stencil_desc.type = BindingType::DepthTarget;
// Four live stencil bytes per slice, with allocation padding.
constexpr uint32_t stencil_slice_size = 16;
layered_stencil_desc.info.stencil = {base + 0x27d6000, 6 * stencil_slice_size};
layered_stencil_desc.view_info.aspect =
vk::ImageAspectFlagBits::eDepth | vk::ImageAspectFlagBits::eStencil;
layered_stencil_desc.view_info.usage =
vk::ImageUsageFlagBits::eDepthStencilAttachment;
const auto layered_stencil_id = texture_cache.FindImage(layered_stencil_desc);
auto &layered_stencil_native = texture_cache.GetImage(layered_stencil_id);
layered_stencil_native.Transit(vk::ImageLayout::eTransferDstOptimal,
vk::AccessFlagBits2::eTransferWrite, {}, scheduler.Current().Handle());
const vk::ClearDepthStencilValue layered_stencil_clear{0.75f, 0x91};
const vk::ImageSubresourceRange layered_stencil_range{
layered_stencil_desc.view_info.aspect, 0, 1, 0, 6};
scheduler.Current().Handle().clearDepthStencilImage(
layered_stencil_native.backing.image,
vk::ImageLayout::eTransferDstOptimal, &layered_stencil_clear, 1,
&layered_stencil_range);
texture_cache.MarkGpuWritten(layered_stencil_id);
std::array<uint32_t, 6> stencil_expected;
stencil_expected.fill(0x91919191u);
constexpr std::array<std::array<uint32_t, 3>, 4> stencil_cases{{
{0, 1, 0x35353535u}, {0, 3, 0x57575757u},
{2, 1, 0x79797979u}, {0, 1, 0xa5a5a5a5u}}};
for (uint32_t pass = 0; pass < stencil_cases.size(); pass++) {
const auto [first, layers, stencil_word] = stencil_cases[pass];
auto partial_stencil_desc = layered_stencil_desc;
partial_stencil_desc.info.data.address += first * 4 * sizeof(float);
partial_stencil_desc.info.data.size = layers * 4 * sizeof(float);
partial_stencil_desc.info.stencil.address += first * stencil_slice_size;
partial_stencil_desc.info.stencil.size = layers * stencil_slice_size;
partial_stencil_desc.info.resources.layers = layers;
partial_stencil_desc.info.mip_layout[0].size =
partial_stencil_desc.info.data.size;
partial_stencil_desc.view_info.type = layers == 1
? vk::ImageViewType::e2D : vk::ImageViewType::e2DArray;
partial_stencil_desc.view_info.layer_count = layers;
const auto stencil = partial_stencil_desc.info.stencil;
if (pass != 3) {
(void)resources.GetBufferCache().ObtainBuffer(stencil.address, stencil.size, true);
resources.GetBufferCache().FillBuffer(
stencil.address, stencil.size, stencil_word, false);
const auto partial_stencil_id = texture_cache.FindImage(partial_stencil_desc);
Require(name, "partial stencil retains depth owner",
partial_stencil_id == layered_stencil_id &&
texture_cache.GetImage(partial_stencil_id).info.resources.layers == 6 &&
partial_stencil_desc.view_info.base_layer == first &&
texture_cache.FindDepthTarget(partial_stencil_id,
partial_stencil_desc) != nullptr,
"a partial stencil view replaced its six-layer depth owner");
} else {
// The old prefix association must survive the owner's shifted binding.
Require(name, "partial stencil buffer clear",
texture_cache.ClearImageFromBuffer(scheduler.Current(),
stencil.address, stencil.size, stencil_word),
"the existing partial stencil association could not be cleared");
const auto rebound = texture_cache.FindImage(partial_stencil_desc);
Require(name, "rebind cleared stencil association",
rebound == layered_stencil_id &&
texture_cache.FindDepthTarget(rebound, partial_stencil_desc) != nullptr,
"the cleared stencil association lost its depth owner");
}
std::fill_n(stencil_expected.begin() + first, layers, stencil_word);
auto readback = CreateHostBuffer(name, 30 * sizeof(uint32_t),
vk::BufferUsageFlagBits::eTransferDst, {});
std::vector<vk::BufferImageCopy> copies(2);
copies[0].imageSubresource = {vk::ImageAspectFlagBits::eDepth, 0, 0, 6};
copies[1].bufferOffset = 24 * sizeof(float);
copies[1].imageSubresource = {vk::ImageAspectFlagBits::eStencil, 0, 0, 6};
for (auto &copy : copies) copy.imageExtent = {4, 1, 1};
texture_cache.GetImage(layered_stencil_id).Download(
copies, readback.buffer, 0, readback.size);
HostReadBarrier(readback.buffer, readback.size,
vk::PipelineStageFlagBits::eTransfer,
vk::AccessFlagBits::eTransferWrite);
scheduler.Finish();
const auto words = ReadBuffer(name, readback, 30);
for (uint32_t layer = 0; layer < 6; layer++) {
for (uint32_t pixel = 0; pixel < 4; pixel++) {
Require(name, "partial stencil preserves depth",
words[layer * 4 + pixel] == std::bit_cast<uint32_t>(0.75f),
"stencil upload changed the retained depth plane");
}
Require(name, "partial stencil preserves other layers",
words[24 + layer] == stencil_expected[layer],
"stencil upload used the retained depth allocation's layer count");
}
DestroyBuffer(&readback);
}
auto native_array_info = array_desc.info;
native_array_info.data = {};
auto native_volume_info = volume_desc.info;
native_volume_info.data = {};
Libs::Graphics::Image native_array(m_runtime_context, scheduler,
native_array_info);
Libs::Graphics::Image native_volume(m_runtime_context, scheduler,
native_volume_info);
native_volume.CopyImage(native_array);
native_array.CopyImage(native_volume);
Require(name, "Image-owned 2D-array/3D copy state",
native_volume.backing.state.layout ==
vk::ImageLayout::eTransferSrcOptimal &&
native_volume.backing.state.access_mask ==
vk::AccessFlagBits2::eTransferRead &&
native_array.backing.state.layout ==
vk::ImageLayout::eGeneral &&
native_array.backing.state.access_mask ==
(vk::AccessFlagBits2::eShaderRead |
vk::AccessFlagBits2::eTransferRead),
"Image::CopyImage did not retain pinned "
"source/destination states");
ImageInfo resolve_source_info{};
resolve_source_info.pixel_format = vk::Format::eR8G8B8A8Unorm;
resolve_source_info.guest_format = Prospero::BufferFormat::k8_8_8_8UNorm;
resolve_source_info.type = Prospero::ImageType::kColor2D;
resolve_source_info.extent = {4, 4, 1};
resolve_source_info.resources = {1, 1};
resolve_source_info.pitch = 4;
resolve_source_info.bytes_per_block = 4;
resolve_source_info.samples = 2;
resolve_source_info.tile_mode = Prospero::TileMode::kRenderTarget;
ImageInfo resolve_destination_info = resolve_source_info;
resolve_destination_info.type = Prospero::ImageType::kColor2D;
resolve_destination_info.samples = 1;
Libs::Graphics::Image resolve_source(m_runtime_context, scheduler,
resolve_source_info);
Libs::Graphics::Image resolve_destination(m_runtime_context, scheduler,
resolve_destination_info);
resolve_destination.Resolve(resolve_source, {}, {});
Require(
name, "Image-owned MSAA resolve",
resolve_source.backing.state.layout ==
vk::ImageLayout::eTransferSrcOptimal &&
resolve_destination.backing.state.layout ==
vk::ImageLayout::eTransferDstOptimal,
"Image::Resolve did not issue the MSAA-to-single-sample transfer");
Libs::Graphics::Image copy_destination(m_runtime_context, scheduler,
resolve_destination_info);
copy_destination.Resolve(resolve_destination, {}, {});
Require(name, "Image-owned single-sample resolve copy",
resolve_destination.backing.state.layout ==
vk::ImageLayout::eTransferSrcOptimal &&
copy_destination.backing.state.layout ==
vk::ImageLayout::eTransferDstOptimal,
"Image::Resolve did not retain the single-sample copy path");
const std::array<uint16_t, 2> multisample_source{0x4000u, 0xc000u};
std::memcpy(memory + 0x2000, &multisample_source,
sizeof(multisample_source));
auto color_desc = MakeLinearDesc(
base + 0x2000, sizeof(multisample_source), vk::Format::eR16G16Unorm,
Prospero::BufferFormat::k16_16UNorm, Prospero::ImageType::kColor2D,
{1, 1, 1}, 1, 4, 1);
const auto color_image = texture_cache.FindImage(color_desc);
(void)texture_cache.FindTexture(color_image, color_desc);
texture_cache.MarkGpuWritten(color_image);
const std::array<uint16_t, 2> refreshed_multisample_source{0x2000u,
0xe000u};
Require(name, "unequal-sample source CPU write",
resources.HandleFault(PageFaultAccess::Write,
color_desc.info.data.address),
"color-to-MS-depth source did not accept a CPU update");
std::memcpy(memory + 0x2000, refreshed_multisample_source.data(),
sizeof(refreshed_multisample_source));
constexpr uint64_t ms_stencil_offset = 0x80000;
constexpr uint64_t ms_stencil_size = 0x10000;
auto ms_stencil_owner = resources.GetBufferCache().ObtainBuffer(
base + ms_stencil_offset, ms_stencil_size, false);
Require(name, "MS stencil buffer allocation",
ms_stencil_owner.first != nullptr,
"failed to create an unequal-sample stencil source");
resources.GetBufferCache().FillBuffer(
base + ms_stencil_offset, ms_stencil_size, 0x41414141u, false);
auto ms_depth_desc = color_desc;
ms_depth_desc.type = BindingType::DepthTarget;
ms_depth_desc.info.stencil = {base + ms_stencil_offset, ms_stencil_size};
ms_depth_desc.info.pixel_format = vk::Format::eD24UnormS8Uint;
ms_depth_desc.info.guest_format = Prospero::BufferFormat::k16UNorm;
ms_depth_desc.info.bytes_per_block = 2;
ms_depth_desc.info.samples = 2;
ms_depth_desc.info.type = Prospero::ImageType::kColor2D;
ms_depth_desc.view_info.format = vk::Format::eD24UnormS8Uint;
ms_depth_desc.view_info.aspect =
vk::ImageAspectFlagBits::eDepth | vk::ImageAspectFlagBits::eStencil;
ms_depth_desc.view_info.usage =
vk::ImageUsageFlagBits::eDepthStencilAttachment;
const auto ms_depth_image = texture_cache.FindImage(ms_depth_desc);
constexpr uint64_t ms_htile_address = base + 0x2600000;
constexpr uint64_t ms_htile_size = 0x10000;
ms_depth_desc.info.metadata.range = {ms_htile_address, ms_htile_size};
ms_depth_desc.info.metadata.kind = ImageMetadataKind::Htile;
const auto ms_depth_with_htile = texture_cache.FindImage(ms_depth_desc);
const auto ms_depth_view =
texture_cache.FindDepthTarget(ms_depth_with_htile, ms_depth_desc);
auto sampled_ms_depth = ms_depth_desc;
sampled_ms_depth.type = BindingType::Texture;
sampled_ms_depth.info.metadata = {};
sampled_ms_depth.view_info.usage = vk::ImageUsageFlagBits::eSampled;
const auto sampled_ms_depth_image =
texture_cache.FindImage(sampled_ms_depth);
const auto ms_stencil_alias = texture_cache.FindImageFromRange(
base + ms_stencil_offset, ms_stencil_size, false);
Require(
name, "unequal-sample depth overlap",
ms_depth_image && ms_depth_with_htile == ms_depth_image &&
sampled_ms_depth_image == ms_depth_image &&
ms_depth_image != color_image && ms_depth_view != nullptr &&
texture_cache.GetImage(ms_depth_image).backing.samples == 2 &&
texture_cache.GetImage(ms_depth_image).backing.state.layout ==
vk::ImageLayout::eDepthStencilAttachmentOptimal &&
texture_cache.GetImage(ms_depth_image).IsGpuModified() &&
!texture_cache.GetImage(ms_depth_image).info.htile_initial_clear &&
texture_cache.GetImage(ms_depth_image).info.metadata.range ==
ms_depth_desc.info.metadata.range &&
texture_cache.IsMeta(ms_htile_address) &&
!texture_cache.IsMetaCleared(ms_htile_address, 0) &&
!resources.GetBufferCache().HasGpuDirtyBytes(
base + ms_stencil_offset, ms_stencil_size) &&
ms_stencil_alias &&
texture_cache.GetImage(ms_stencil_alias).depth_id == ms_depth_image &&
texture_cache.GetImage(ms_stencil_alias).backing.image == nullptr,
"unequal-sample overlap did not run the color-to-MS-depth pass "
"without manufacturing stencil ownership");
auto &oversized_ms = texture_cache.GetImage(ms_depth_image);
const auto ms_data_size = oversized_ms.info.data.size;
oversized_ms.info.data.size = (32ull << 20) + 4;
const bool oversized_ms_readback =
!TextureCacheTestAccess::TryDownload(texture_cache, ms_depth_image);
oversized_ms.info.data.size = ms_data_size;
Require(name, "oversized multisample download rejection",
oversized_ms_readback && oversized_ms.IsGpuModified() &&
!oversized_ms.IsBufferModified() &&
!resources.GetBufferCache().HasGpuDirtyBytes(
ms_depth_desc.info.data.address, ms_data_size),
"an unsupported multisample source reserved download storage or "
"published Buffer ownership");
const std::array<uint16_t, 4> multisample_source_4x{0x0000u, 0x4000u,
0x8000u, 0xffffu};
std::memcpy(memory + 0x7000, multisample_source_4x.data(),
sizeof(multisample_source_4x));
auto color_desc_4x =
MakeLinearDesc(base + 0x7000, sizeof(multisample_source_4x),
vk::Format::eR16G16B16A16Unorm,
Prospero::BufferFormat::k16_16_16_16UNorm,
Prospero::ImageType::kColor2D, {1, 1, 1}, 1, 8, 1);
const auto color_image_4x = texture_cache.FindImage(color_desc_4x);
auto ms_depth_desc_4x = color_desc_4x;
ms_depth_desc_4x.type = BindingType::DepthTarget;
ms_depth_desc_4x.info.pixel_format = vk::Format::eD16Unorm;
ms_depth_desc_4x.info.guest_format = Prospero::BufferFormat::k16UNorm;
ms_depth_desc_4x.info.bytes_per_block = 2;
ms_depth_desc_4x.info.samples = 4;
ms_depth_desc_4x.info.type = Prospero::ImageType::kColor2D;
ms_depth_desc_4x.view_info.format = vk::Format::eD16Unorm;
ms_depth_desc_4x.view_info.aspect = vk::ImageAspectFlagBits::eDepth;
ms_depth_desc_4x.view_info.usage =
vk::ImageUsageFlagBits::eDepthStencilAttachment;
const auto ms_depth_image_4x = texture_cache.FindImage(ms_depth_desc_4x);
Require(name, "four-sample depth overlap",
ms_depth_image_4x && ms_depth_image_4x != color_image_4x &&
texture_cache.GetImage(ms_depth_image_4x).backing.samples ==
4,
"four-sample color packing did not produce a depth image");
const auto ms_depth_backing =
texture_cache.GetImage(ms_depth_image).backing.image;
const auto first_stencil_association = texture_cache.FindImageFromRange(
base + ms_stencil_offset, ms_stencil_size, false);
Require(
name, "stencil association shape",
first_stencil_association &&
first_stencil_association != ms_depth_image &&
texture_cache.GetImage(first_stencil_association)
.info.pixel_format == vk::Format::eS8Uint &&
texture_cache.GetImage(first_stencil_association).backing.image ==
nullptr &&
texture_cache.GetImage(first_stencil_association).depth_id ==
ms_depth_image,
"stencil address did not create a lightweight depth association");
Require(
name, "stencil association download rejection",
!TextureCacheTestAccess::TryDownload(texture_cache,
first_stencil_association),
"a lightweight stencil association entered image download planning");
auto exact_ms_depth_alias = ms_depth_desc;
exact_ms_depth_alias.type = BindingType::Texture;
exact_ms_depth_alias.info.stencil = {};
exact_ms_depth_alias.info.pixel_format = vk::Format::eR8G8B8A8Unorm;
exact_ms_depth_alias.info.guest_format =
Prospero::BufferFormat::k8_8_8_8UNorm;
exact_ms_depth_alias.info.bytes_per_block = 4;
exact_ms_depth_alias.view_info.format = vk::Format::eR8G8B8A8Unorm;
exact_ms_depth_alias.view_info.aspect = vk::ImageAspectFlagBits::eColor;
exact_ms_depth_alias.view_info.usage = vk::ImageUsageFlagBits::eSampled;
const auto exact_ms_depth_image =
texture_cache.FindImage(exact_ms_depth_alias, true);
Require(
name, "exact multisample depth/stencil coexistence",
exact_ms_depth_image && exact_ms_depth_image != ms_depth_image &&
TextureCacheTestAccess::Contains(texture_cache, ms_depth_image) &&
texture_cache.GetImage(ms_depth_image).backing.image ==
ms_depth_backing &&
texture_cache.GetImage(first_stencil_association).depth_id ==
ms_depth_image,
"exact multisample depth/stencil lookup replaced the old image "
"or stole its stencil association");
constexpr uint64_t second_stencil_offset = 0x90000;
auto stencil_storage_desc = MakeLinearDesc(
base + second_stencil_offset, ms_stencil_size, vk::Format::eR8Uint,
Prospero::BufferFormat::k8UInt, Prospero::ImageType::kColor2D,
ms_depth_desc.info.extent, 1, 1, 1);
stencil_storage_desc.type = BindingType::Storage;
stencil_storage_desc.view_info.usage = vk::ImageUsageFlagBits::eStorage;
const auto stencil_storage_image =
texture_cache.FindImage(stencil_storage_desc);
(void)texture_cache.FindTexture(stencil_storage_image,
stencil_storage_desc);
auto switched_ms_depth = ms_depth_desc;
switched_ms_depth.info.stencil = {base + second_stencil_offset,
ms_stencil_size};
const auto switched_ms_depth_id =
texture_cache.FindImage(switched_ms_depth);
TextureCacheTestAccess::AssociateStencil(
texture_cache, switched_ms_depth_id, switched_ms_depth.info.stencil);
const auto second_stencil_association = texture_cache.FindImageFromRange(
base + second_stencil_offset, ms_stencil_size, false);
Require(name, "stencil association switch",
switched_ms_depth_id == ms_depth_image &&
texture_cache.GetImage(switched_ms_depth_id).backing.image ==
ms_depth_backing &&
texture_cache.FindImageFromRange(base + ms_stencil_offset,
ms_stencil_size, false) ==
first_stencil_association &&
texture_cache.GetImage(first_stencil_association).depth_id ==
ms_depth_image &&
second_stencil_association &&
second_stencil_association == stencil_storage_image &&
texture_cache.GetImage(second_stencil_association).IsTracked() &&
texture_cache.GetImage(second_stencil_association).depth_id ==
ms_depth_image,
"changing stencil address recreated depth or discarded an "
"existing lightweight association");
Require(name, "CPU write to tracked stencil association",
resources.HandleFault(PageFaultAccess::Write,
base + second_stencil_offset) &&
!texture_cache.GetImage(second_stencil_association).IsTracked() &&
texture_cache.GetImage(second_stencil_association).IsCpuDirty() &&
texture_cache.GetImage(second_stencil_association).depth_id ==
ms_depth_image,
"CPU invalidation left a reused stencil image write-protected");
memory[second_stencil_offset] = 0x5a;
constexpr uint64_t added_stencil_depth_offset = 0x60000;
constexpr uint64_t added_stencil_offset = 0x70000;
constexpr uint64_t added_stencil_size = 0x10000;
constexpr float added_stencil_depth_value = 0.75f;
std::memcpy(memory + added_stencil_depth_offset,
&added_stencil_depth_value,
sizeof(added_stencil_depth_value));
auto depth_only_source = MakeLinearDesc(
base + added_stencil_depth_offset, 0x10000, vk::Format::eD32Sfloat,
Prospero::BufferFormat::k32Float, Prospero::ImageType::kColor2D,
{1, 1, 1}, 1, 4, 1);
depth_only_source.type = BindingType::DepthTarget;
depth_only_source.view_info.format = vk::Format::eD32Sfloat;
depth_only_source.view_info.aspect = vk::ImageAspectFlagBits::eDepth;
depth_only_source.view_info.usage =
vk::ImageUsageFlagBits::eDepthStencilAttachment;
const auto depth_only_source_image =
texture_cache.FindImage(depth_only_source);
auto combined_destination = depth_only_source;
combined_destination.info.stencil = {base + added_stencil_offset,
added_stencil_size};
combined_destination.info.pixel_format = vk::Format::eD32SfloatS8Uint;
combined_destination.view_info.format = vk::Format::eD32SfloatS8Uint;
combined_destination.view_info.aspect =
vk::ImageAspectFlagBits::eDepth | vk::ImageAspectFlagBits::eStencil;
const auto combined_destination_image =
texture_cache.FindImage(combined_destination);
TextureCacheTestAccess::AssociateStencil(
texture_cache, combined_destination_image,
combined_destination.info.stencil);
const auto combined_stencil_association =
texture_cache.FindImageFromRange(base + added_stencil_offset,
added_stencil_size, false);
Require(
name, "depth-only to combined association",
combined_destination_image != depth_only_source_image &&
combined_stencil_association &&
texture_cache.GetImage(combined_stencil_association).depth_id ==
combined_destination_image &&
texture_cache.GetImage(combined_stencil_association)
.backing.image == nullptr,
"stencil addition did not recreate depth and register its "
"lightweight association");
const uint32_t mirror_value = 0xa1b2c3d4u;
std::memcpy(memory + 0x5000, &mirror_value, sizeof(mirror_value));
auto mirror_desc =
MakeLinearDesc(base + 0x5000, sizeof(mirror_value),
vk::Format::eR32Uint, Prospero::BufferFormat::k32UInt,
Prospero::ImageType::kColor2D, {1, 1, 1}, 1, 4, 1);
const auto mirror_image = texture_cache.FindImage(mirror_desc);
(void)texture_cache.FindTexture(mirror_image, mirror_desc);
texture_cache.MarkGpuWritten(mirror_image);
constexpr uint32_t mirror_cpu_value = 0x0f1e2d3cu;
Require(name, "CPU-dirty formatted mirror fault",
resources.HandleFault(PageFaultAccess::Write,
mirror_desc.info.data.address),
"GPU image did not accept a CPU write before Buffer mirroring");
std::memcpy(memory + 0x5000, &mirror_cpu_value, sizeof(mirror_cpu_value));
auto mirror_binding = resources.GetBufferCache().ObtainBuffer(
base + 0x5000, sizeof(mirror_value), false, true);
Require(name, "CPU-dirty formatted mirror source",
mirror_binding.first != nullptr &&
!texture_cache.GetImage(mirror_image).IsBufferModified(),
"formatted mirror did not expose a readable Buffer source");
auto mirror_cpu_readback = CreateHostBuffer(
name, sizeof(mirror_cpu_value), vk::BufferUsageFlagBits::eTransferDst,
std::vector<u32>{0});
const vk::BufferCopy mirror_cpu_copy{mirror_binding.second, 0,
sizeof(mirror_cpu_value)};
command.Handle().copyBuffer(mirror_binding.first->Handle(),
mirror_cpu_readback.buffer, 1,
&mirror_cpu_copy);
HostReadBarrier(mirror_cpu_readback.buffer, mirror_cpu_readback.size,
vk::PipelineStageFlagBits::eTransfer,
vk::AccessFlagBits::eTransferWrite);
auto mirror_refresh_desc = mirror_desc;
const auto mirror_refresh = texture_cache.FindImage(mirror_refresh_desc);
(void)texture_cache.FindTexture(mirror_refresh, mirror_refresh_desc);
Require(name, "buffer-to-image ownership",
mirror_refresh == mirror_image &&
!texture_cache.GetImage(mirror_refresh).IsBufferModified(),
"buffer-backed refresh incorrectly transferred dirty ownership "
"to the image");
texture_cache.MarkGpuWritten(mirror_refresh);
// Unity copies BC mip regions and packed mip tails through integer images.
// The CPU allocation remains white; later array sampling must see GPU writes.
for (uint32_t format_index = 0; format_index < 3; ++format_index) {
constexpr std::array formats{vk::Format::eBc4UnormBlock,
vk::Format::eBc5UnormBlock,
vk::Format::eBc7UnormBlock};
constexpr std::array guest_formats{Prospero::BufferFormat::kBc4UNorm,
Prospero::BufferFormat::kBc5UNorm,
Prospero::BufferFormat::kBc7UNorm};
constexpr uint32_t width = 128, levels = 8, layers = 3;
const uint32_t block_bytes = format_index == 0 ? 8 : 16;
const uint64_t address = base + 0x1900000 + format_index * 0x40000;
auto array = MakeLinearDesc(address, 0, formats[format_index],
guest_formats[format_index], Prospero::ImageType::kColor2D,
{width, width, 1}, layers, block_bytes, 1);
array.info.resources.levels = levels;
array.info.tile_mode = Prospero::TileMode::kStandard4KB;
array.info.UpdateSize();
array.view_info.level_count = levels;
std::memset(memory + address - base, 0xff, array.info.data.size);
auto value_for = [&](uint32_t layer, uint64_t offset) {
return 0x7c000000u | (format_index << 16) | (layer << 8) |
static_cast<uint32_t>(offset / 4096);
};
auto write_raw = [&](uint32_t layer, uint32_t level, uint32_t value) {
const auto &mip = array.info.mip_layout[level];
auto raw = MakeLinearDesc(
address + layer * array.info.tiled_slice_stride + mip.offset, 0,
block_bytes == 8 ? vk::Format::eR32G32Uint
: vk::Format::eR32G32B32A32Uint,
block_bytes == 8 ? Prospero::BufferFormat::k32_32UInt
: Prospero::BufferFormat::k32_32_32_32UInt,
Prospero::ImageType::kColor2D, {mip.pitch, mip.height, 1},
1, block_bytes, 1);
raw.type = BindingType::Storage;
raw.info.tile_mode = array.info.tile_mode;
raw.info.UpdateSize();
raw.view_info.usage = vk::ImageUsageFlagBits::eStorage;
const auto producer = texture_cache.FindImage(raw);
(void)texture_cache.FindTexture(producer, raw);
vk::ClearValue clear{};
std::fill_n(clear.color.uint32.data(), 4, value);
TextureCacheTestAccess::ClearImage(texture_cache, scheduler.Current(),
producer, {vk::ImageAspectFlagBits::eColor, 0, 1, 0, 1}, clear);
};
for (const uint32_t layer : {0u, 2u}) {
uint64_t previous_offset = UINT64_MAX;
for (uint32_t level = 0; level < levels; ++level) {
const auto offset = array.info.mip_layout[level].offset;
if (offset == previous_offset) continue;
previous_offset = offset;
write_raw(layer, level, value_for(layer, offset));
}
}
std::vector<vk::BufferImageCopy> probes;
std::vector<uint32_t> expected;
for (uint32_t layer = 0; layer < layers; ++layer) {
for (uint32_t level = 0; level < levels; ++level) {
vk::BufferImageCopy copy{};
copy.bufferOffset = expected.size() * sizeof(uint32_t);
copy.imageSubresource = {vk::ImageAspectFlagBits::eColor, level, layer, 1};
copy.imageExtent = {std::min(width >> level, 4u),
std::min(width >> level, 4u), 1};
probes.push_back(copy);
expected.insert(expected.end(), block_bytes / sizeof(uint32_t),
layer == 1 ? UINT32_MAX
: value_for(layer, array.info.mip_layout[level].offset));
}
}
auto verify_array = [&] {
const auto consumer = texture_cache.FindImage(array);
(void)texture_cache.FindTexture(consumer, array);
auto readback = CreateHostBuffer(name, expected.size() * sizeof(uint32_t),
vk::BufferUsageFlagBits::eTransferDst, std::vector<uint32_t>(expected.size()));
texture_cache.GetImage(consumer).Download(probes, readback.buffer, 0, readback.size);
HostReadBarrier(readback.buffer, readback.size, vk::PipelineStageFlagBits::eTransfer,
vk::AccessFlagBits::eTransferWrite);
scheduler.Finish();
Require(name, "raw image producers populate BC array",
ReadBuffer(name, readback, expected.size()) == expected,
"BC array lost GPU mip/tail writes or overwrote an untouched layer");
DestroyBuffer(&readback);
};
verify_array();
constexpr uint32_t replacement = 0x13579bdfu;
write_raw(2, array.info.first_tail_level, replacement);
for (uint32_t level = array.info.first_tail_level; level < levels; ++level) {
const auto word = (2 * levels + level) * block_bytes / sizeof(uint32_t);
std::fill_n(expected.begin() + word, block_bytes / sizeof(uint32_t), replacement);
}
verify_array();
}
// A write through a separate buffer must not replace untouched image
// texels with stale backing bytes. Exact-base writes still refresh it.
constexpr uint64_t buffer_alias_offset = 0x1a00000;
constexpr uint32_t buffer_alias_image_value = 0x13579bdfu;
constexpr uint32_t buffer_alias_write_value = 0x2468ace0u;
std::memset(memory + buffer_alias_offset, 0, 4 * sizeof(uint32_t));
auto buffer_alias_desc = MakeLinearDesc(
base + buffer_alias_offset, 4 * sizeof(uint32_t), vk::Format::eR32Uint,
Prospero::BufferFormat::k32UInt, Prospero::ImageType::kColor2D,
{4, 1, 1}, 1, 4, 1);
const auto buffer_alias_image = texture_cache.FindImage(buffer_alias_desc);
for (const uint32_t write_offset : {0u, 12u}) {
vk::ClearValue painted{};
painted.color.uint32 = std::array{buffer_alias_image_value, 0u, 0u, 0u};
TextureCacheTestAccess::ClearImage(
texture_cache, scheduler.Current(), buffer_alias_image,
{vk::ImageAspectFlagBits::eColor, 0, 1, 0, 1}, painted);
const auto write_address = buffer_alias_desc.info.data.address + write_offset;
auto writable = resources.GetBufferCache().ObtainBuffer(
write_address, sizeof(uint32_t), true, false);
Require(name, "image alias buffer allocation", writable.first != nullptr,
"failed to allocate the image alias write buffer");
writable.first->Fill(writable.second, sizeof(uint32_t), buffer_alias_write_value);
texture_cache.InvalidateMemoryFromGPU(write_address, sizeof(uint32_t));
(void)texture_cache.FindTexture(buffer_alias_image, buffer_alias_desc);
const bool exact_base = write_offset == 0;
const uint32_t read_texels = exact_base ? 1 : 3;
Require(name, exact_base ? "exact-base buffer write refresh"
: "overlapping buffer preserves untouched texels",
ReadCachedTexel(name, context, buffer_alias_image, {},
{read_texels, 1, 1}) ==
std::vector<u32>(read_texels, exact_base ? buffer_alias_write_value
: buffer_alias_image_value),
exact_base ? "exact-base buffer write did not refresh the native image"
: "overlapping buffer write replaced untouched GPU image texels");
}
constexpr uint64_t exact_buffer_offset = 0x90000;
constexpr uint32_t exact_buffer_value = 0x3f234567u;
std::memcpy(memory + exact_buffer_offset, &exact_buffer_value,
sizeof(exact_buffer_value));
auto exact_buffer_desc =
MakeLinearDesc(base + exact_buffer_offset, sizeof(exact_buffer_value),
vk::Format::eR32Uint, Prospero::BufferFormat::k32UInt,
Prospero::ImageType::kColor2D, {1, 1, 1}, 1, 4, 1);
const auto exact_buffer_image =
texture_cache.FindImage(exact_buffer_desc);
(void)texture_cache.FindTexture(exact_buffer_image, exact_buffer_desc);
texture_cache.MarkGpuWritten(exact_buffer_image);
auto exact_buffer_binding = resources.GetBufferCache().ObtainBuffer(
exact_buffer_desc.info.data.address, exact_buffer_desc.info.data.size,
false, true);
Require(
name, "exact replacement Buffer synchronization",
exact_buffer_binding.first != nullptr &&
!texture_cache.GetImage(exact_buffer_image).IsBufferModified() &&
texture_cache.GetImage(exact_buffer_image).IsGpuModified(),
"exact replacement copy transferred cache ownership");
auto exact_float_desc = exact_buffer_desc;
exact_float_desc.info.pixel_format = vk::Format::eR32Sfloat;
exact_float_desc.info.guest_format = Prospero::BufferFormat::k32Float;
exact_float_desc.view_info.format = vk::Format::eR32Sfloat;
const auto exact_float_image =
texture_cache.FindImage(exact_float_desc, true);
(void)texture_cache.FindTexture(exact_float_image, exact_float_desc);
Require(
name, "Buffer-superseded exact coexistence",
exact_float_image != exact_buffer_image &&
TextureCacheTestAccess::Contains(texture_cache,
exact_buffer_image) &&
texture_cache.GetImage(exact_buffer_image).IsGpuModified() &&
!texture_cache.GetImage(exact_float_image).IsBufferModified() &&
!texture_cache.GetImage(exact_float_image).IsGpuModified(),
"exact-format lookup retired its old record or transferred Buffer "
"ownership");
auto exact_buffer_readback = CreateHostBuffer(
name, sizeof(exact_buffer_value),
vk::BufferUsageFlagBits::eTransferDst, std::vector<u32>{0});
auto &exact_float_native = texture_cache.GetImage(exact_float_image);
exact_float_native.Transit(vk::ImageLayout::eTransferSrcOptimal,
vk::AccessFlagBits2::eTransferRead, {},
command.Handle());
vk::BufferImageCopy exact_float_copy{};
exact_float_copy.imageSubresource.aspectMask =
vk::ImageAspectFlagBits::eColor;
exact_float_copy.imageSubresource.layerCount = 1;
exact_float_copy.imageExtent = {1, 1, 1};
command.Handle().copyImageToBuffer(exact_float_native.backing.image,
vk::ImageLayout::eTransferSrcOptimal,
exact_buffer_readback.buffer, 1,
&exact_float_copy);
HostReadBarrier(exact_buffer_readback.buffer, exact_buffer_readback.size,
vk::PipelineStageFlagBits::eTransfer,
vk::AccessFlagBits::eTransferWrite);
const uint32_t clear_source = 0x01010101u;
std::memcpy(memory + 0x6000, &clear_source, sizeof(clear_source));
auto clear_desc = MakeLinearDesc(
base + 0x6000, sizeof(clear_source), vk::Format::eR8G8B8A8Unorm,
Prospero::BufferFormat::k8_8_8_8UNorm, Prospero::ImageType::kColor2D,
{1, 1, 1}, 1, 4, 1);
const auto clear_image = texture_cache.FindImage(clear_desc);
auto buffer_write = resources.GetBufferCache().ObtainBuffer(
base + 0x6000, sizeof(clear_source), true, true);
texture_cache.InvalidateMemoryFromGPU(base + 0x6000,
sizeof(clear_source));
Require(name, "buffer-write ownership",
buffer_write.first != nullptr &&
texture_cache.GetImage(clear_image).IsBufferModified(),
"formatted buffer write did not supersede the image");
Require(name, "buffer-owned full image clear",
texture_cache.ClearImageFromBuffer(
command, base + 0x6000, sizeof(clear_source), 0xaabbccddu) &&
!texture_cache.GetImage(clear_image).IsBufferModified() &&
texture_cache.GetImage(clear_image).IsGpuModified(),
"full image clear retained dual buffer/image GPU ownership");
texture_cache.MarkGpuWritten(clear_image);
auto clear_read_desc = clear_desc;
Require(name, "post-clear image reuse",
texture_cache.FindImage(clear_read_desc) == clear_image,
"cleared image could not be read or written after ownership "
"transfer");
auto stencil_clear_desc = MakeLinearDesc(
base + 0x27e0000, 0x1000, vk::Format::eD32SfloatS8Uint,
Prospero::BufferFormat::k32Float, Prospero::ImageType::kColor2D,
{13, 11, 1}, 2, 4, 1);
stencil_clear_desc.info.stencil = {base + 0x27e2000, 0x1000};
stencil_clear_desc.info.resources.levels = 2;
stencil_clear_desc.info.mip_layout[0] = {0, 0x800, 13, 11, 0, 0x400, 13};
stencil_clear_desc.info.mip_layout[1] = {0x800, 0x800, 6, 5, 0x800, 0x400, 6};
stencil_clear_desc.info.linear_slice_stride = 0x400;
stencil_clear_desc.view_info.aspect =
vk::ImageAspectFlagBits::eDepth | vk::ImageAspectFlagBits::eStencil;
const auto stencil_clear_id = texture_cache.FindImage(stencil_clear_desc);
auto &stencil_clear_image = texture_cache.GetImage(stencil_clear_id);
stencil_clear_image.Transit(vk::ImageLayout::eTransferDstOptimal,
vk::AccessFlagBits2::eTransferWrite, {}, command.Handle());
const vk::ClearDepthStencilValue initial_depth_stencil{0.75f, 0x91};
const vk::ImageSubresourceRange all_depth_stencil{
stencil_clear_desc.view_info.aspect, 0, 2, 0, 2};
command.Handle().clearDepthStencilImage(stencil_clear_image.backing.image,
vk::ImageLayout::eTransferDstOptimal, &initial_depth_stencil, 1,
&all_depth_stencil);
texture_cache.MarkGpuWritten(stencil_clear_id);
vk::ClearValue stencil_value{};
stencil_value.depthStencil = vk::ClearDepthStencilValue{0.f, 0x35};
TextureCacheTestAccess::ClearImage(texture_cache, command, stencil_clear_id,
{vk::ImageAspectFlagBits::eStencil, 1, 1, 1, 1}, stencil_value);
// Read every mip/layer of both aspects to prove that the stencil clear is local.
auto stencil_readback = CreateHostBuffer(name, 0x4000,
vk::BufferUsageFlagBits::eTransferDst, {});
std::vector<vk::BufferImageCopy> stencil_copies;
for (uint32_t mip = 0; mip < 2; mip++) {
for (uint32_t layer = 0; layer < 2; layer++) {
for (const auto aspect : {vk::ImageAspectFlagBits::eDepth,
vk::ImageAspectFlagBits::eStencil}) {
vk::BufferImageCopy copy{};
copy.bufferOffset = stencil_copies.size() * 0x800;
copy.imageSubresource = {aspect, mip, layer, 1};
copy.imageExtent = {13u >> mip, 11u >> mip, 1};
stencil_copies.push_back(copy);
}
}
}
stencil_clear_image.Download(stencil_copies, stencil_readback.buffer, 0,
stencil_readback.size);
HostReadBarrier(stencil_readback.buffer, stencil_readback.size,
vk::PipelineStageFlagBits::eTransfer,
vk::AccessFlagBits::eTransferWrite);
scheduler.Finish();
const auto stencil_words = ReadBuffer(name, stencil_readback, 0x1000);
const auto *stencil_bytes = reinterpret_cast<const uint8_t *>(stencil_words.data());
for (const auto &copy : stencil_copies) {
const auto &sub = copy.imageSubresource;
const auto count = copy.imageExtent.width * copy.imageExtent.height;
const auto *bytes = stencil_bytes + copy.bufferOffset;
const bool depth = sub.aspectMask == vk::ImageAspectFlagBits::eDepth;
uint32_t expected = 0x91u;
if (depth) {
expected = std::bit_cast<uint32_t>(0.75f);
} else if (sub.mipLevel == 1 && sub.baseArrayLayer == 1) {
expected = 0x35u;
}
for (uint32_t pixel = 0; pixel < count; pixel++) {
uint32_t actual = bytes[pixel];
if (depth) std::memcpy(&actual, bytes + pixel * 4, 4);
Require(name, "stencil clear preserves depth and neighbors", actual == expected,
"selected stencil clear changed depth, another mip, or another layer");
}
}
DestroyBuffer(&stencil_readback);
constexpr uint64_t partial_image_offset = 0xa000;
constexpr uint64_t partial_buffer_offset = 0xa010;
constexpr uint64_t partial_clean_offset = 0xa020;
constexpr uint32_t partial_image_value = 0x31415926u;
constexpr uint32_t partial_buffer_value = 0x27182818u;
constexpr uint32_t partial_clean_value = 0xabcdef01u;
std::memcpy(memory + partial_clean_offset, &partial_clean_value,
sizeof(partial_clean_value));
auto partial_write = resources.GetBufferCache().ObtainBuffer(
base + partial_image_offset, sizeof(partial_image_value), true);
Require(name, "partial-page buffer allocation",
partial_write.first != nullptr,
"partial-page GPU write did not create a native buffer");
resources.GetBufferCache().FillBuffer(base + partial_image_offset,
sizeof(partial_image_value),
partial_image_value, false);
resources.GetBufferCache().FillBuffer(base + partial_buffer_offset,
sizeof(partial_buffer_value),
partial_buffer_value, false);
auto partial_desc = MakeLinearDesc(
base + partial_image_offset, sizeof(partial_image_value),
vk::Format::eR32Uint, Prospero::BufferFormat::k32UInt,
Prospero::ImageType::kColor2D, {1, 1, 1}, 1, 4, 1);
const auto partial_image = texture_cache.FindImage(partial_desc);
Require(name, "partial-page image upload",
!texture_cache.GetImage(partial_image).IsGpuModified(),
"image upload incorrectly consumed buffer dirty ownership");
auto partial_image_mirror = resources.GetBufferCache().ObtainBuffer(
base + partial_image_offset, sizeof(partial_image_value), false,
true);
Require(name, "partial-page image mirror",
partial_image_mirror.first != nullptr &&
!texture_cache.GetImage(partial_image).IsGpuModified(),
"same-page image upload lost its buffer source");
const auto partial_clean_source =
resources.GetBufferCache().ObtainBufferForImage(
base + partial_clean_offset, sizeof(partial_clean_value));
Require(name, "partial-page clean source",
partial_clean_source.first != nullptr,
"clean same-page bytes inherited unrelated buffer ownership");
Require(name, "partial-page remaining fault",
resources.HandleFault(PageFaultAccess::Read,
base + partial_buffer_offset),
"same-page buffer ownership was lost after image transfer");
uint32_t partial_buffer_backing = 0;
std::memcpy(&partial_buffer_backing, memory + partial_buffer_offset,
sizeof(partial_buffer_backing));
uint32_t partial_image_backing = 0;
std::memcpy(&partial_image_backing, memory + partial_image_offset,
sizeof(partial_image_backing));
Require(name, "partial-page readback values",
partial_image_backing == partial_image_value &&
partial_buffer_backing == partial_buffer_value,
"same-page image/buffer readback published incorrect bytes");
constexpr uint32_t partial_cpu_refresh_value = 0x13579bdfu;
Require(name, "partial-page CPU write fault",
resources.HandleFault(PageFaultAccess::Write,
base + partial_clean_offset),
"same-page CPU write did not invalidate cached ownership");
std::memcpy(memory + partial_clean_offset, &partial_cpu_refresh_value,
sizeof(partial_cpu_refresh_value));
constexpr std::array<uint32_t, 2> partial_neighbors{0x2468ace0u,
0xfedcba98u};
std::memcpy(memory + partial_clean_offset - sizeof(uint32_t),
&partial_neighbors[0], sizeof(uint32_t));
std::memcpy(memory + partial_clean_offset + sizeof(uint32_t),
&partial_neighbors[1], sizeof(uint32_t));
std::vector<u32> partial_cpu_refresh_expected(TRACKER_PAGE_SIZE / sizeof(u32));
std::memcpy(partial_cpu_refresh_expected.data(), memory + partial_image_offset,
TRACKER_PAGE_SIZE);
const auto partial_cpu_refresh_source =
resources.GetBufferCache().ObtainBufferForImage(
base + partial_clean_offset, sizeof(partial_cpu_refresh_value));
Require(name, "partial-page CPU refresh source",
partial_cpu_refresh_source.first != nullptr,
"CPU-dirty same-page bytes did not resolve through the cached "
"buffer");
auto partial_cpu_refresh_readback =
CreateHostBuffer(name, TRACKER_PAGE_SIZE,
vk::BufferUsageFlagBits::eTransferDst, {0});
const vk::BufferCopy partial_cpu_refresh_copy{
partial_cpu_refresh_source.first->Offset(base + partial_image_offset), 0,
TRACKER_PAGE_SIZE};
command.Handle().copyBuffer(partial_cpu_refresh_source.first->Handle(),
partial_cpu_refresh_readback.buffer, 1,
&partial_cpu_refresh_copy);
vk::BufferMemoryBarrier partial_cpu_refresh_barrier{};
partial_cpu_refresh_barrier.sType =
vk::StructureType::eBufferMemoryBarrier;
partial_cpu_refresh_barrier.srcAccessMask =
vk::AccessFlagBits::eTransferWrite;
partial_cpu_refresh_barrier.dstAccessMask = vk::AccessFlagBits::eHostRead;
partial_cpu_refresh_barrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
partial_cpu_refresh_barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
partial_cpu_refresh_barrier.buffer = partial_cpu_refresh_readback.buffer;
partial_cpu_refresh_barrier.size = partial_cpu_refresh_readback.size;
command.Handle().pipelineBarrier(vk::PipelineStageFlagBits::eTransfer,
vk::PipelineStageFlagBits::eHost, {}, 0,
nullptr, 1, &partial_cpu_refresh_barrier,
0, nullptr);
const uint32_t fault_a = 0x01020304u;
const uint32_t fault_b = 0x11121314u;
std::memcpy(memory + 0x8000, &fault_a, sizeof(fault_a));
std::memcpy(memory + 0x8010, &fault_b, sizeof(fault_b));
auto fault_a_desc =
MakeLinearDesc(base + 0x8000, sizeof(fault_a), vk::Format::eR32Uint,
Prospero::BufferFormat::k32UInt,
Prospero::ImageType::kColor2D, {1, 1, 1}, 1, 4, 1);
auto fault_b_desc = fault_a_desc;
fault_b_desc.info.data.address = base + 0x8010;
const auto fault_a_image = texture_cache.FindImage(fault_a_desc);
const auto fault_b_image = texture_cache.FindImage(fault_b_desc);
(void)texture_cache.FindTexture(fault_a_image, fault_a_desc);
(void)texture_cache.FindTexture(fault_b_image, fault_b_desc);
texture_cache.MarkGpuWritten(fault_a_image);
texture_cache.MarkGpuWritten(fault_b_image);
Require(name, "per-image watcher installation",
texture_cache.GetImage(fault_a_image).IsTracked() &&
texture_cache.GetImage(fault_b_image).IsTracked(),
"same-page images did not install independent write watchers");
constexpr uint64_t padding_fault_offset = 0x8080;
uint32_t write_only_read_a = 0;
uint32_t write_only_read_b = 0;
std::memcpy(&write_only_read_a, memory + 0x8000,
sizeof(write_only_read_a));
std::memcpy(&write_only_read_b, memory + 0x8010,
sizeof(write_only_read_b));
Require(name, "write-only image read policy",
write_only_read_a == fault_a && write_only_read_b == fault_b &&
texture_cache.GetImage(fault_a_image).IsGpuModified() &&
texture_cache.GetImage(fault_b_image).IsGpuModified(),
"TextureCache claimed a CPU read fault instead of preserving "
"GPU image ownership");
Require(name, "same-page image write invalidation",
resources.HandleFault(PageFaultAccess::Write,
base + padding_fault_offset) &&
texture_cache.GetImage(fault_a_image).IsGpuModified() &&
texture_cache.GetImage(fault_b_image).IsGpuModified() &&
!texture_cache.GetImage(fault_a_image).IsTracked() &&
!texture_cache.GetImage(fault_b_image).IsTracked() &&
texture_cache.GetImage(fault_a_image).IsMaybeCpuDirty() &&
texture_cache.GetImage(fault_b_image).IsMaybeCpuDirty() &&
!texture_cache.GetImage(fault_a_image).IsDefinitelyCpuDirty() &&
!texture_cache.GetImage(fault_b_image).IsDefinitelyCpuDirty(),
"a byte-disjoint CPU write discarded authoritative images");
const auto retracked_a = texture_cache.FindImage(fault_a_desc);
const auto retracked_b = texture_cache.FindImage(fault_b_desc);
(void)texture_cache.FindTexture(retracked_a, fault_a_desc);
(void)texture_cache.FindTexture(retracked_b, fault_b_desc);
auto fault_a_mirror = resources.GetBufferCache().ObtainBuffer(
base + 0x8000, sizeof(fault_a), false, true);
Require(name, "same-page image re-track",
retracked_a == fault_a_image && retracked_b == fault_b_image &&
texture_cache.GetImage(fault_a_image).IsTracked() &&
texture_cache.GetImage(fault_b_image).IsTracked() &&
!texture_cache.GetImage(fault_a_image).IsCpuDirty() &&
!texture_cache.GetImage(fault_b_image).IsCpuDirty() &&
fault_a_mirror.first != nullptr &&
texture_cache.GetImage(fault_a_image).IsTracked() &&
texture_cache.GetImage(fault_b_image).IsTracked() &&
texture_cache.GetImage(fault_a_image).IsGpuModified() &&
texture_cache.GetImage(fault_b_image).IsGpuModified(),
"copying one same-page image lost an authoritative owner");
Require(name, "same-page survivor write fault",
resources.HandleFault(PageFaultAccess::Write, base + 0x8010) &&
texture_cache.GetImage(fault_b_image).IsGpuModified() &&
texture_cache.GetImage(fault_b_image).IsDefinitelyCpuDirty(),
"the surviving image was not protected after its alias retired");
constexpr uint32_t fault_b_cpu = 0xa5a6a7a8u;
std::memcpy(memory + 0x8010, &fault_b_cpu, sizeof(fault_b_cpu));
const auto refreshed_b = texture_cache.FindImage(fault_b_desc);
(void)texture_cache.FindTexture(refreshed_b, fault_b_desc);
Require(
name, "same-page survivor refresh",
refreshed_b == fault_b_image &&
!texture_cache.GetImage(fault_b_image).IsDefinitelyCpuDirty() &&
texture_cache.GetImage(fault_b_image).IsGpuModified(),
"the surviving image could not reconcile its CPU write");
// Further writes on the unprotected page must retain the first baseline.
Require(name, "same-page repeated-write fault",
resources.HandleFault(PageFaultAccess::Write,
base + padding_fault_offset),
"the padding write fault was not handled");
constexpr uint32_t fault_a_cpu = 0x81828384u;
std::memcpy(memory + 0x8000, &fault_a_cpu, sizeof(fault_a_cpu));
texture_cache.InvalidateMemory(base + padding_fault_offset, 1);
Require(name, "repeated same-page invalidation",
texture_cache.GetImage(fault_a_image).IsMaybeCpuDirty() &&
!texture_cache.GetImage(fault_a_image).IsDefinitelyCpuDirty(),
"a repeated padding notification lost maybe-dirty ownership");
texture_cache.InvalidateMemory(base + 0x8010, sizeof(fault_b));
Require(name, "definite write supersedes maybe-dirty state",
texture_cache.GetImage(fault_b_image).IsDefinitelyCpuDirty() &&
!texture_cache.GetImage(fault_b_image).IsMaybeCpuDirty(),
"an overlapping CPU write retained the padding-only dirty state");
texture_cache.UpdateImage(fault_a_image);
texture_cache.UpdateImage(fault_b_image);
auto fault_readback = CreateHostBuffer(
name, 2 * sizeof(uint32_t), vk::BufferUsageFlagBits::eTransferDst, {0, 0});
vk::BufferImageCopy fault_copy{};
fault_copy.imageSubresource = {vk::ImageAspectFlagBits::eColor, 0, 0, 1};
fault_copy.imageExtent = {1, 1, 1};
for (const auto id : {fault_a_image, fault_b_image}) {
texture_cache.GetImage(id).Download(
{&fault_copy, 1}, fault_readback.buffer, fault_copy.bufferOffset,
sizeof(uint32_t));
fault_copy.bufferOffset += sizeof(uint32_t);
}
HostReadBarrier(fault_readback.buffer, fault_readback.size,
vk::PipelineStageFlagBits::eTransfer,
vk::AccessFlagBits::eTransferWrite);
scheduler.Finish();
Require(name, "same-page CPU refresh contents",
ReadBuffer(name, fault_readback, 2) ==
std::vector<u32>{fault_a_cpu, fault_b_cpu},
"repeated padding invalidation lost a CPU write or definite "
"invalidation failed to refresh the native image");
DestroyBuffer(&fault_readback);
constexpr uint64_t publish_image_offset = 0x26000;
constexpr uint64_t publish_buffer_offset = 0x26010;
constexpr uint32_t publish_image_value = 0x31415926u;
constexpr uint32_t publish_buffer_value = 0x27182818u;
std::memcpy(memory + publish_image_offset, &publish_image_value,
sizeof(publish_image_value));
auto publish_image_desc = MakeLinearDesc(
base + publish_image_offset, sizeof(publish_image_value),
vk::Format::eR32Uint, Prospero::BufferFormat::k32UInt,
Prospero::ImageType::kColor2D, {1, 1, 1}, 1, 4, 1);
const auto publish_image = texture_cache.FindImage(publish_image_desc);
(void)texture_cache.FindTexture(publish_image, publish_image_desc);
texture_cache.MarkGpuWritten(publish_image);
resources.GetBufferCache().FillBuffer(base + publish_buffer_offset,
sizeof(publish_buffer_value),
publish_buffer_value, false);
auto publish_replacement_desc = publish_image_desc;
publish_replacement_desc.info.pixel_format = vk::Format::eR32Sfloat;
publish_replacement_desc.info.guest_format =
Prospero::BufferFormat::k32Float;
publish_replacement_desc.view_info.format = vk::Format::eR32Sfloat;
const auto publish_replacement =
texture_cache.FindImage(publish_replacement_desc, true);
Require(name, "exact-disjoint image replacement",
publish_replacement && publish_replacement != publish_image,
"exact-format lookup did not replace the image");
Require(name, "exact-disjoint image publication",
!resources.GetBufferCache().HasGpuDirtyBytes(
base + publish_buffer_offset, sizeof(publish_buffer_value)),
"clean neighboring buffer unexpectedly became GPU-owned");
uint32_t published_image_backing = 0;
uint32_t published_buffer_backing = 0;
std::memcpy(&published_image_backing, memory + publish_image_offset,
sizeof(published_image_backing));
std::memcpy(&published_buffer_backing, memory + publish_buffer_offset,
sizeof(published_buffer_backing));
Require(name, "exact-disjoint publication contents",
published_image_backing == publish_image_value &&
published_buffer_backing == publish_buffer_value,
"image publication or neighboring buffer readback changed bytes");
for (const auto [offset, samples] :
std::array<std::pair<uint64_t, uint32_t>, 2>{
{{0x10000, 2}, {0x11000, 4}}}) {
std::memset(memory + offset, 0, samples * sizeof(uint16_t));
auto standalone_ms = MakeLinearDesc(
base + offset, samples * sizeof(uint16_t), vk::Format::eD16Unorm,
Prospero::BufferFormat::k16UNorm, Prospero::ImageType::kColor2D,
{1, 1, 1}, 1, 2, samples);
standalone_ms.type = BindingType::DepthTarget;
standalone_ms.view_info.aspect = vk::ImageAspectFlagBits::eDepth;
standalone_ms.view_info.usage =
vk::ImageUsageFlagBits::eDepthStencilAttachment;
const auto image = texture_cache.FindImage(standalone_ms);
(void)texture_cache.FindDepthTarget(image, standalone_ms);
texture_cache.MarkGpuWritten(image);
uint16_t standalone_cpu_read = 1;
std::memcpy(&standalone_cpu_read, memory + offset,
sizeof(standalone_cpu_read));
Require(
name, "standalone MSAA ownership",
texture_cache.GetImage(image).IsGpuModified() &&
standalone_cpu_read == 0 &&
texture_cache.GetImage(image).IsGpuModified() &&
resources.HandleFault(PageFaultAccess::Write, base + offset) &&
texture_cache.GetImage(image).IsGpuModified() &&
texture_cache.GetImage(image).IsDefinitelyCpuDirty(),
"fresh multisample target violated write-only ownership");
}
// A one-layer guest descriptor can alias layer 1 of a cached native array.
constexpr uint64_t alias_dcc_address = base + 0x2120000;
auto dcc_array = MakeLinearDesc(
base + 0x2100000, 0x20000, vk::Format::eR16G16B16A16Sfloat,
Prospero::BufferFormat::k16_16_16_16Float, Prospero::ImageType::kColor2D,
{128, 64, 1}, 2, 8, 1);
dcc_array.type = BindingType::RenderTarget;
dcc_array.info.tile_mode = Prospero::TileMode::kRenderTarget;
dcc_array.info.pitch = TileGetRenderTargetPitch(128, 8);
dcc_array.info.UpdateSize();
dcc_array.info.metadata.kind = ImageMetadataKind::Dcc;
dcc_array.info.metadata.range = {alias_dcc_address, 0x2000};
dcc_array.view_info.usage = vk::ImageUsageFlagBits::eColorAttachment;
WriteMetadata(context, alias_dcc_address, 0x2000, UINT32_MAX);
WriteMetadata(context, alias_dcc_address + 0x2000, 0x1000, 0x5a5a5a5au);
const auto dcc_array_id = texture_cache.FindImage(dcc_array);
vk::ClearValue array_paint{};
array_paint.color.float32 = std::array{1.0f, 0.0f, 1.0f, 1.0f};
TextureCacheTestAccess::ClearImage(texture_cache, scheduler.Current(), dcc_array_id,
{vk::ImageAspectFlagBits::eColor, 0, 1, 0, 2}, array_paint);
auto dcc_alias = dcc_array;
dcc_alias.info.data = {dcc_array.info.data.address + 0x10000, 0x10000};
dcc_alias.info.resources.layers = 1;
dcc_alias.info.UpdateSize();
dcc_alias.info.metadata.range = {alias_dcc_address + 0x1000, 0x1000};
dcc_alias.view_info.type = vk::ImageViewType::e2D;
dcc_alias.view_info.layer_count = 1;
WriteMetadata(context, dcc_alias.info.metadata.range.address, 0x1000, 0x40404040u);
Require(name, "DCC alias host layer",
texture_cache.FindImage(dcc_alias) == dcc_array_id &&
dcc_alias.view_info.base_layer == 1 &&
ReadCachedTexel(name, context, dcc_array_id) ==
std::vector<u32>{0x00003c00u, 0x3c003c00u} &&
ReadCachedTexel(name, context, dcc_array_id, {}, {1, 1, 1}, 1) ==
std::vector<u32>{0, 0x3c000000u},
"guest metadata layer 0 did not clear only remapped native layer 1");
resources.GetBufferCache().ReadMemory(alias_dcc_address, 0x3000, false);
std::array<uint8_t, 0x3000> alias_metadata{};
Require(name, "DCC alias metadata bounds",
LibKernel::Memory::TryReadBacking(alias_dcc_address, alias_metadata.data(),
alias_metadata.size()) &&
std::all_of(alias_metadata.begin(), alias_metadata.begin() + 0x2000,
[](uint8_t value) { return value == 0xff; }) &&
std::all_of(alias_metadata.begin() + 0x2000, alias_metadata.end(),
[](uint8_t value) { return value == 0x5a; }),
"remapped host layer changed metadata outside the guest alias allocation");
RenderColorInfo retired_alias{};
retired_alias.desc = dcc_alias;
retired_alias.image_id = dcc_array_id;
retired_alias.guest_mip_level = 0;
retired_alias.guest_array_layer = 0;
texture_cache.UnmapMemory(dcc_array.info.data.address, dcc_array.info.data.size);
WriteMetadata(context, dcc_alias.info.metadata.range.address, 0x1000, 0x40404040u);
RenderExecutorTestAccess::PrepareGraphicsBindings(
context.GetRenderExecutor(), std::span<PreparedBindings *const>{},
std::span{&retired_alias, 1u});
RenderDepthInfo alias_depth{};
const auto alias_rendering = RenderExecutorTestAccess::AcquireRenderTargets(
context.GetRenderExecutor(), scheduler.Current(), &retired_alias, 1, alias_depth);
Require(name, "retired DCC alias native coordinates",
retired_alias.image_id != dcc_array_id &&
retired_alias.desc.view_info.base_level == 0 &&
retired_alias.desc.view_info.base_layer == 0 &&
retired_alias.desc.info.resources.layers == 1 &&
alias_rendering.color_attachments[0].image_view != nullptr &&
ReadCachedTexel(name, context, retired_alias.image_id) ==
std::vector<u32>{0, 0x3c000000u},
"rediscovery treated a retired host layer as the native DCC slice");
resources.GetBufferCache().ReadMemory(alias_dcc_address, 0x3000);
Require(name, "retired DCC alias metadata bounds",
LibKernel::Memory::TryReadBacking(alias_dcc_address, alias_metadata.data(),
alias_metadata.size()) &&
std::all_of(alias_metadata.begin(), alias_metadata.begin() + 0x2000,
[](uint8_t byte) { return byte == 0xff; }) &&
std::all_of(alias_metadata.begin() + 0x2000, alias_metadata.end(),
[](uint8_t byte) { return byte == 0x5a; }),
"retired alias rediscovery overwrote metadata outside its native slice");
RenderExecutorTestAccess::ResetBindings(context.GetRenderExecutor());
constexpr uint64_t metadata_data_a = 0x12000;
constexpr uint64_t metadata_data_b = 0x12100;
constexpr uint64_t metadata_a = 0x13000;
constexpr uint64_t metadata_b = 0x13100;
auto MakeMetadataDepth = [&](uint64_t data, uint64_t metadata) {
auto desc = MakeLinearDesc(
base + data, sizeof(uint32_t), vk::Format::eD32Sfloat,
Prospero::BufferFormat::k32Float, Prospero::ImageType::kColor2D,
{1, 1, 1}, 1, 4, 1);
desc.type = BindingType::DepthTarget;
desc.info.metadata.kind = ImageMetadataKind::Htile;
desc.info.metadata.range = {base + metadata, 0x80};
desc.view_info.aspect = vk::ImageAspectFlagBits::eDepth;
desc.view_info.usage = vk::ImageUsageFlagBits::eDepthStencilAttachment;
return desc;
};
auto metadata_depth_a = MakeMetadataDepth(metadata_data_a, metadata_a);
auto metadata_depth_b = MakeMetadataDepth(metadata_data_b, metadata_b);
metadata_depth_a.info.htile_initial_clear = false;
const auto metadata_depth_a_id =
texture_cache.FindImage(metadata_depth_a);
const auto metadata_depth_b_id =
texture_cache.FindImage(metadata_depth_b);
WriteMetadata(context, base + metadata_a, 0x80, 0);
Require(
name, "unbound metadata separation",
!texture_cache.IsMeta(base + metadata_a),
"unbound metadata bytes were exposed as registered HTile");
const auto metadata_depth_a_view =
texture_cache.FindDepthTarget(metadata_depth_a_id, metadata_depth_a);
const auto metadata_depth_b_view =
texture_cache.FindDepthTarget(metadata_depth_b_id, metadata_depth_b);
Require(name, "shared-page metadata state",
metadata_depth_a_view != nullptr &&
metadata_depth_b_view != nullptr &&
!texture_cache.IsMetaCleared(base + metadata_a, 0) &&
!texture_cache.IsMetaCleared(base + metadata_a, 86) &&
!texture_cache.IsMetaCleared(base + metadata_a, 8191) &&
texture_cache.ClearMeta(base + metadata_a) &&
texture_cache.ClearMeta(base + metadata_b),
"shared-page metadata did not retain logical clear state");
auto metadata_alias = MakeLinearDesc(
base + metadata_a, sizeof(uint32_t), vk::Format::eR32Sfloat,
Prospero::BufferFormat::k32Float, Prospero::ImageType::kColor2D,
{1, 1, 1}, 1, 4, 1);
metadata_alias.type = BindingType::RenderTarget;
metadata_alias.view_info.usage = vk::ImageUsageFlagBits::eColorAttachment;
const auto metadata_alias_id =
texture_cache.FindImage(metadata_alias, true);
const auto metadata_alias_view =
texture_cache.FindRenderTarget(metadata_alias_id, metadata_alias);
Require(
name, "exact alias metadata coexistence",
metadata_alias_view != nullptr &&
texture_cache.GetImage(metadata_depth_a_id).IsGpuModified() &&
texture_cache.GetImage(metadata_depth_b_id).IsGpuModified() &&
texture_cache.IsMeta(base + metadata_a) &&
texture_cache.IsMetaCleared(base + metadata_a, 0) &&
texture_cache.IsMeta(base + metadata_b) &&
texture_cache.IsMetaCleared(base + metadata_b, 0),
"image discovery retired aliased metadata or its live depth image");
const bool touched_metadata = texture_cache.TouchMeta(base + metadata_a, 0, false);
WriteMetadata(context, base + metadata_a, 0x80, 0);
Require(
name, "registered DCC/HTile separation",
touched_metadata && !texture_cache.IsMetaCleared(base + metadata_a, 0),
"a PS5 DCC fill changed registered HTile state");
Require(
name, "metadata first-touch state",
texture_cache.TouchMeta(base + metadata_b, 0, true) &&
texture_cache.IsMetaCleared(base + metadata_b, 0) &&
texture_cache.TouchMeta(base + metadata_b, 0, false) &&
!texture_cache.IsMetaCleared(base + metadata_b, 0),
"per-slice metadata update incorrectly required prior GPU ownership");
constexpr std::array htile_slices{31u, 32u, 63u, 64u, 86u, 8191u};
Require(name, "HTile clear across slice words",
texture_cache.ClearMeta(base + metadata_b),
"registered metadata did not accept a full clear");
for (const auto slice : htile_slices) {
Require(name, "independent high HTile slice",
texture_cache.IsMetaCleared(base + metadata_b, slice) &&
texture_cache.TouchMeta(base + metadata_b, slice, false) &&
!texture_cache.IsMetaCleared(base + metadata_b, slice) &&
texture_cache.IsMetaCleared(base + metadata_b, 0),
"high slice clear state was missing or aliased a lower slice");
}
(void)texture_cache.FindDepthTarget(metadata_depth_b_id, metadata_depth_b);
Require(name, "HTile registration retains consumed slices",
std::ranges::none_of(htile_slices, [&](uint32_t slice) {
return texture_cache.IsMetaCleared(base + metadata_b, slice);
}) && texture_cache.TouchMeta(base + metadata_b, 86, true) &&
texture_cache.IsMetaCleared(base + metadata_b, 86) &&
!texture_cache.IsMetaCleared(base + metadata_b, 64) &&
!texture_cache.IsMetaCleared(base + metadata_b, 8192) &&
!texture_cache.TouchMeta(base + metadata_b, 8192, true) &&
texture_cache.ClearMeta(base + metadata_b) &&
std::ranges::all_of(htile_slices, [&](uint32_t slice) {
return texture_cache.IsMetaCleared(base + metadata_b, slice);
}),
"registration reset consumed slices or full clear failed to rearm them");
resources.GetBufferCache().FillBuffer(base + metadata_b, sizeof(uint32_t),
0, false);
Require(
name, "metadata buffer fill state",
texture_cache.IsMetaCleared(base + metadata_b, 0),
"BufferCache fill did not publish an exact-address metadata clear");
// R-Type Final 3 reuses an old color allocation inside a new stencil plane,
// with the same metadata address changing from DCC to HTile.
auto reused_depth = MakeMetadataDepth(0x12300, 0x28000);
reused_depth.info.pixel_format = vk::Format::eD32SfloatS8Uint;
reused_depth.info.stencil = {base + 0x12200, 0x80};
reused_depth.info.htile_initial_clear = false;
reused_depth.view_info.format = vk::Format::eD32SfloatS8Uint;
reused_depth.view_info.aspect = vk::ImageAspectFlagBits::eDepth |
vk::ImageAspectFlagBits::eStencil;
auto reused_color = MakeLinearDesc(
base + 0x12240, sizeof(uint32_t), vk::Format::eR8G8B8A8Unorm,
Prospero::BufferFormat::k8_8_8_8UNorm, Prospero::ImageType::kColor2D,
{1, 1, 1}, 1, 4, 1);
reused_color.type = BindingType::RenderTarget;
reused_color.info.metadata.kind = ImageMetadataKind::Dcc;
reused_color.info.metadata.range = {base + 0x28000, 0x1000};
reused_color.view_info.usage = vk::ImageUsageFlagBits::eColorAttachment;
const auto reused_color_id = texture_cache.FindImage(reused_color);
(void)texture_cache.FindRenderTarget(reused_color_id, reused_color);
// Repeated CPU stores must remain visible through ordinary buffer ownership.
for (uint32_t repeat = 0; repeat < 2; repeat++) {
vk::ClearValue painted{};
painted.color.float32 = std::array{1.0f, 0.0f, 1.0f, 1.0f};
TextureCacheTestAccess::ClearImage(
texture_cache, scheduler.Current(), reused_color_id,
{vk::ImageAspectFlagBits::eColor, 0, 1, 0, 1}, painted);
std::memset(memory + 0x28000, 0, 0x1000);
Require(name, "CPU metadata rediscovery",
texture_cache.FindImage(reused_color) == reused_color_id,
"metadata writes replaced the live expanded image");
(void)texture_cache.FindRenderTarget(reused_color_id, reused_color);
Require(name, "repeated CPU metadata clear",
ReadCachedTexel(name, context, reused_color_id) == std::vector<u32>{0},
"a same-value CPU metadata write lost its clear after rendering");
}
WriteMetadata(context, base + 0x28000, 0x1000, 0);
const auto reused_depth_id = texture_cache.FindImage(reused_depth);
(void)texture_cache.FindDepthTarget(reused_depth_id, reused_depth);
Require(
name, "DCC allocation reused as HTile",
TextureCacheTestAccess::Contains(texture_cache, reused_color_id) &&
!texture_cache.IsMetaCleared(base + 0x28000, 0) &&
texture_cache.ClearMeta(base + 0x28000) &&
texture_cache.TouchMeta(base + 0x28000, 0, false),
"depth binding retained incompatible DCC clear state");
texture_cache.UnmapMemory(reused_color.info.data.address,
reused_color.info.data.size);
(void)texture_cache.FindDepthTarget(reused_depth_id, reused_depth);
Require(
name, "reused metadata owner retirement",
!TextureCacheTestAccess::Contains(texture_cache, reused_color_id) &&
texture_cache.IsMeta(base + 0x28000) &&
!texture_cache.IsMetaCleared(base + 0x28000, 0) &&
texture_cache.IsMetaCleared(base + 0x28000, 1),
"retiring the old DCC image erased the live HTile slice state");
texture_cache.UnmapMemory(reused_depth.info.data.address,
reused_depth.info.data.size);
Require(name, "reused metadata final retirement",
!texture_cache.IsMeta(base + 0x28000),
"retiring the HTile image left its metadata registered");
constexpr uint64_t partial_unmap_image_offset = 0x2700000;
auto partial_unmap_image =
MakeLinearDesc(base + partial_unmap_image_offset, 0x2000,
vk::Format::eR32Uint, Prospero::BufferFormat::k32UInt,
Prospero::ImageType::kColor2D, {2048, 1, 1}, 1, 4, 1);
const auto partial_unmap_image_id =
texture_cache.FindImage(partial_unmap_image);
const auto image_address = partial_unmap_image.info.data.address;
Require(name, "registered image boundaries",
texture_cache.IsRegionRegistered(image_address, 1) &&
texture_cache.IsRegionRegistered(image_address + 0x1fff, 1) &&
!texture_cache.IsRegionRegistered(image_address - 1, 1) &&
!texture_cache.IsRegionRegistered(image_address + 0x2000, 1) &&
texture_cache.IsRegionRegistered(image_address - 1, 0x2002),
"image lookup mishandled a partial overlap or adjacent range");
scheduler.Finish();
scheduler.DrainPriorityOperations();
const auto image_unmap_tick = scheduler.CurrentTick();
resources.UnmapMemory(image_address + 0x1000, 0x1000);
Require(name, "partial image unmap tracking",
partial_unmap_image_id &&
scheduler.CurrentTick() == image_unmap_tick + 1 &&
!texture_cache.IsRegionRegistered(image_address, 0x2000) &&
!texture_cache.FindImageFromRange(image_address, 0x2000, false),
"partial image unmap skipped its drain or retained the image");
constexpr uint64_t unformatted_alias_offset = 0x2500000;
auto unformatted_alias =
MakeLinearDesc(base + unformatted_alias_offset, sizeof(uint32_t),
vk::Format::eR32Uint, Prospero::BufferFormat::k32UInt,
Prospero::ImageType::kColor2D, {1, 1, 1}, 1, 4, 1);
const auto unformatted_alias_image =
texture_cache.FindImage(unformatted_alias);
(void)texture_cache.FindTexture(unformatted_alias_image,
unformatted_alias);
texture_cache.MarkGpuWritten(unformatted_alias_image);
auto unformatted_alias_buffer = resources.GetBufferCache().ObtainBuffer(
unformatted_alias.info.data.address, unformatted_alias.info.data.size,
false);
Require(
name, "unformatted buffer/image alias",
unformatted_alias_buffer.first != nullptr &&
texture_cache.GetImage(unformatted_alias_image).IsGpuModified() &&
!texture_cache.GetImage(unformatted_alias_image)
.IsBufferModified(),
"read-only unformatted buffer alias did not follow ordinary "
"BufferCache acquisition");
constexpr uint64_t layered_offset = 0x14000;
constexpr uint64_t layered_guest_size = 0x800;
const std::array<float, 10> layered_values{0.0f, 0.125f, 0.25f, 0.375f,
0.5f, 0.625f, 0.75f, 0.875f,
1.0f, 0.0625f};
std::memset(memory + layered_offset, 0, layered_guest_size);
const auto layered_layout = MakeTilingImage(
Prospero::BufferFormat::k32Float, 2, 2, 2, 2,
Prospero::TileMode::kLinear, layered_guest_size, false);
const auto layered_upload_regions = layered_layout.BufferCopies();
Require(name, "layered guest layout", layered_upload_regions.size() == 2,
"linear array upload did not batch each mip");
for (const auto &region : layered_upload_regions) {
for (uint32_t layer = 0; layer < region.imageSubresource.layerCount;
++layer) {
for (uint32_t y = 0; y < region.imageExtent.height; ++y) {
for (uint32_t x = 0; x < region.imageExtent.width; ++x) {
const uint32_t absolute_layer =
region.imageSubresource.baseArrayLayer + layer;
const uint32_t logical = region.imageSubresource.mipLevel == 0
? absolute_layer * 4 + y * 2 + x
: 8 + absolute_layer;
const uint64_t byte_offset =
region.bufferOffset +
((uint64_t{layer} * region.bufferImageHeight + y) *
region.bufferRowLength +
x) *
sizeof(float);
Require(name, "layered guest layout bounds",
byte_offset + sizeof(float) <= layered_guest_size,
"layered guest texel lies outside its backing range");
std::memcpy(memory + layered_offset + byte_offset,
&layered_values[logical], sizeof(float));
}
}
}
}
auto layered_color = MakeLinearDesc(
base + layered_offset, layered_guest_size, vk::Format::eR32Sfloat,
Prospero::BufferFormat::k32Float, Prospero::ImageType::kColor2D,
{2, 2, 1}, 2, 4, 1);
layered_color.info.resources.levels = 2;
layered_color.info.mip_layout[0] = {0, 32, 2, 2};
layered_color.info.mip_layout[1] = {32, 8, 1, 1};
layered_color.view_info.level_count = 2;
layered_color.info.UpdateSize();
const auto layered_color_image = texture_cache.FindImage(layered_color);
(void)texture_cache.FindTexture(layered_color_image, layered_color);
texture_cache.MarkGpuWritten(layered_color_image);
auto layered_depth = layered_color;
layered_depth.type = BindingType::DepthTarget;
layered_depth.info.resources = {1, 4};
layered_depth.info.mip_layout = {};
layered_depth.info.mip_layout[0] = {0, layered_guest_size, 2, 2};
layered_depth.view_info.level_count = 1;
layered_depth.view_info.layer_count = 4;
layered_depth.info.pixel_format = vk::Format::eD32Sfloat;
layered_depth.view_info.format = vk::Format::eD32Sfloat;
layered_depth.view_info.aspect = vk::ImageAspectFlagBits::eDepth;
layered_depth.view_info.usage =
vk::ImageUsageFlagBits::eDepthStencilAttachment;
layered_depth.info.UpdateSize();
const auto layered_depth_image = texture_cache.FindImage(layered_depth);
const auto &layered_native = texture_cache.GetImage(layered_depth_image);
Require(
name, "layered mipped depth alias",
layered_depth_image != layered_color_image &&
layered_native.backing.layers == 4 &&
layered_native.backing.mip_levels == 1 &&
layered_native.info.resources == layered_depth.info.resources &&
layered_native.info.mip_layout == layered_depth.info.mip_layout &&
layered_native.IsGpuModified(),
"depth/color conversion mixed the source mip count with the "
"requested layout");
const volatile auto layered_guest_byte =
*reinterpret_cast<const volatile uint8_t *>(memory + layered_offset);
(void)layered_guest_byte;
Require(name, "layered depth write-only read policy",
texture_cache.GetImage(layered_depth_image).IsGpuModified(),
"a CPU read invalidated layered depth GPU ownership");
constexpr uint64_t replacement_offset = 0x18000;
constexpr uint64_t replacement_size = 0x2000;
constexpr uint32_t replacement_pixel = 0xcafebabeu;
constexpr uint32_t replacement_tail = 0x0badf00du;
std::memset(memory + replacement_offset, 0, replacement_size);
std::memcpy(memory + replacement_offset, &replacement_pixel,
sizeof(replacement_pixel));
std::memcpy(memory + replacement_offset + replacement_size -
sizeof(replacement_tail),
&replacement_tail, sizeof(replacement_tail));
auto replacement_source =
MakeLinearDesc(base + replacement_offset, sizeof(uint32_t),
vk::Format::eR32Uint, Prospero::BufferFormat::k32UInt,
Prospero::ImageType::kColor2D, {1, 1, 1}, 1, 4, 1);
const auto replacement_source_image =
texture_cache.FindImage(replacement_source);
(void)texture_cache.FindTexture(replacement_source_image,
replacement_source);
texture_cache.MarkGpuWritten(replacement_source_image);
auto replacement_desc =
MakeLinearDesc(base + replacement_offset, replacement_size,
vk::Format::eR32Uint, Prospero::BufferFormat::k32UInt,
Prospero::ImageType::kColor2D, {1, 1, 1}, 2, 4, 1);
const auto replacement = texture_cache.FindImage(replacement_desc);
const bool replacement_changed = replacement != replacement_source_image;
const bool replacement_gpu_before =
texture_cache.GetImage(replacement).IsGpuModified();
const bool replacement_fault = resources.HandleFault(
PageFaultAccess::Write, base + replacement_offset + 0x1000);
const bool replacement_gpu_after =
texture_cache.GetImage(replacement).IsGpuModified();
uint32_t replacement_pixel_after = 0;
uint32_t replacement_tail_after = 0;
std::memcpy(&replacement_pixel_after, memory + replacement_offset,
sizeof(replacement_pixel_after));
std::memcpy(&replacement_tail_after,
memory + replacement_offset + replacement_size -
sizeof(replacement_tail_after),
sizeof(replacement_tail_after));
Require(name, "replacement tracking",
replacement_changed && replacement_gpu_before &&
replacement_fault && replacement_gpu_after &&
texture_cache.GetImage(replacement).IsDefinitelyCpuDirty() &&
replacement_pixel_after == replacement_pixel &&
replacement_tail_after == replacement_tail,
"replacement write invalidation lost native authority or guest "
"backing");
const uint32_t compressed_value = 0x55667788u;
std::memcpy(memory + 0xc000, &compressed_value, sizeof(compressed_value));
auto compressed_desc = MakeLinearDesc(
base + 0xc000, sizeof(compressed_value), vk::Format::eR8G8B8A8Srgb,
Prospero::BufferFormat::k8_8_8_8Srgb, Prospero::ImageType::kColor2D,
{1, 1, 1}, 1, 4, 1);
compressed_desc.type = BindingType::RenderTarget;
compressed_desc.info.metadata.kind = ImageMetadataKind::Dcc;
compressed_desc.info.metadata.range = {base + 0xd000, 0};
compressed_desc.info.metadata.compression =
VideoOutCompression::Dcc256_256_0;
compressed_desc.view_info.usage =
vk::ImageUsageFlagBits::eColorAttachment;
const auto compressed_image = texture_cache.FindImage(compressed_desc);
(void)texture_cache.FindRenderTarget(compressed_image, compressed_desc);
uint32_t compressed_cpu_read = 0;
std::memcpy(&compressed_cpu_read, memory + 0xc000,
sizeof(compressed_cpu_read));
Require(name, "compressed write-only read policy",
compressed_cpu_read == compressed_value &&
texture_cache.GetImage(compressed_image).IsGpuModified(),
"compressed image incorrectly claimed a CPU read fault");
Require(name, "compressed download rejection",
!TextureCacheTestAccess::TryDownload(texture_cache,
compressed_image) &&
texture_cache.GetImage(compressed_image).IsGpuModified() &&
!texture_cache.GetImage(compressed_image).IsBufferModified(),
"a compressed image escaped the unified download guard");
auto compressed_video_desc = compressed_desc;
compressed_video_desc.type = BindingType::VideoOut;
compressed_video_desc.view_info.usage = vk::ImageUsageFlagBits::eSampled;
const auto compressed_video =
texture_cache.FindImage(compressed_video_desc);
const auto &compressed_video_image =
texture_cache.GetImage(compressed_video);
Require(
name, "cache-only video-out discovery",
compressed_video == compressed_image &&
!compressed_video_image.binding.is_bound &&
!compressed_video_image.binding.is_target &&
!compressed_video_image.binding.needs_rebind &&
!compressed_video_image.usage.video_out,
"FindImage claimed caller-owned video-out usage or binding state");
constexpr uint64_t mixed_source_offset = 0x20000;
constexpr uint64_t mixed_source_size = 0x1004;
constexpr uint32_t mixed_source_width = 1025;
constexpr uint32_t mixed_cpu_value = 0x1234abcdu;
constexpr uint32_t mixed_gpu_value = 0x9876fedcu;
auto mixed_owner = resources.GetBufferCache().ObtainBuffer(
base + mixed_source_offset, mixed_source_size, true);
Require(name, "mixed-page source allocation",
mixed_owner.first != nullptr,
"mixed CPU/GPU image source did not create a containing buffer");
Require(name, "mixed-page CPU write fault",
resources.HandleFault(PageFaultAccess::Write,
base + mixed_source_offset),
"mixed image source could not dirty its first page");
std::memcpy(memory + mixed_source_offset, &mixed_cpu_value,
sizeof(mixed_cpu_value));
resources.GetBufferCache().FillBuffer(base + mixed_source_offset + 0x1000,
sizeof(mixed_gpu_value),
mixed_gpu_value, false);
auto mixed_source_desc = MakeLinearDesc(
base + mixed_source_offset, mixed_source_size, vk::Format::eR32Uint,
Prospero::BufferFormat::k32UInt, Prospero::ImageType::kColor2D,
{mixed_source_width, 1, 1}, 1, 4, 1);
const auto mixed_source_image =
texture_cache.FindImage(mixed_source_desc);
(void)texture_cache.FindTexture(mixed_source_image, mixed_source_desc);
Require(name, "mixed-page image upload",
!texture_cache.GetImage(mixed_source_image).IsGpuModified(),
"clean Buffer source manufactured GPU image ownership");
auto mixed_source_readback = CreateHostBuffer(
name, mixed_source_size, vk::BufferUsageFlagBits::eTransferDst,
std::vector<u32>(mixed_source_size / sizeof(u32), 0));
auto &mixed_source_native = texture_cache.GetImage(mixed_source_image);
mixed_source_native.Transit(vk::ImageLayout::eTransferSrcOptimal,
vk::AccessFlagBits2::eTransferRead, {},
command.Handle());
vk::BufferImageCopy mixed_source_copy{};
mixed_source_copy.bufferRowLength = mixed_source_width;
mixed_source_copy.imageSubresource.aspectMask =
vk::ImageAspectFlagBits::eColor;
mixed_source_copy.imageSubresource.layerCount = 1;
mixed_source_copy.imageExtent = {mixed_source_width, 1, 1};
command.Handle().copyImageToBuffer(mixed_source_native.backing.image,
vk::ImageLayout::eTransferSrcOptimal,
mixed_source_readback.buffer, 1,
&mixed_source_copy);
HostReadBarrier(mixed_source_readback.buffer, mixed_source_readback.size,
vk::PipelineStageFlagBits::eTransfer,
vk::AccessFlagBits::eTransferWrite);
constexpr uint64_t bridged_gpu_offset = 0x101000;
constexpr uint64_t bridged_cpu_offset = 0x105000;
constexpr uint64_t bridged_texel_size = 16;
constexpr uint64_t bridged_source_size =
bridged_cpu_offset - bridged_gpu_offset + bridged_texel_size;
constexpr uint32_t bridged_source_width =
bridged_source_size / bridged_texel_size;
constexpr uint32_t bridged_gpu_value = 0x10293847u;
constexpr uint32_t bridged_cpu_value = 0x56473829u;
constexpr uint32_t bridged_gpu_guest_value = 0xa5a5a5a5u;
std::array<uint32_t, bridged_texel_size / sizeof(uint32_t)>
bridged_cpu_values{};
bridged_cpu_values.fill(bridged_cpu_value);
std::memcpy(memory + bridged_cpu_offset, bridged_cpu_values.data(),
sizeof(bridged_cpu_values));
auto bridged_cpu_owner = resources.GetBufferCache().ObtainBuffer(
base + bridged_cpu_offset, bridged_texel_size, true);
Require(name, "bridged CPU owner allocation",
bridged_cpu_owner.first != nullptr,
"bridged image source did not create its CPU-side cache owner");
resources.GetBufferCache().FillBuffer(base + bridged_cpu_offset,
bridged_texel_size, 0, false);
Require(
name, "bridged CPU write fault",
resources.HandleFault(PageFaultAccess::Write,
base + bridged_cpu_offset),
"bridged image source could not transfer its second owner to CPU");
std::memcpy(memory + bridged_cpu_offset, bridged_cpu_values.data(),
sizeof(bridged_cpu_values));
std::array<uint32_t, bridged_texel_size / sizeof(uint32_t)>
bridged_gpu_guest_values{};
bridged_gpu_guest_values.fill(bridged_gpu_guest_value);
std::memcpy(memory + bridged_gpu_offset, bridged_gpu_guest_values.data(),
sizeof(bridged_gpu_guest_values));
auto bridged_gpu_owner = resources.GetBufferCache().ObtainBuffer(
base + bridged_gpu_offset, bridged_texel_size, true);
Require(name, "bridged GPU owner allocation",
bridged_gpu_owner.first != nullptr &&
bridged_gpu_owner.first != bridged_cpu_owner.first,
"bridged image source did not retain two disjoint cache owners");
resources.GetBufferCache().FillBuffer(base + bridged_gpu_offset,
bridged_texel_size,
bridged_gpu_value, false);
Require(name, "bridged ownership split",
resources.GetBufferCache().HasGpuDirtyBytes(
base + bridged_gpu_offset, bridged_texel_size) &&
!resources.GetBufferCache().HasGpuDirtyBytes(
base + bridged_cpu_offset, bridged_texel_size) &&
resources.GetBufferCache().IsRegionCpuModified(
base + bridged_cpu_offset, bridged_texel_size),
"bridged image source did not retain distinct GPU and CPU "
"ownership domains");
auto bridged_source_desc = MakeLinearDesc(
base + bridged_gpu_offset, bridged_source_size,
vk::Format::eR32G32B32A32Uint,
Prospero::BufferFormat::k32_32_32_32UInt,
Prospero::ImageType::kColor2D, {bridged_source_width, 1, 1}, 1,
bridged_texel_size, 1);
const auto bridged_source_image =
texture_cache.FindImage(bridged_source_desc);
(void)texture_cache.FindTexture(bridged_source_image,
bridged_source_desc);
auto bridged_source_readback = CreateHostBuffer(
name, bridged_source_size, vk::BufferUsageFlagBits::eTransferDst,
std::vector<u32>(bridged_source_size / sizeof(u32), 0));
auto &bridged_source_native =
texture_cache.GetImage(bridged_source_image);
bridged_source_native.Transit(vk::ImageLayout::eTransferSrcOptimal,
vk::AccessFlagBits2::eTransferRead, {},
scheduler.Current().Handle());
vk::BufferImageCopy bridged_source_copy{};
bridged_source_copy.bufferRowLength = bridged_source_width;
bridged_source_copy.imageSubresource.aspectMask =
vk::ImageAspectFlagBits::eColor;
bridged_source_copy.imageSubresource.layerCount = 1;
bridged_source_copy.imageExtent = {bridged_source_width, 1, 1};
scheduler.Current().Handle().copyImageToBuffer(
bridged_source_native.backing.image,
vk::ImageLayout::eTransferSrcOptimal, bridged_source_readback.buffer,
1, &bridged_source_copy);
HostReadBarrier(bridged_source_readback.buffer,
bridged_source_readback.size,
vk::PipelineStageFlagBits::eTransfer,
vk::AccessFlagBits::eTransferWrite);
constexpr uint64_t byte_mirror_page_offset = 0x24000;
constexpr uint64_t byte_mirror_offset = byte_mirror_page_offset + 1;
const std::array<uint8_t, 4> byte_mirror_guest{0xacu, 0x5au, 0xbdu,
0xceu};
std::memcpy(memory + byte_mirror_page_offset, byte_mirror_guest.data(),
byte_mirror_guest.size());
auto byte_mirror_desc =
MakeLinearDesc(base + byte_mirror_offset, 1, vk::Format::eR8Unorm,
Prospero::BufferFormat::k8UNorm,
Prospero::ImageType::kColor2D, {1, 1, 1}, 1, 1, 1);
const auto byte_mirror_image = texture_cache.FindImage(byte_mirror_desc);
(void)texture_cache.FindTexture(byte_mirror_image, byte_mirror_desc);
texture_cache.MarkGpuWritten(byte_mirror_image);
auto byte_mirror = resources.GetBufferCache().ObtainBuffer(
base + byte_mirror_offset, 1, false, true);
Require(
name, "byte image mirror",
byte_mirror.first != nullptr &&
!texture_cache.GetImage(byte_mirror_image).IsBufferModified() &&
texture_cache.GetImage(byte_mirror_image).IsGpuModified(),
"one-byte image copy transferred cache ownership");
const std::array<uint16_t, 4> bgra16_guest{0x3c00u, 0x4000u, 0x4200u,
0x4400u};
std::memcpy(memory + 0xb000, bgra16_guest.data(), sizeof(bgra16_guest));
auto bgra16_desc = MakeLinearDesc(
base + 0xb000, sizeof(bgra16_guest), vk::Format::eR16G16B16A16Sfloat,
Prospero::BufferFormat::k16_16_16_16Float,
Prospero::ImageType::kColor2D, {1, 1, 1}, 1, 8, 1);
bgra16_desc.type = BindingType::RenderTarget;
bgra16_desc.info.bgra16 = true;
bgra16_desc.view_info.usage = vk::ImageUsageFlagBits::eColorAttachment;
const auto bgra16_image = texture_cache.FindImage(bgra16_desc);
(void)texture_cache.FindRenderTarget(bgra16_image, bgra16_desc);
auto bgra16_readback =
CreateHostBuffer(name, sizeof(bgra16_guest),
vk::BufferUsageFlagBits::eTransferDst, {0, 0});
auto &bgra16_native = texture_cache.GetImage(bgra16_image);
bgra16_native.Transit(vk::ImageLayout::eTransferSrcOptimal,
vk::AccessFlagBits2::eTransferRead, {},
command.Handle());
vk::BufferImageCopy bgra16_copy{};
bgra16_copy.imageSubresource.aspectMask = vk::ImageAspectFlagBits::eColor;
bgra16_copy.imageSubresource.layerCount = 1;
bgra16_copy.imageExtent = {1, 1, 1};
command.Handle().copyImageToBuffer(
bgra16_native.backing.image, vk::ImageLayout::eTransferSrcOptimal,
bgra16_readback.buffer, 1, &bgra16_copy);
HostReadBarrier(bgra16_readback.buffer, bgra16_readback.size,
vk::PipelineStageFlagBits::eTransfer,
vk::AccessFlagBits::eTransferWrite);
auto layered_readback = CreateHostBuffer(
name, 8 * sizeof(float), vk::BufferUsageFlagBits::eTransferDst,
std::vector<u32>(8, 0));
auto &layered_depth_native = texture_cache.GetImage(layered_depth_image);
layered_depth_native.Transit(vk::ImageLayout::eTransferSrcOptimal,
vk::AccessFlagBits2::eTransferRead, {},
command.Handle());
vk::BufferImageCopy layered_copy{};
layered_copy.imageSubresource = {vk::ImageAspectFlagBits::eDepth, 0, 0, 2};
layered_copy.imageExtent = {2, 2, 1};
command.Handle().copyImageToBuffer(
layered_depth_native.backing.image,
vk::ImageLayout::eTransferSrcOptimal, layered_readback.buffer,
1, &layered_copy);
HostReadBarrier(layered_readback.buffer, layered_readback.size,
vk::PipelineStageFlagBits::eTransfer,
vk::AccessFlagBits::eTransferWrite);
vk::ShaderModuleCreateInfo module_info{};
module_info.sType = vk::StructureType::eShaderModuleCreateInfo;
module_info.codeSize = sizeof(GPU_TEST_MS_DEPTH_SPV);
module_info.pCode = GPU_TEST_MS_DEPTH_SPV;
vk::ShaderModule ms_depth_module = nullptr;
RequireVk(
name, "MS depth content",
m_device.createShaderModule(&module_info, nullptr, &ms_depth_module),
"vkCreateShaderModule");
std::array<vk::DescriptorSetLayoutBinding, 2> observer_bindings{};
observer_bindings[0].binding = 0;
observer_bindings[0].descriptorType = vk::DescriptorType::eSampledImage;
observer_bindings[0].descriptorCount = 1;
observer_bindings[0].stageFlags = vk::ShaderStageFlagBits::eCompute;
observer_bindings[1].binding = 1;
observer_bindings[1].descriptorType = vk::DescriptorType::eStorageBuffer;
observer_bindings[1].descriptorCount = 1;
observer_bindings[1].stageFlags = vk::ShaderStageFlagBits::eCompute;
vk::DescriptorSetLayoutCreateInfo observer_layout_info{};
observer_layout_info.sType =
vk::StructureType::eDescriptorSetLayoutCreateInfo;
observer_layout_info.bindingCount =
static_cast<uint32_t>(observer_bindings.size());
observer_layout_info.pBindings = observer_bindings.data();
vk::DescriptorSetLayout observer_set_layout = nullptr;
RequireVk(name, "MS depth content",
m_device.createDescriptorSetLayout(
&observer_layout_info, nullptr, &observer_set_layout),
"vkCreateDescriptorSetLayout");
const vk::PushConstantRange observer_push{
vk::ShaderStageFlagBits::eCompute, 0, sizeof(uint32_t)};
vk::PipelineLayoutCreateInfo observer_pipeline_layout_info{};
observer_pipeline_layout_info.sType =
vk::StructureType::ePipelineLayoutCreateInfo;
observer_pipeline_layout_info.setLayoutCount = 1;
observer_pipeline_layout_info.pSetLayouts = &observer_set_layout;
observer_pipeline_layout_info.pushConstantRangeCount = 1;
observer_pipeline_layout_info.pPushConstantRanges = &observer_push;
vk::PipelineLayout observer_pipeline_layout = nullptr;
RequireVk(name, "MS depth content",
m_device.createPipelineLayout(&observer_pipeline_layout_info,
nullptr,
&observer_pipeline_layout),
"vkCreatePipelineLayout");
vk::PipelineShaderStageCreateInfo observer_stage{};
observer_stage.sType = vk::StructureType::ePipelineShaderStageCreateInfo;
observer_stage.stage = vk::ShaderStageFlagBits::eCompute;
observer_stage.module = ms_depth_module;
observer_stage.pName = "main";
vk::ComputePipelineCreateInfo observer_pipeline_info{};
observer_pipeline_info.sType =
vk::StructureType::eComputePipelineCreateInfo;
observer_pipeline_info.stage = observer_stage;
observer_pipeline_info.layout = observer_pipeline_layout;
vk::Pipeline observer_depth_pipeline = nullptr;
RequireVk(
name, "MS depth content",
m_device.createComputePipelines(nullptr, 1, &observer_pipeline_info,
nullptr, &observer_depth_pipeline),
"vkCreateComputePipelines");
const std::array<vk::DescriptorPoolSize, 2> observer_pool_sizes{{
{vk::DescriptorType::eSampledImage, 2},
{vk::DescriptorType::eStorageBuffer, 2},
}};
vk::DescriptorPoolCreateInfo observer_pool_info{};
observer_pool_info.sType = vk::StructureType::eDescriptorPoolCreateInfo;
observer_pool_info.maxSets = 2;
observer_pool_info.poolSizeCount =
static_cast<uint32_t>(observer_pool_sizes.size());
observer_pool_info.pPoolSizes = observer_pool_sizes.data();
vk::DescriptorPool observer_pool = nullptr;
RequireVk(name, "MS depth content",
m_device.createDescriptorPool(&observer_pool_info, nullptr,
&observer_pool),
"vkCreateDescriptorPool");
const std::array<vk::DescriptorSetLayout, 2> observer_layouts{
observer_set_layout, observer_set_layout};
std::array<vk::DescriptorSet, 2> observer_sets{};
vk::DescriptorSetAllocateInfo observer_allocate{};
observer_allocate.sType = vk::StructureType::eDescriptorSetAllocateInfo;
observer_allocate.descriptorPool = observer_pool;
observer_allocate.descriptorSetCount =
static_cast<uint32_t>(observer_sets.size());
observer_allocate.pSetLayouts = observer_layouts.data();
RequireVk(name, "MS depth content",
m_device.allocateDescriptorSets(&observer_allocate,
observer_sets.data()),
"vkAllocateDescriptorSets");
std::array<Buffer, 2> ms_observer_outputs{
CreateHostBuffer(name, 4 * sizeof(u32),
vk::BufferUsageFlagBits::eStorageBuffer,
std::vector<u32>(4, 0)),
CreateHostBuffer(name, 4 * sizeof(u32),
vk::BufferUsageFlagBits::eStorageBuffer,
std::vector<u32>(4, 0))};
const std::array<ImageId, 2> observed_images{ms_depth_image,
ms_depth_image_4x};
const std::array<uint32_t, 2> observed_samples{2, 4};
for (uint32_t index = 0; index < observed_images.size(); index++) {
auto &observed = texture_cache.GetImage(observed_images[index]);
observed.Transit(vk::ImageLayout::eShaderReadOnlyOptimal,
vk::AccessFlagBits2::eShaderRead, {},
command.Handle());
ImageViewInfo view_info{};
view_info.format = observed.backing.format;
view_info.type = vk::ImageViewType::e2D;
view_info.aspect = vk::ImageAspectFlagBits::eDepth;
view_info.mapping = {vk::ComponentSwizzle::eR, vk::ComponentSwizzle::eR,
vk::ComponentSwizzle::eR,
vk::ComponentSwizzle::eR};
view_info.usage = vk::ImageUsageFlagBits::eSampled;
const auto view = observed.FindView(view_info);
const vk::DescriptorImageInfo image_info{
nullptr, view, vk::ImageLayout::eShaderReadOnlyOptimal};
const vk::DescriptorBufferInfo buffer_info{
ms_observer_outputs[index].buffer, 0,
ms_observer_outputs[index].size};
std::array<vk::WriteDescriptorSet, 2> writes{};
writes[0].sType = vk::StructureType::eWriteDescriptorSet;
writes[0].dstSet = observer_sets[index];
writes[0].dstBinding = 0;
writes[0].descriptorCount = 1;
writes[0].descriptorType = vk::DescriptorType::eSampledImage;
writes[0].pImageInfo = &image_info;
writes[1].sType = vk::StructureType::eWriteDescriptorSet;
writes[1].dstSet = observer_sets[index];
writes[1].dstBinding = 1;
writes[1].descriptorCount = 1;
writes[1].descriptorType = vk::DescriptorType::eStorageBuffer;
writes[1].pBufferInfo = &buffer_info;
m_device.updateDescriptorSets(static_cast<uint32_t>(writes.size()),
writes.data(), 0, nullptr);
command.Handle().bindPipeline(vk::PipelineBindPoint::eCompute,
observer_depth_pipeline);
command.Handle().bindDescriptorSets(vk::PipelineBindPoint::eCompute,
observer_pipeline_layout, 0, 1,
&observer_sets[index], 0, nullptr);
command.Handle().pushConstants(
observer_pipeline_layout, vk::ShaderStageFlagBits::eCompute, 0,
sizeof(observed_samples[index]), &observed_samples[index]);
command.Handle().dispatch(1, 1, 1);
HostReadBarrier(ms_observer_outputs[index].buffer,
ms_observer_outputs[index].size,
vk::PipelineStageFlagBits::eComputeShader,
vk::AccessFlagBits::eShaderWrite);
}
scheduler.Finish();
const auto mip_prefix_words = ReadBuffer(
name, mip_prefix_readback, mip_prefix_size / sizeof(uint32_t));
const auto volume_words =
ReadBuffer(name, volume_readback, volume_guest.size());
Require(name, "formatted volume content",
volume_words[0] == volume_values[0] &&
volume_words[volume_slice_words] == volume_values[1],
"volume synchronization lost a padded depth slice");
DestroyBuffer(&volume_readback);
const auto mip_formatted_words = ReadBuffer(
name, mip_formatted_readback, mip_guest_size / sizeof(uint32_t));
const auto mip0_word = mip_sizes[0].offset / sizeof(uint32_t);
Require(name, "formatted mip prefix content",
mip0_word < mip_prefix_words.size() &&
mip0_word < mip_native.size() &&
mip_prefix_words[mip0_word] == mip_native[mip0_word],
"fitting mip-prefix synchronization copied "
"stale guest backing");
Require(name, "formatted Buffer content",
mip0_word < mip_formatted_words.size() &&
mip0_word < mip_native.size() &&
mip_formatted_words[mip0_word] == mip_native[mip0_word],
"formatted Buffer read bypassed authoritative "
"native image mip data");
Require(name, "CPU-dirty formatted mirror content",
ReadBuffer(name, mirror_cpu_readback, 1) ==
std::vector<u32>{mirror_cpu_value},
"formatted Buffer mirror published stale "
"native image bytes");
Require(name, "Buffer-superseded exact content",
ReadBuffer(name, exact_buffer_readback, 1) ==
std::vector<u32>{exact_buffer_value},
"exact-format recreation initialized from "
"stale guest bytes");
Require(name, "partial-page CPU refresh content",
ReadBuffer(name, partial_cpu_refresh_readback,
partial_cpu_refresh_expected.size()) ==
partial_cpu_refresh_expected,
"cached buffer uploaded bytes outside the exact "
"staged guest range");
const auto mixed_source_words = ReadBuffer(
name, mixed_source_readback, mixed_source_size / sizeof(u32));
Require(name, "mixed-page image content",
mixed_source_words.front() == mixed_cpu_value &&
mixed_source_words[0x1000 / sizeof(u32)] == mixed_gpu_value,
"mixed CPU/GPU source upload lost one "
"ownership domain");
const auto bridged_source_words = ReadBuffer(
name, bridged_source_readback, bridged_source_size / sizeof(u32));
Require(name, "bridged-owner image content",
bridged_source_words.front() == bridged_gpu_value &&
bridged_source_words.back() == bridged_cpu_value,
"image upload bridging separate cache owners lost a GPU or CPU "
"ownership domain");
Require(name, "BGRA16 content",
ReadBuffer(name, bgra16_readback, 2) ==
std::vector<u32>{0x40004200u, 0x44003c00u},
"GPU BGRA16 swap did not publish RGBA half-word order");
scheduler.Finish();
const auto layered_words = ReadBuffer(name, layered_readback, 8);
const std::array<float, 8> layered_expected{
layered_values[0], layered_values[1], layered_values[2],
layered_values[3], layered_values[4], layered_values[5],
layered_values[6], layered_values[7]};
bool layered_content = layered_words.size() == layered_expected.size();
for (uint32_t index = 0;
layered_content && index < layered_expected.size(); index++) {
layered_content &=
layered_words[index] == std::bit_cast<u32>(layered_expected[index]);
}
Require(name, "layered depth common-mip content", layered_content,
"depth/color conversion changed a shared array-layer value");
const std::array<std::array<float, 4>, 2> expected_ms_depth{{
{0x2000 / 65535.0f, 0xe000 / 65535.0f, 0.0f, 0.0f},
{0.0f, 0x4000 / 65535.0f, 0x8000 / 65535.0f, 1.0f},
}};
for (uint32_t image = 0; image < expected_ms_depth.size(); image++) {
const auto words = ReadBuffer(name, ms_observer_outputs[image], 4);
bool content = words.size() == 4;
for (uint32_t sample = 0; content && sample < observed_samples[image];
sample++) {
const float actual = std::bit_cast<float>(words[sample]);
content &= std::abs(actual - expected_ms_depth[image][sample]) <=
1.5f / 65535.0f;
}
Require(name,
image == 0 ? "two-sample depth content"
: "four-sample depth content",
content,
"color-to-multisample-depth changed a packed sample value");
}
auto unrelated_stencil_alias = MakeLinearDesc(
base + second_stencil_offset, ms_stencil_size, vk::Format::eR32Uint,
Prospero::BufferFormat::k32UInt, Prospero::ImageType::kColor2D,
{static_cast<uint32_t>(ms_stencil_size / sizeof(uint32_t)), 1, 1}, 1,
4, 1);
const auto unrelated_stencil_image =
texture_cache.FindImage(unrelated_stencil_alias);
Require(name, "stencil association filtering",
unrelated_stencil_image &&
unrelated_stencil_image != ms_depth_image &&
!texture_cache.GetImage(unrelated_stencil_image).depth_id,
"an unrelated color alias was hijacked by the stencil "
"association record");
texture_cache.UnmapMemory(base + second_stencil_offset, ms_stencil_size);
Require(name, "stencil proxy unmap",
!texture_cache.FindImageFromRange(base + second_stencil_offset,
ms_stencil_size, false) &&
texture_cache.GetImage(ms_depth_image).backing.image ==
ms_depth_backing,
"stencil unmap removed the depth backing or retained its proxy");
const auto reassociated_depth =
texture_cache.FindImage(switched_ms_depth);
TextureCacheTestAccess::AssociateStencil(
texture_cache, reassociated_depth, switched_ms_depth.info.stencil);
const auto reassociated_proxy = texture_cache.FindImageFromRange(
base + second_stencil_offset, ms_stencil_size, false);
Require(name, "stencil proxy reassociation",
reassociated_depth == ms_depth_image && reassociated_proxy &&
texture_cache.GetImage(reassociated_proxy).depth_id ==
ms_depth_image,
"stencil proxy did not re-associate with the surviving depth");
texture_cache.UnmapMemory(ms_depth_desc.info.data.address,
ms_depth_desc.info.data.size);
Require(name, "depth unmap proxy retirement",
!texture_cache.FindImageFromRange(base + ms_stencil_offset,
ms_stencil_size, false) &&
!texture_cache.FindImageFromRange(base + second_stencil_offset,
ms_stencil_size, false),
"unmapping depth retained one of its stencil proxies");
auto reused_stencil_color = MakeLinearDesc(
base + 0x2200000, 0x40000, vk::Format::eR8Unorm,
Prospero::BufferFormat::k8UNorm, Prospero::ImageType::kColor2D,
{480, 408, 1}, 1, 1, 1);
const auto reused_stencil_color_id =
texture_cache.FindImage(reused_stencil_color);
auto reused_stencil_depth = MakeLinearDesc(
base + 0x2240000, 0x10000, vk::Format::eD32SfloatS8Uint,
Prospero::BufferFormat::k32Float, Prospero::ImageType::kColor2D,
{128, 128, 1}, 1, 4, 1);
reused_stencil_depth.type = BindingType::DepthTarget;
reused_stencil_depth.info.stencil = {reused_stencil_color.info.data.address,
0x10000};
reused_stencil_depth.view_info.aspect =
vk::ImageAspectFlagBits::eDepth | vk::ImageAspectFlagBits::eStencil;
reused_stencil_depth.view_info.usage =
vk::ImageUsageFlagBits::eDepthStencilAttachment;
const auto reused_stencil_depth_id =
texture_cache.FindImage(reused_stencil_depth);
TextureCacheTestAccess::AssociateStencil(
texture_cache, reused_stencil_depth_id,
reused_stencil_depth.info.stencil);
const auto reused_stencil_proxy = texture_cache.FindImageFromRange(
reused_stencil_depth.info.stencil.address,
reused_stencil_depth.info.stencil.size, false);
Require(name, "stencil address reuse preserves color",
reused_stencil_proxy &&
reused_stencil_proxy != reused_stencil_color_id &&
!texture_cache.GetImage(reused_stencil_color_id).depth_id &&
texture_cache.GetImage(reused_stencil_proxy).depth_id ==
reused_stencil_depth_id &&
texture_cache.GetImage(reused_stencil_proxy).info.data ==
reused_stencil_depth.info.stencil &&
texture_cache.GetImage(reused_stencil_proxy).info.extent ==
reused_stencil_depth.info.extent,
"a smaller stencil plane took ownership of an existing R8 image");
vk::ClearValue reused_stencil_color_clear{};
reused_stencil_color_clear.color.float32 =
std::array{1.0f, 0.0f, 0.0f, 0.0f};
TextureCacheTestAccess::ClearImage(
texture_cache, scheduler.Current(), reused_stencil_color_id,
{vk::ImageAspectFlagBits::eColor, 0, 1, 0, 1},
reused_stencil_color_clear);
Require(name, "reused stencil plane clear",
texture_cache.ClearImageFromBuffer(
scheduler.Current(), reused_stencil_depth.info.stencil.address,
reused_stencil_depth.info.stencil.size, 0x5a5a5a5au) &&
texture_cache.GetImage(reused_stencil_depth_id).IsGpuModified(),
"the separate stencil association did not resolve to depth");
Require(name, "color clear after stencil address reuse",
ReadCachedTexel(name, context, reused_stencil_color_id, {},
{4, 1, 1}) == std::vector<u32>{0xffffffffu},
"stencil address reuse changed the native R8 clear contents");
auto successor_depth = reused_stencil_depth;
successor_depth.info.data.address += successor_depth.info.data.size;
const auto successor_depth_id = texture_cache.FindImage(successor_depth);
TextureCacheTestAccess::AssociateStencil(
texture_cache, successor_depth_id, successor_depth.info.stencil);
texture_cache.UnmapMemory(reused_stencil_depth.info.data.address,
reused_stencil_depth.info.data.size);
Require(name, "reparented stencil survives old depth retirement",
!TextureCacheTestAccess::Contains(texture_cache,
reused_stencil_depth_id) &&
TextureCacheTestAccess::Contains(texture_cache,
successor_depth_id) &&
texture_cache.FindImageFromRange(
successor_depth.info.stencil.address,
successor_depth.info.stencil.size, false) ==
reused_stencil_proxy &&
texture_cache.GetImage(reused_stencil_proxy).depth_id ==
successor_depth_id,
"retiring the previous depth owner removed its reparented proxy");
texture_cache.UnmapMemory(successor_depth.info.data.address,
successor_depth.info.data.size);
Require(name, "reparented stencil retires with new depth",
!TextureCacheTestAccess::Contains(texture_cache,
reused_stencil_proxy),
"retiring the new depth owner retained its reparented proxy");
texture_cache.UnmapMemory(reused_stencil_color.info.data.address,
reused_stencil_color.info.data.size);
constexpr uint64_t exact_image_offset = 0x334100;
constexpr uint64_t dirty_sibling_offset = 0x334200;
constexpr uint32_t dirty_sibling_value = 0xc001d00du;
auto dirty_sibling = resources.GetBufferCache().ObtainBuffer(
base + dirty_sibling_offset, sizeof(dirty_sibling_value), true);
Require(name, "same-page dirty sibling allocation",
dirty_sibling.first != nullptr,
"failed to create the disjoint same-page Buffer owner");
resources.GetBufferCache().FillBuffer(base + dirty_sibling_offset,
sizeof(dirty_sibling_value),
dirty_sibling_value, false);
auto exact_image_desc =
MakeLinearDesc(base + exact_image_offset, sizeof(uint32_t),
vk::Format::eR32Uint, Prospero::BufferFormat::k32UInt,
Prospero::ImageType::kColor2D, {1, 1, 1}, 1, 4, 1);
const auto exact_image = texture_cache.FindImage(exact_image_desc);
(void)texture_cache.FindTexture(exact_image, exact_image_desc);
texture_cache.MarkGpuWritten(exact_image);
Require(name, "same-page exact alias ownership",
resources.GetBufferCache().HasGpuDirtyBytes(
base + dirty_sibling_offset, sizeof(dirty_sibling_value)) &&
!resources.GetBufferCache().HasGpuDirtyBytes(
base + exact_image_offset, sizeof(uint32_t)) &&
texture_cache.GetImage(exact_image).IsGpuModified(),
"image ownership discarded or conflicted with disjoint dirty "
"Buffer bytes on the same tracker page");
auto exact_buffer = resources.GetBufferCache().ObtainBuffer(
base + exact_image_offset, sizeof(uint32_t), true);
Require(name, "overlapping buffer ownership allocation",
exact_buffer.first != nullptr,
"failed to create the overlapping dirty Buffer owner");
exact_buffer.first->Fill(exact_buffer.second, sizeof(uint32_t),
dirty_sibling_value);
texture_cache.InvalidateMemoryFromGPU(base + exact_image_offset,
sizeof(uint32_t));
Libs::Graphics::Buffer exact_download(
m_runtime_context, scheduler, MemoryUsage::DeviceLocal,
base + exact_image_offset, AllFlags, sizeof(uint32_t));
Require(name, "overlapping buffer ownership rejection",
!texture_cache.FindImageFromRange(base + exact_image_offset,
sizeof(uint32_t)) &&
!TextureCacheTestAccess::TryDownload(texture_cache,
exact_image) &&
!BufferCacheTestAccess::SynchronizeBufferFromImage(
resources.GetBufferCache(), exact_download,
base + exact_image_offset, sizeof(uint32_t)),
"dirty Buffer bytes did not block the stale image source");
resources.GetBufferCache().ReadMemory(base + exact_image_offset,
sizeof(uint32_t));
resources.GetBufferCache().ReadMemory(base + dirty_sibling_offset,
sizeof(dirty_sibling_value));
constexpr std::array<uint64_t, 2> gc_image_offsets{0x330000, 0x332000};
constexpr std::array<uint32_t, 2> gc_image_values{0x76543210u,
0x89abcdefu};
constexpr std::array<uint32_t, 2> gc_stale_values{0x10293847u,
0x56473829u};
for (size_t index = 0; index < gc_image_offsets.size(); index++) {
std::memcpy(memory + gc_image_offsets[index], &gc_image_values[index],
sizeof(uint32_t));
}
auto gc_image_desc_a =
MakeLinearDesc(base + gc_image_offsets[0], sizeof(uint32_t),
vk::Format::eR32Uint, Prospero::BufferFormat::k32UInt,
Prospero::ImageType::kColor2D, {1, 1, 1}, 1, 4, 1);
auto gc_image_desc_b = gc_image_desc_a;
gc_image_desc_b.info.data.address = base + gc_image_offsets[1];
auto clean_buffer_alias = resources.GetBufferCache().ObtainBuffer(
gc_image_desc_a.info.data.address, gc_image_desc_a.info.data.size,
true);
Require(name, "clean pre-image buffer alias",
clean_buffer_alias.first != nullptr,
"failed to create the clean cached Buffer alias");
resources.GetBufferCache().ReadMemory(gc_image_desc_a.info.data.address,
gc_image_desc_a.info.data.size);
const std::array gc_images{texture_cache.FindImage(gc_image_desc_a),
texture_cache.FindImage(gc_image_desc_b)};
Require(
name, "non-GPU image range validity",
!texture_cache.FindImageFromRange(gc_image_desc_a.info.data.address,
gc_image_desc_a.info.data.size),
"FindImageFromRange accepted an image without current GPU "
"contents");
(void)texture_cache.FindTexture(gc_images[0], gc_image_desc_a);
(void)texture_cache.FindTexture(gc_images[1], gc_image_desc_b);
for (const auto image : gc_images) {
texture_cache.MarkGpuWritten(image);
}
Require(name, "GPU image range validity",
texture_cache.FindImageFromRange(
gc_image_desc_a.info.data.address,
gc_image_desc_a.info.data.size) == gc_images[0],
"FindImageFromRange rejected a clean GPU-current image");
auto &gc_native = texture_cache.GetImage(gc_images[0]);
texture_cache.InvalidateMemory(gc_image_desc_a.info.data.address,
gc_image_desc_a.info.data.size);
Require(
name, "CPU-dirty image range validity",
!texture_cache.FindImageFromRange(gc_image_desc_a.info.data.address,
gc_image_desc_a.info.data.size) &&
!TextureCacheTestAccess::TryDownload(texture_cache, gc_images[0]) &&
gc_native.IsDefinitelyCpuDirty() && gc_native.IsGpuModified() &&
!gc_native.IsTracked(),
"CPU writes did not supersede native contents while retaining GPU history");
texture_cache.MarkGpuWritten(gc_images[0]);
Require(name, "GPU image reacquisition",
gc_native.SafeToDownload(),
"a new GPU write did not reclaim image authority");
for (size_t index = 0; index < gc_image_offsets.size(); index++) {
Libs::LibKernel::Memory::WriteBacking(base + gc_image_offsets[index],
&gc_stale_values[index],
sizeof(uint32_t));
}
TextureCacheTestAccess::ConfigureGarbageCollection(
texture_cache, gc_images, 17, UINT64_MAX);
texture_cache.RunGarbageCollector();
Require(name, "downloadable image pre-pressure retention",
std::ranges::all_of(gc_images,
[&](ImageId image) {
return TextureCacheTestAccess::Contains(
texture_cache, image);
}),
"GC retired safely downloadable GPU images before pressure");
const auto gc_batch_tick = scheduler.CurrentTick();
TextureCacheTestAccess::ConfigureGarbageCollection(texture_cache,
gc_images, 81, 0);
texture_cache.RunGarbageCollector();
std::array<uint32_t, 2> gc_before_completion{};
for (size_t index = 0; index < gc_image_offsets.size(); index++) {
Libs::LibKernel::Memory::TryReadBacking(base + gc_image_offsets[index],
&gc_before_completion[index],
sizeof(uint32_t));
}
Require(name, "batched image pressure retirement",
std::ranges::none_of(gc_images,
[&](ImageId image) {
return TextureCacheTestAccess::Contains(
texture_cache, image);
}) &&
scheduler.CurrentTick() == gc_batch_tick &&
gc_before_completion == gc_stale_values,
"GC submitted per image or published a readback before GPU "
"completion");
scheduler.Finish();
scheduler.DrainPriorityOperations();
auto refreshed_buffer_alias = resources.GetBufferCache().ObtainBuffer(
gc_image_desc_a.info.data.address, gc_image_desc_a.info.data.size,
false);
Require(name, "post-publication Buffer reacquire",
refreshed_buffer_alias.first != nullptr,
"Buffer lookup failed after the retired image publication");
std::array<uint32_t, 2> gc_after_completion{};
for (size_t index = 0; index < gc_image_offsets.size(); index++) {
Libs::LibKernel::Memory::TryReadBacking(base + gc_image_offsets[index],
&gc_after_completion[index],
sizeof(uint32_t));
}
Require(name, "batched image readback publication",
scheduler.CurrentTick() == gc_batch_tick + 1 &&
gc_after_completion == gc_image_values,
"one submission did not publish both deferred image readbacks");
Require(name, "refreshed post-image buffer alias",
refreshed_buffer_alias.first != nullptr,
"failed to reacquire the cached Buffer alias after image "
"publication");
auto alias_readback = CreateHostBuffer(
name, sizeof(uint32_t), vk::BufferUsageFlagBits::eTransferDst, {0});
const vk::BufferCopy alias_copy{refreshed_buffer_alias.second, 0,
sizeof(uint32_t)};
scheduler.Current().Handle().copyBuffer(
refreshed_buffer_alias.first->Handle(), alias_readback.buffer, 1,
&alias_copy);
HostReadBarrier(alias_readback.buffer, alias_readback.size,
vk::PipelineStageFlagBits::eTransfer,
vk::AccessFlagBits::eTransferWrite);
scheduler.Finish();
Require(name, "post-image Buffer alias content",
ReadBuffer(name, alias_readback, 1) ==
std::vector<u32>{gc_image_values[0]},
"a clean cached Buffer alias survived image ownership with "
"stale native bytes");
DestroyBuffer(&alias_readback);
constexpr uint64_t submit_readback_offset = 0x336000;
constexpr uint32_t submit_readback_value = 0x13579bdfu;
constexpr uint32_t submit_readback_stale = 0x2468ace0u;
auto submit_readback_desc =
MakeLinearDesc(base + submit_readback_offset, sizeof(uint32_t),
vk::Format::eR32Uint, Prospero::BufferFormat::k32UInt,
Prospero::ImageType::kColor2D, {1, 1, 1}, 1, 4, 1);
submit_readback_desc.type = BindingType::Storage;
submit_readback_desc.view_info.usage = vk::ImageUsageFlagBits::eStorage;
TextureCacheTestAccess::SetLinearReadback(texture_cache, true);
const auto submit_readback_image =
texture_cache.FindImage(submit_readback_desc);
auto submit_buffer = resources.GetBufferCache().ObtainBuffer(
submit_readback_desc.info.data.address,
submit_readback_desc.info.data.size, true);
Require(name, "linear submit dirty buffer allocation",
submit_buffer.first != nullptr,
"failed to create the older GPU Buffer contents");
submit_buffer.first->Fill(submit_buffer.second, sizeof(uint32_t),
submit_readback_stale);
texture_cache.InvalidateMemoryFromGPU(
submit_readback_desc.info.data.address,
submit_readback_desc.info.data.size);
(void)texture_cache.FindTexture(submit_readback_image,
submit_readback_desc);
Require(name, "linear submit image clear",
texture_cache.ClearImageFromBuffer(
command, submit_readback_desc.info.data.address,
submit_readback_desc.info.data.size, submit_readback_value),
"failed to create GPU-current linear storage contents");
Require(name, "older buffer permits current image readback",
resources.GetBufferCache().HasGpuDirtyBytes(
submit_readback_desc.info.data.address,
submit_readback_desc.info.data.size) &&
texture_cache.FindImageFromRange(
submit_readback_desc.info.data.address,
submit_readback_desc.info.data.size) ==
submit_readback_image,
"historical GPU Buffer dirtiness rejected newer image contents");
TextureCacheTestAccess::TrackDownload(texture_cache,
submit_readback_image);
Libs::LibKernel::Memory::WriteBacking(
submit_readback_desc.info.data.address, &submit_readback_stale,
sizeof(submit_readback_stale));
const auto submit_readback_tick = scheduler.CurrentTick();
texture_cache.ProcessDownloadImages();
TextureCacheTestAccess::SetLinearReadback(texture_cache, false);
uint32_t submit_before_completion = 0;
Libs::LibKernel::Memory::TryReadBacking(
submit_readback_desc.info.data.address, &submit_before_completion,
sizeof(submit_before_completion));
Require(name, "linear submit deferred publication",
submit_before_completion == submit_readback_stale &&
scheduler.CurrentTick() == submit_readback_tick,
"submit-time linear readback published synchronously");
scheduler.Finish();
scheduler.DrainPriorityOperations();
uint32_t submit_after_completion = 0;
Libs::LibKernel::Memory::TryReadBacking(
submit_readback_desc.info.data.address, &submit_after_completion,
sizeof(submit_after_completion));
Require(name, "linear submit readback reuse",
submit_after_completion == submit_readback_value &&
TextureCacheTestAccess::Contains(texture_cache,
submit_readback_image) &&
texture_cache.GetImage(submit_readback_image).IsGpuModified(),
"submit-time readback did not publish or retained no reusable "
"GPU-current image");
constexpr uint64_t linear_depth_offset = 0x33a000;
constexpr uint64_t linear_stencil_offset = 0x33b000;
constexpr uint32_t linear_depth_width = 3;
constexpr uint32_t linear_depth_height = 2;
constexpr uint32_t linear_depth_pitch = 4;
constexpr uint32_t linear_depth_layers = 2;
constexpr uint32_t linear_depth_words =
linear_depth_pitch * linear_depth_height * linear_depth_layers;
constexpr uint32_t linear_depth_clear = 0x3f400000u;
std::array<uint32_t, linear_depth_words> linear_depth_guest{};
for (uint32_t index = 0; index < linear_depth_words; index++) {
linear_depth_guest[index] = 0x51000000u + index;
}
constexpr std::array<uint8_t, 16> linear_stencil_guest{
0x91, 0x82, 0x73, 0x64, 0x55, 0x46, 0x37, 0x28,
0x19, 0x2a, 0x3b, 0x4c, 0x5d, 0x6e, 0x7f, 0x80};
std::memcpy(memory + linear_depth_offset, linear_depth_guest.data(),
sizeof(linear_depth_guest));
std::memcpy(memory + linear_stencil_offset, linear_stencil_guest.data(),
sizeof(linear_stencil_guest));
auto linear_depth_desc = MakeLinearDesc(
base + linear_depth_offset, sizeof(linear_depth_guest),
vk::Format::eD32SfloatS8Uint, Prospero::BufferFormat::k32Float,
Prospero::ImageType::kColor2D,
{linear_depth_width, linear_depth_height, 1}, linear_depth_layers, 4,
1);
linear_depth_desc.type = BindingType::DepthTarget;
linear_depth_desc.info.pitch = linear_depth_pitch;
linear_depth_desc.info.stencil = {base + linear_stencil_offset,
sizeof(linear_stencil_guest)};
linear_depth_desc.info.mip_layout[0] = {0, sizeof(linear_depth_guest),
linear_depth_pitch,
linear_depth_height};
linear_depth_desc.info.UpdateSize();
linear_depth_desc.view_info.aspect =
vk::ImageAspectFlagBits::eDepth | vk::ImageAspectFlagBits::eStencil;
linear_depth_desc.view_info.usage =
vk::ImageUsageFlagBits::eDepthStencilAttachment;
const auto linear_depth_image =
texture_cache.FindImage(linear_depth_desc);
Require(name, "linear depth native clear",
texture_cache.ClearImageFromBuffer(
command, linear_depth_desc.info.data.address,
linear_depth_desc.info.data.size, linear_depth_clear),
"failed to create GPU-current padded linear depth contents");
TextureCacheTestAccess::SetLinearReadback(texture_cache, true);
TextureCacheTestAccess::TrackDownload(texture_cache, linear_depth_image);
const auto linear_depth_tick = scheduler.CurrentTick();
texture_cache.ProcessDownloadImages();
TextureCacheTestAccess::SetLinearReadback(texture_cache, false);
std::array<uint32_t, linear_depth_words> linear_depth_before{};
std::memcpy(linear_depth_before.data(), memory + linear_depth_offset,
sizeof(linear_depth_before));
Require(name, "linear depth deferred publication",
linear_depth_before == linear_depth_guest &&
scheduler.CurrentTick() == linear_depth_tick,
"linear depth readback published before GPU completion");
scheduler.Finish();
scheduler.DrainPriorityOperations();
std::array<uint32_t, linear_depth_words> linear_depth_after{};
std::array<uint8_t, linear_stencil_guest.size()> linear_stencil_after{};
std::memcpy(linear_depth_after.data(), memory + linear_depth_offset,
sizeof(linear_depth_after));
std::memcpy(linear_stencil_after.data(), memory + linear_stencil_offset,
sizeof(linear_stencil_after));
bool linear_depth_matches = true;
uint32_t linear_depth_mismatch = UINT32_MAX;
for (uint32_t layer = 0; layer < linear_depth_layers; layer++) {
for (uint32_t y = 0; y < linear_depth_height; y++) {
for (uint32_t x = 0; x < linear_depth_pitch; x++) {
const auto index =
layer * linear_depth_pitch * linear_depth_height +
y * linear_depth_pitch + x;
const auto expected = x < linear_depth_width
? linear_depth_clear
: linear_depth_guest[index];
if (linear_depth_after[index] != expected &&
linear_depth_mismatch == UINT32_MAX) {
linear_depth_mismatch = index;
}
linear_depth_matches &= linear_depth_after[index] == expected;
}
}
}
Require(name, "linear depth native contents",
linear_depth_matches &&
linear_stencil_after == linear_stencil_guest &&
TextureCacheTestAccess::Contains(texture_cache,
linear_depth_image) &&
texture_cache.GetImage(linear_depth_image).IsGpuModified(),
fmt::format(
"linear depth mismatch={} actual=0x{:08x} expected=0x{:08x} "
"stencil={} contains={} gpu={}",
linear_depth_mismatch,
linear_depth_mismatch == UINT32_MAX
? 0
: linear_depth_after[linear_depth_mismatch],
linear_depth_mismatch == UINT32_MAX
? 0
: (linear_depth_mismatch % linear_depth_pitch <
linear_depth_width
? linear_depth_clear
: linear_depth_guest[linear_depth_mismatch]),
linear_stencil_after == linear_stencil_guest,
TextureCacheTestAccess::Contains(texture_cache,
linear_depth_image),
texture_cache.GetImage(linear_depth_image).IsGpuModified())
.c_str());
constexpr uint64_t tiled_depth_offset = 0x380000;
constexpr uint32_t tiled_depth_width = 3;
constexpr uint32_t tiled_depth_height = 2;
constexpr uint32_t tiled_depth_pitch = 4;
constexpr uint32_t tiled_depth_layers = 2;
constexpr uint64_t tiled_depth_slice = 0x10000;
constexpr uint64_t tiled_depth_size =
tiled_depth_slice * tiled_depth_layers;
constexpr uint32_t tiled_depth_clear = 0x3f200000u;
constexpr uint32_t tiled_depth_stale = 0xdeadbeefu;
std::vector<uint32_t> tiled_depth_guest(
tiled_depth_size / sizeof(uint32_t), tiled_depth_stale);
std::memcpy(memory + tiled_depth_offset, tiled_depth_guest.data(),
tiled_depth_size);
auto tiled_depth_desc = MakeLinearDesc(
base + tiled_depth_offset, tiled_depth_size, vk::Format::eD32Sfloat,
Prospero::BufferFormat::k32Float, Prospero::ImageType::kColor2D,
{tiled_depth_width, tiled_depth_height, 1}, tiled_depth_layers, 4, 1);
tiled_depth_desc.type = BindingType::DepthTarget;
tiled_depth_desc.info.pitch = tiled_depth_pitch;
tiled_depth_desc.info.tile_mode = Prospero::TileMode::kDepth;
tiled_depth_desc.info.mip_layout[0] = {
0, tiled_depth_size, tiled_depth_pitch, tiled_depth_height};
tiled_depth_desc.view_info.aspect = vk::ImageAspectFlagBits::eDepth;
tiled_depth_desc.view_info.usage =
vk::ImageUsageFlagBits::eDepthStencilAttachment;
tiled_depth_desc.info.UpdateSize();
const auto tiled_depth_image = texture_cache.FindImage(tiled_depth_desc);
Require(name, "tiled depth native clear",
texture_cache.ClearImageFromBuffer(
command, tiled_depth_desc.info.data.address,
tiled_depth_desc.info.data.size, tiled_depth_clear),
"failed to create GPU-current layered tiled depth contents");
const auto [tiled_prefix, tiled_prefix_offset] =
TextureCacheTestAccess::MapDownload(texture_cache, 64, 64);
Require(name, "tiled depth download prefix",
tiled_prefix != nullptr && tiled_prefix_offset != 0,
"failed to force a nonzero reusable-download offset");
const auto tiled_depth_tick = scheduler.CurrentTick();
Require(
name, "tiled depth download queue",
TextureCacheTestAccess::TryDownload(texture_cache, tiled_depth_image),
"layered tiled depth readback was rejected");
std::vector<uint32_t> tiled_depth_before(tiled_depth_guest.size());
std::memcpy(tiled_depth_before.data(), memory + tiled_depth_offset,
tiled_depth_size);
Require(name, "tiled depth deferred publication",
tiled_depth_before == tiled_depth_guest &&
scheduler.CurrentTick() == tiled_depth_tick,
"tiled depth readback published before GPU completion");
scheduler.Finish();
scheduler.DrainPriorityOperations();
std::vector<uint32_t> tiled_depth_after(tiled_depth_guest.size());
std::memcpy(tiled_depth_after.data(), memory + tiled_depth_offset,
tiled_depth_size);
const auto active_tiled_words =
tiled_depth_width * tiled_depth_height * tiled_depth_layers;
const auto clear_tiled_words =
std::ranges::count(tiled_depth_after, tiled_depth_clear);
const auto stale_tiled_words =
std::ranges::count(tiled_depth_after, tiled_depth_stale);
Require(
name, "tiled depth backing preservation",
clear_tiled_words == active_tiled_words &&
stale_tiled_words ==
tiled_depth_after.size() - active_tiled_words,
fmt::format("tiled depth readback changed clear={}/{} stale={}/{}",
clear_tiled_words, active_tiled_words, stale_tiled_words,
tiled_depth_after.size() - active_tiled_words)
.c_str());
TileBlockLayout depth_block{};
Require(name, "tiled depth block layout",
TileGetBlockLayout(TileBlockFamily::Depth64KB, sizeof(uint32_t),
depth_block),
"failed to describe the tiled depth test surface");
auto tiled_depth_info =
MakeTilingImage(Prospero::BufferFormat::k32Float, tiled_depth_width,
tiled_depth_height, 1, tiled_depth_layers,
Prospero::TileMode::kDepth, tiled_depth_size, false);
tiled_depth_info.linear_slice_stride = tiled_depth_slice;
tiled_depth_info.mip_layout[0].linear_size = tiled_depth_slice;
tiled_depth_info.mip_layout[0].linear_pitch = tiled_depth_pitch;
auto tiled_depth_input =
CreateHostBuffer(name, tiled_depth_size, AllFlags, tiled_depth_after);
auto tiled_depth_linear =
TextureCacheTestAccess::Tiler(texture_cache)
.Detile(tiled_depth_input.buffer, 0, tiled_depth_info);
auto tiled_depth_output =
CreateHostBuffer(name, tiled_depth_size, AllFlags,
std::vector<uint32_t>(tiled_depth_guest.size(), 0));
const vk::BufferCopy tiled_depth_copy{tiled_depth_linear.offset, 0,
tiled_depth_size};
scheduler.Current().Handle().copyBuffer(tiled_depth_linear.buffer,
tiled_depth_output.buffer, 1,
&tiled_depth_copy);
HostReadBarrier(tiled_depth_output.buffer, tiled_depth_size,
vk::PipelineStageFlagBits::eTransfer,
vk::AccessFlagBits::eTransferWrite);
scheduler.Finish();
const auto tiled_depth_words =
ReadBuffer(name, tiled_depth_output, tiled_depth_guest.size());
bool tiled_depth_matches = true;
for (uint32_t layer = 0; layer < tiled_depth_layers; layer++) {
for (uint32_t y = 0; y < tiled_depth_height; y++) {
for (uint32_t x = 0; x < tiled_depth_width; x++) {
const auto index = layer * (tiled_depth_slice / sizeof(uint32_t)) +
y * tiled_depth_pitch + x;
tiled_depth_matches &=
tiled_depth_words[index] == tiled_depth_clear;
}
}
}
Require(name, "tiled depth layered contents",
tiled_depth_matches &&
TextureCacheTestAccess::Contains(texture_cache,
tiled_depth_image) &&
texture_cache.GetImage(tiled_depth_image).IsGpuModified(),
"tiled depth readback lost a padded slice or retired its image");
DestroyBuffer(&tiled_depth_output);
DestroyBuffer(&tiled_depth_input);
constexpr uint64_t tiled_d16_offset = 0x3a0000;
constexpr uint64_t tiled_d16_stencil_offset = 0x3c0000;
constexpr uint16_t tiled_d16_clear = 0x8000u;
constexpr uint16_t tiled_d16_stale = 0xbeefu;
std::vector<uint16_t> tiled_d16_guest(tiled_depth_size / sizeof(uint16_t),
tiled_d16_stale);
constexpr std::array<uint8_t, 16> tiled_d16_stencil{
0x10, 0x21, 0x32, 0x43, 0x54, 0x65, 0x76, 0x87,
0x98, 0xa9, 0xba, 0xcb, 0xdc, 0xed, 0xfe, 0x0f};
std::memcpy(memory + tiled_d16_offset, tiled_d16_guest.data(),
tiled_depth_size);
std::memcpy(memory + tiled_d16_stencil_offset, tiled_d16_stencil.data(),
tiled_d16_stencil.size());
auto tiled_d16_desc = MakeLinearDesc(
base + tiled_d16_offset, tiled_depth_size,
vk::Format::eD32SfloatS8Uint, Prospero::BufferFormat::k16UNorm,
Prospero::ImageType::kColor2D,
{tiled_depth_width, tiled_depth_height, 1}, tiled_depth_layers, 2, 1);
tiled_d16_desc.type = BindingType::DepthTarget;
tiled_d16_desc.info.pitch = tiled_depth_pitch;
tiled_d16_desc.info.tile_mode = Prospero::TileMode::kDepth;
tiled_d16_desc.info.stencil = {base + tiled_d16_stencil_offset,
tiled_d16_stencil.size()};
tiled_d16_desc.info.mip_layout[0] = {
0, tiled_depth_size, tiled_depth_pitch, tiled_depth_height};
tiled_d16_desc.view_info.aspect =
vk::ImageAspectFlagBits::eDepth | vk::ImageAspectFlagBits::eStencil;
tiled_d16_desc.view_info.usage =
vk::ImageUsageFlagBits::eDepthStencilAttachment;
tiled_d16_desc.info.UpdateSize();
const auto tiled_d16_image = texture_cache.FindImage(tiled_d16_desc);
Require(name, "tiled D16 native clear",
texture_cache.ClearImageFromBuffer(
command, tiled_d16_desc.info.data.address,
tiled_d16_desc.info.data.size,
EncodeD16AsD32(tiled_d16_clear)),
"failed to clear a layered tiled D16 fallback image");
Require(
name, "tiled D16 download queue",
TextureCacheTestAccess::TryDownload(texture_cache, tiled_d16_image),
"layered tiled D16 fallback readback was rejected");
scheduler.Finish();
scheduler.DrainPriorityOperations();
std::vector<uint16_t> tiled_d16_after(tiled_d16_guest.size());
std::array<uint8_t, tiled_d16_stencil.size()> tiled_d16_stencil_after{};
std::memcpy(tiled_d16_after.data(), memory + tiled_d16_offset,
tiled_depth_size);
std::memcpy(tiled_d16_stencil_after.data(),
memory + tiled_d16_stencil_offset,
tiled_d16_stencil_after.size());
Require(name, "tiled D16 backing preservation",
std::ranges::count(tiled_d16_after, tiled_d16_clear) ==
active_tiled_words &&
std::ranges::count(tiled_d16_after, tiled_d16_stale) ==
tiled_d16_after.size() - active_tiled_words &&
tiled_d16_stencil_after == tiled_d16_stencil,
"tiled D16 fallback overwrote inactive depth or stencil backing");
std::vector<uint32_t> tiled_d16_words(tiled_depth_size /
sizeof(uint32_t));
std::memcpy(tiled_d16_words.data(), tiled_d16_after.data(),
tiled_depth_size);
Require(name, "tiled D16 block layout",
TileGetBlockLayout(TileBlockFamily::Depth64KB, sizeof(uint16_t),
depth_block),
"failed to describe the tiled D16 test surface");
auto tiled_d16_info =
MakeTilingImage(Prospero::BufferFormat::k16UNorm, tiled_depth_width,
tiled_depth_height, 1, tiled_depth_layers,
Prospero::TileMode::kDepth, tiled_depth_size, false);
tiled_d16_info.linear_slice_stride = tiled_depth_slice;
tiled_d16_info.mip_layout[0].linear_size = tiled_depth_slice;
tiled_d16_info.mip_layout[0].linear_pitch = tiled_depth_pitch;
auto tiled_d16_input =
CreateHostBuffer(name, tiled_depth_size, AllFlags, tiled_d16_words);
auto tiled_d16_linear =
TextureCacheTestAccess::Tiler(texture_cache)
.Detile(tiled_d16_input.buffer, 0, tiled_d16_info);
auto tiled_d16_output =
CreateHostBuffer(name, tiled_depth_size, AllFlags,
std::vector<uint32_t>(tiled_d16_words.size(), 0));
const vk::BufferCopy tiled_d16_copy{tiled_d16_linear.offset, 0,
tiled_depth_size};
scheduler.Current().Handle().copyBuffer(
tiled_d16_linear.buffer, tiled_d16_output.buffer, 1, &tiled_d16_copy);
HostReadBarrier(tiled_d16_output.buffer, tiled_depth_size,
vk::PipelineStageFlagBits::eTransfer,
vk::AccessFlagBits::eTransferWrite);
scheduler.Finish();
const auto tiled_d16_linear_words =
ReadBuffer(name, tiled_d16_output, tiled_d16_words.size());
std::vector<uint16_t> tiled_d16_linear_values(tiled_d16_guest.size());
std::memcpy(tiled_d16_linear_values.data(), tiled_d16_linear_words.data(),
tiled_depth_size);
bool tiled_d16_matches = true;
for (uint32_t layer = 0; layer < tiled_depth_layers; layer++) {
for (uint32_t y = 0; y < tiled_depth_height; y++) {
for (uint32_t x = 0; x < tiled_depth_width; x++) {
const auto index = layer * (tiled_depth_slice / sizeof(uint16_t)) +
y * tiled_depth_pitch + x;
tiled_d16_matches &=
tiled_d16_linear_values[index] == tiled_d16_clear;
}
}
}
Require(name, "tiled D16 layered contents",
tiled_d16_matches &&
TextureCacheTestAccess::Contains(texture_cache,
tiled_d16_image) &&
texture_cache.GetImage(tiled_d16_image).IsGpuModified(),
"tiled D16 fallback lost a converted layer or retired its image");
DestroyBuffer(&tiled_d16_output);
DestroyBuffer(&tiled_d16_input);
constexpr uint64_t d16_fallback_offset = 0x33c000;
constexpr uint64_t d16_fallback_stencil_offset = 0x33d000;
constexpr std::array<uint16_t, 4> d16_fallback_values{0x0000u, 0x2468u,
0xabcdu, 0xffffu};
std::memcpy(memory + d16_fallback_offset, d16_fallback_values.data(),
sizeof(d16_fallback_values));
std::memset(memory + d16_fallback_stencil_offset, 0x6d, 4);
auto d16_depth_desc = MakeLinearDesc(
base + d16_fallback_offset, sizeof(d16_fallback_values),
vk::Format::eD24UnormS8Uint, Prospero::BufferFormat::k16UNorm,
Prospero::ImageType::kColor2D, {4, 1, 1}, 1, 2, 1);
d16_depth_desc.type = BindingType::DepthTarget;
d16_depth_desc.info.stencil = {base + d16_fallback_stencil_offset, 4};
d16_depth_desc.view_info.aspect =
vk::ImageAspectFlagBits::eDepth | vk::ImageAspectFlagBits::eStencil;
d16_depth_desc.view_info.usage =
vk::ImageUsageFlagBits::eDepthStencilAttachment;
const auto d16_depth_image = texture_cache.FindImage(d16_depth_desc);
(void)texture_cache.FindDepthTarget(d16_depth_image, d16_depth_desc);
texture_cache.MarkGpuWritten(d16_depth_image);
auto d16_storage_desc = d16_depth_desc;
d16_storage_desc.type = BindingType::Storage;
d16_storage_desc.info.stencil = {};
d16_storage_desc.info.pixel_format = vk::Format::eR16Unorm;
d16_storage_desc.view_info.format = vk::Format::eR16Unorm;
d16_storage_desc.view_info.aspect = vk::ImageAspectFlagBits::eColor;
d16_storage_desc.view_info.usage = vk::ImageUsageFlagBits::eStorage;
const auto d16_storage_image =
texture_cache.FindImage(d16_storage_desc, true);
auto d16_storage_readback = CreateHostBuffer(
name, sizeof(d16_fallback_values),
vk::BufferUsageFlagBits::eTransferDst, std::vector<u32>(2, 0));
auto &d16_storage_native = texture_cache.GetImage(d16_storage_image);
d16_storage_native.Transit(vk::ImageLayout::eTransferSrcOptimal,
vk::AccessFlagBits2::eTransferRead, {},
scheduler.Current().Handle());
vk::BufferImageCopy d16_storage_copy{};
d16_storage_copy.bufferRowLength = 4;
d16_storage_copy.imageSubresource = {vk::ImageAspectFlagBits::eColor, 0,
0, 1};
d16_storage_copy.imageExtent = {4, 1, 1};
scheduler.Current().Handle().copyImageToBuffer(
d16_storage_native.backing.image,
vk::ImageLayout::eTransferSrcOptimal, d16_storage_readback.buffer, 1,
&d16_storage_copy);
HostReadBarrier(d16_storage_readback.buffer, d16_storage_readback.size,
vk::PipelineStageFlagBits::eTransfer,
vk::AccessFlagBits::eTransferWrite);
scheduler.Finish();
const auto d16_storage_words = ReadBuffer(name, d16_storage_readback, 2);
std::array<uint16_t, 4> d16_storage_values{};
std::memcpy(d16_storage_values.data(), d16_storage_words.data(),
sizeof(d16_storage_values));
Require(
name, "D16 fallback depth/storage copy",
d16_storage_values == d16_fallback_values &&
d16_storage_image != d16_depth_image &&
texture_cache.GetImage(d16_storage_image).IsGpuModified(),
"D16 fallback reinterpreted four-byte host depth as two-byte color");
DestroyBuffer(&d16_storage_readback);
if (!texture_cache.GetImage(combined_destination_image).IsGpuModified()) {
texture_cache.MarkGpuWritten(combined_destination_image);
}
constexpr float stale_added_stencil_depth = 0.125f;
Libs::LibKernel::Memory::WriteBacking(
combined_destination.info.data.address, &stale_added_stencil_depth,
sizeof(stale_added_stencil_depth));
const auto depth_gc_tick = scheduler.CurrentTick();
TextureCacheTestAccess::ConfigureGarbageCollection(
texture_cache, std::array{combined_destination_image}, 81, 0);
texture_cache.RunGarbageCollector();
float depth_before_completion = 0.0f;
Libs::LibKernel::Memory::TryReadBacking(
combined_destination.info.data.address, &depth_before_completion,
sizeof(depth_before_completion));
const bool depth_image_retired = !TextureCacheTestAccess::Contains(
texture_cache, combined_destination_image);
const bool depth_proxy_retired = !texture_cache.FindImageFromRange(
base + added_stencil_offset, added_stencil_size, false);
Require(
name, "depth/stencil deferred pressure retirement",
depth_image_retired && depth_proxy_retired &&
scheduler.CurrentTick() == depth_gc_tick &&
depth_before_completion == stale_added_stencil_depth,
fmt::format(
"GC failed to retire/defer depth: image={} proxy={} tick={}/{} "
"depth={}/{}",
depth_image_retired, depth_proxy_retired, scheduler.CurrentTick(),
depth_gc_tick, depth_before_completion, stale_added_stencil_depth)
.c_str());
scheduler.Finish();
scheduler.DrainPriorityOperations();
float depth_after_completion = 0.0f;
Libs::LibKernel::Memory::TryReadBacking(
combined_destination.info.data.address, &depth_after_completion,
sizeof(depth_after_completion));
Require(name, "depth/stencil depth-plane preservation",
depth_after_completion == added_stencil_depth_value,
"GC discarded the current depth plane of a depth/stencil "
"image");
constexpr size_t gc_depth_pair_count = 6;
constexpr uint64_t gc_depth_pair_offset = 0x350000;
constexpr uint64_t gc_depth_pair_stride = 0x1000;
std::array<ImageId, gc_depth_pair_count> gc_depth_images{};
std::array<ImageId, gc_depth_pair_count> gc_stencil_images{};
std::array<ImageId, gc_depth_pair_count * 2> gc_depth_lru{};
for (size_t index = 0; index < gc_depth_pair_count; index++) {
auto depth = MakeLinearDesc(
base + gc_depth_pair_offset + index * gc_depth_pair_stride,
sizeof(float), vk::Format::eD32SfloatS8Uint,
Prospero::BufferFormat::k32Float, Prospero::ImageType::kColor2D,
{1, 1, 1}, 1, 4, 1);
depth.type = BindingType::DepthTarget;
depth.info.stencil = {depth.info.data.address +
gc_depth_pair_stride / 2,
sizeof(uint32_t)};
depth.view_info.format = vk::Format::eD32SfloatS8Uint;
depth.view_info.aspect =
vk::ImageAspectFlagBits::eDepth | vk::ImageAspectFlagBits::eStencil;
depth.view_info.usage = vk::ImageUsageFlagBits::eDepthStencilAttachment;
gc_depth_images[index] = texture_cache.FindImage(depth);
TextureCacheTestAccess::AssociateStencil(
texture_cache, gc_depth_images[index], depth.info.stencil);
gc_stencil_images[index] = texture_cache.FindImageFromRange(
depth.info.stencil.address, depth.info.stencil.size, false);
gc_depth_lru[index * 2] = gc_depth_images[index];
gc_depth_lru[index * 2 + 1] = gc_stencil_images[index];
}
Require(
name, "depth/stencil GC fixtures",
std::ranges::all_of(gc_depth_images,
[](ImageId image) { return bool{image}; }) &&
std::ranges::all_of(gc_stencil_images,
[](ImageId image) { return bool{image}; }),
"failed to create six depth/stencil association pairs");
TextureCacheTestAccess::ConfigureGarbageCollection(
texture_cache, gc_depth_lru, 81, UINT64_MAX);
texture_cache.RunGarbageCollector();
bool gc_depth_budget = true;
for (size_t index = 0; index < gc_depth_pair_count; index++) {
const bool expected_live = index == gc_depth_pair_count - 1;
gc_depth_budget &=
TextureCacheTestAccess::Contains(
texture_cache, gc_depth_images[index]) == expected_live &&
TextureCacheTestAccess::Contains(
texture_cache, gc_stencil_images[index]) == expected_live;
}
Require(name, "depth/stencil GC traversal budget", gc_depth_budget,
"recursive association deletion stopped LRU traversal or "
"exceeded the ten-entry deletion budget");
constexpr uint64_t large_offset = 0x400000;
constexpr uint32_t large_width = 4096;
constexpr uint32_t large_height = 2047;
constexpr uint64_t large_size =
uint64_t{large_width} * large_height * sizeof(uint32_t);
static_assert(large_size < (32ull << 20));
std::memset(memory + large_offset, 0, static_cast<size_t>(large_size));
auto large_desc = MakeLinearDesc(
base + large_offset, large_size, vk::Format::eR8G8B8A8Unorm,
Prospero::BufferFormat::k8_8_8_8UNorm, Prospero::ImageType::kColor2D,
{large_width, large_height, 1}, 1, 4, 1);
const auto RunLargeReadback = [&](uint32_t clear_value) {
const auto image = texture_cache.FindImage(large_desc);
Require(name, "near-capacity image clear",
texture_cache.ClearImageFromBuffer(
command, large_desc.info.data.address,
large_desc.info.data.size, clear_value),
"failed to write the near-capacity readback source image");
const std::array<uint64_t, 3> sample_offsets{
0, large_size / 2, large_size - sizeof(uint32_t)};
constexpr uint32_t stale = 0;
for (const auto offset : sample_offsets) {
Libs::LibKernel::Memory::WriteBacking(base + large_offset + offset,
&stale, sizeof(stale));
}
TextureCacheTestAccess::ConfigureGarbageCollection(
texture_cache, std::array{image}, 81, 0);
const auto tick = scheduler.CurrentTick();
texture_cache.RunGarbageCollector();
const auto handle =
BufferCacheTestAccess::DownloadBuffer(resources.GetBufferCache())
.Handle();
Require(
name, "near-capacity deferred retirement",
!TextureCacheTestAccess::Contains(texture_cache, image) &&
scheduler.CurrentTick() == tick,
"near-capacity readback was rejected or synchronously submitted");
scheduler.Finish();
scheduler.DrainPriorityOperations();
bool content = true;
for (const auto offset : sample_offsets) {
uint32_t value = 0;
Libs::LibKernel::Memory::TryReadBacking(base + large_offset + offset,
&value, sizeof(value));
content &= value == clear_value;
}
Require(
name, "near-capacity readback content", content,
"near-capacity image readback did not publish its GPU contents");
return handle;
};
const auto large_download = RunLargeReadback(0xa5a5a5a5u);
const auto reused_large_download = RunLargeReadback(0x5a5a5a5au);
Require(name, "near-capacity shared download reuse",
large_download == reused_large_download,
"successive near-capacity image transfers replaced the shared "
"download buffer");
constexpr uint64_t tile_alias_offset = 0x2000000;
constexpr uint64_t tile_alias_size = 0x400000;
constexpr uint32_t tile_alias_extent = 1024;
std::memset(memory + tile_alias_offset, 0,
static_cast<size_t>(tile_alias_size));
auto render_target_alias = MakeLinearDesc(
base + tile_alias_offset, tile_alias_size, vk::Format::eR8G8B8A8Unorm,
Prospero::BufferFormat::k8_8_8_8UNorm, Prospero::ImageType::kColor2D,
{tile_alias_extent, tile_alias_extent, 1}, 1, 4, 1);
render_target_alias.type = BindingType::Storage;
render_target_alias.info.tile_mode = Prospero::TileMode::kRenderTarget;
render_target_alias.info.UpdateSize();
render_target_alias.view_info.usage = vk::ImageUsageFlagBits::eStorage;
const auto render_target_alias_image =
texture_cache.FindImage(render_target_alias);
auto standard_4kb_alias = render_target_alias;
standard_4kb_alias.type = BindingType::Texture;
standard_4kb_alias.info.tile_mode = Prospero::TileMode::kStandard4KB;
standard_4kb_alias.info.UpdateSize();
standard_4kb_alias.view_info.usage = vk::ImageUsageFlagBits::eSampled;
const auto standard_4kb_alias_image =
texture_cache.FindImage(standard_4kb_alias);
auto repeated_standard_4kb_alias = standard_4kb_alias;
const auto repeated_standard_4kb_alias_image =
texture_cache.FindImage(repeated_standard_4kb_alias);
Require(
name, "equal-size tile-mode alias",
render_target_alias_image && standard_4kb_alias_image &&
standard_4kb_alias_image != render_target_alias_image &&
repeated_standard_4kb_alias_image == standard_4kb_alias_image &&
texture_cache.GetImage(render_target_alias_image)
.info.tile_mode == Prospero::TileMode::kRenderTarget &&
texture_cache.GetImage(standard_4kb_alias_image).info.tile_mode ==
Prospero::TileMode::kStandard4KB,
"equal address/size lookup reused an incompatible "
"tiled backing");
for (auto &output : ms_observer_outputs) {
DestroyBuffer(&output);
}
DestroyBuffer(&layered_readback);
DestroyBuffer(&bgra16_readback);
DestroyBuffer(&mixed_source_readback);
DestroyBuffer(&bridged_source_readback);
DestroyBuffer(&mip_formatted_readback);
DestroyBuffer(&mip_prefix_readback);
DestroyBuffer(&partial_cpu_refresh_readback);
DestroyBuffer(&mirror_cpu_readback);
DestroyBuffer(&exact_buffer_readback);
m_device.destroyDescriptorPool(observer_pool, nullptr);
m_device.destroyPipeline(observer_depth_pipeline, nullptr);
m_device.destroyPipelineLayout(observer_pipeline_layout, nullptr);
m_device.destroyDescriptorSetLayout(observer_set_layout, nullptr);
m_device.destroyShaderModule(ms_depth_module, nullptr);
constexpr uint64_t mapped_end = base + allocation_size;
constexpr uint64_t final_word_address = mapped_end - sizeof(uint32_t);
auto *final_word = reinterpret_cast<volatile uint32_t *>(final_word_address);
auto *final_byte = reinterpret_cast<volatile uint8_t *>(mapped_end - 1);
Require(name, "fault boundary containment",
resources.IsMapped(final_word_address, sizeof(uint32_t)) &&
!resources.IsMapped(final_word_address, sizeof(uint64_t)) &&
!resources.HandleFault(PageFaultAccess::Write, mapped_end) &&
!resources.HandleFault(PageFaultAccess::Read, mapped_end),
"fault handling changed the mapped range's exclusive end");
Require(name, "fault boundary setup",
resources.InvalidateMemory(final_word_address, sizeof(uint32_t)),
"the final mapped word could not be initialized");
*final_word = 0;
auto final_word_desc = MakeLinearDesc(
final_word_address, sizeof(uint32_t), vk::Format::eR32Uint,
Prospero::BufferFormat::k32UInt, Prospero::ImageType::kColor2D,
{1, 1, 1}, 1, 4, 1);
const auto final_word_image = texture_cache.FindImage(final_word_desc);
(void)texture_cache.FindTexture(final_word_image, final_word_desc);
Require(name, "fault boundary image tracking",
texture_cache.GetImage(final_word_image).IsTracked() &&
!texture_cache.GetImage(final_word_image).IsCpuDirty(),
"the final mapped word's image was not clean before its CPU store");
*final_word = 45;
Require(name, "fault boundary CPU store",
*final_word == 45 &&
texture_cache.GetImage(final_word_image).IsDefinitelyCpuDirty(),
"a valid four-byte store at the mapping end did not invalidate its image");
constexpr uint32_t final_gpu_value = 0xa1b2c3d4u;
auto &boundary_buffer_cache = resources.GetBufferCache();
(void)boundary_buffer_cache.ObtainBuffer(final_word_address, sizeof(uint32_t), true);
boundary_buffer_cache.FillBuffer(final_word_address, sizeof(uint32_t),
final_gpu_value, false);
Require(name, "fault boundary GPU ownership",
boundary_buffer_cache.HasGpuDirtyBytes(final_word_address, sizeof(uint32_t)) &&
!HostMemoryIsReadable(mapped_end - 1),
"the final mapped byte was not protected by GPU ownership");
const uint8_t final_gpu_byte = *final_byte;
Require(name, "fault boundary GPU readback",
final_gpu_byte == static_cast<uint8_t>(final_gpu_value >> 24u) &&
*final_word == final_gpu_value &&
!boundary_buffer_cache.HasGpuDirtyBytes(final_word_address, sizeof(uint32_t)),
"a final-byte CPU read did not publish the GPU-owned final word");
resources.UnmapMemory(base, allocation_size);
scheduler.Finish();
LibKernel::Memory::InstallGpuResources(nullptr);
}
context.ShutdownGpu();
Require(name, "unmap direct backing",
Libs::LibKernel::Memory::KernelMunmap(base, allocation_size) == 0,
"cache direct-memory mapping release failed");
Require(name, "release direct backing",
Libs::LibKernel::Memory::KernelReleaseDirectMemory(
direct_offset, allocation_size) == 0,
"cache direct-memory allocation release failed");
std::printf("[host] %-32s ok\n", name);
}
void CheckBgra16Readback() {
constexpr const char *name = "Bgra16Readback";
constexpr uintptr_t base = 0x0000000204000000ull;
constexpr uint64_t allocation_size = 0x200000;
constexpr auto format = Prospero::BufferFormat::k16_16_16_16Float;
constexpr auto tile = Prospero::TileMode::kRenderTarget;
EnsureRuntimeContext();
int64_t direct_offset = -1;
Require(name, "direct allocation",
Libs::LibKernel::Memory::KernelAllocateDirectMemory(
0, Libs::LibKernel::Memory::KernelGetDirectMemorySize(),
allocation_size, allocation_size, 0, &direct_offset) == 0,
"BGRA16 direct-memory allocation failed");
void *mapped = reinterpret_cast<void *>(base);
Require(name, "direct mapping",
Libs::LibKernel::Memory::KernelMapDirectMemory(
&mapped, allocation_size, 0x3, 0x10, direct_offset,
allocation_size) == 0 &&
mapped == reinterpret_cast<void *>(base),
"BGRA16 fixed mapping failed");
RenderContext context(m_runtime_context);
auto &scheduler = context.GetCommandScheduler();
HW::Context registers{};
HW::UserConfig user_config{};
HW::Shader shaders{};
scheduler.Begin(registers, user_config, shaders);
{
auto &resources = context;
resources.MapMemory(base, allocation_size);
const uint32_t pitch = TileGetTexturePitch(format, 1, tile);
TileSizeAlign total{};
TileSizeOffset mip{};
TilePaddedSize padded{};
TileGetTextureSize(format, 1, 1, 1, tile, &total, &mip, &padded);
Require(name, "tiled layout",
total.size >= 8 && total.size <= allocation_size,
"BGRA16 tiled layout exceeds its guest allocation");
std::memset(mapped, 0x5a, total.size);
constexpr std::array<uint16_t, 4> initial{0x3c00u, 0x4000u, 0x4200u, 0x4400u};
std::memcpy(mapped, initial.data(), sizeof(initial));
ImageDesc desc{};
desc.type = BindingType::RenderTarget;
desc.info.data = {base, total.size};
desc.info.pixel_format = vk::Format::eR16G16B16A16Sfloat;
desc.info.guest_format = format;
desc.info.type = Prospero::ImageType::kColor2D;
desc.info.extent = {1, 1, 1};
desc.info.resources = {1, 1};
desc.info.pitch = pitch;
desc.info.bytes_per_block = 8;
desc.info.samples = 1;
desc.info.tile_mode = tile;
desc.info.bgra16 = true;
desc.info.UpdateSize();
desc.view_info.format = desc.info.pixel_format;
desc.view_info.type = vk::ImageViewType::e2D;
desc.view_info.aspect = vk::ImageAspectFlagBits::eColor;
desc.view_info.usage = vk::ImageUsageFlagBits::eColorAttachment;
auto &cache = resources.GetTextureCache();
const auto id = cache.FindImage(desc);
(void)cache.FindRenderTarget(id, desc);
Require(name, "tiled BGRA16 upload",
ReadCachedTexel(name, context, id) ==
std::vector<u32>{0x40004200u, 0x44003c00u},
"fused detiling lost the BGRA16 component transform");
auto &image = cache.GetImage(id);
image.Transit(vk::ImageLayout::eTransferDstOptimal,
vk::AccessFlagBits2::eTransferWrite, {},
scheduler.Current().Handle());
vk::ClearColorValue clear{};
clear.float32[0] = 1.0f;
clear.float32[1] = 2.0f;
clear.float32[2] = 3.0f;
clear.float32[3] = 4.0f;
const vk::ImageSubresourceRange range{vk::ImageAspectFlagBits::eColor, 0,
1, 0, 1};
scheduler.Current().Handle().clearColorImage(
image.backing.image, vk::ImageLayout::eTransferDstOptimal, clear,
range);
cache.MarkGpuWritten(id);
Require(name, "guest readback queue",
TextureCacheTestAccess::TryDownload(cache, id),
"tiled BGRA16 guest readback was rejected");
auto mirror = resources.GetBufferCache().ObtainBuffer(base, total.size,
false, true);
Require(name, "mirror owner", mirror.first != nullptr,
"tiled BGRA16 mirror has no BufferCache owner");
auto mirror_readback = CreateHostBuffer(
name, 8, vk::BufferUsageFlagBits::eTransferDst, {0, 0});
const vk::BufferCopy copy{mirror.second, 0, 8};
scheduler.Current().Handle().copyBuffer(mirror.first->Handle(),
mirror_readback.buffer, 1, &copy);
vk::BufferMemoryBarrier barrier{};
barrier.sType = vk::StructureType::eBufferMemoryBarrier;
barrier.srcAccessMask = vk::AccessFlagBits::eTransferWrite;
barrier.dstAccessMask = vk::AccessFlagBits::eHostRead;
barrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
barrier.buffer = mirror_readback.buffer;
barrier.size = mirror_readback.size;
scheduler.Current().Handle().pipelineBarrier(
vk::PipelineStageFlagBits::eTransfer,
vk::PipelineStageFlagBits::eHost, {}, 0, nullptr, 1, &barrier, 0,
nullptr);
scheduler.Finish();
scheduler.DrainPriorityOperations();
std::vector<uint8_t> backing(total.size);
Require(name, "guest backing",
Libs::LibKernel::Memory::TryReadBacking(base, backing.data(),
backing.size()),
"tiled BGRA16 guest backing is unreadable");
const std::array<uint16_t, 4> expected{0x4200u, 0x4000u, 0x3c00u,
0x4400u};
std::array<uint16_t, 4> observed{};
std::memcpy(observed.data(), backing.data(), sizeof(observed));
Require(name, "guest component order",
observed == expected &&
std::all_of(backing.begin() + sizeof(observed), backing.end(),
[](uint8_t value) { return value == 0x5a; }),
"tiled BGRA16 guest readback changed component order or padding");
Require(name, "mirror component order",
ReadBuffer(name, mirror_readback, 2) ==
std::vector<u32>{0x40004200u, 0x44003c00u},
"tiled BGRA16 Buffer mirror changed guest component order");
DestroyBuffer(&mirror_readback);
resources.UnmapMemory(base, allocation_size);
scheduler.Finish();
}
Require(name, "unmap",
Libs::LibKernel::Memory::KernelMunmap(base, allocation_size) == 0,
"BGRA16 fixed mapping release failed");
Require(name, "release",
Libs::LibKernel::Memory::KernelReleaseDirectMemory(
direct_offset, allocation_size) == 0,
"BGRA16 direct-memory release failed");
std::printf("[gpu] %-32s ok\n", name);
}
void CheckRenderExecutorColorDiscovery() {
constexpr const char *name = "RenderExecutorColorDiscovery";
constexpr uintptr_t base = 0x0000000203c00000ull;
constexpr uint32_t width = 512;
constexpr uint32_t height = 1;
constexpr uint32_t bytes_per_element = 4;
constexpr uint64_t allocation_alignment = 0x10000;
EnsureRuntimeContext();
const auto pitch =
TileGetRenderTargetPitch(width, bytes_per_element, 0);
TileSizeAlign layout{};
Require(name, "tile layout",
pitch != 0 && TileGetRenderTargetSize(
width, height, pitch, bytes_per_element, layout, 0),
"1D render-target layout is unavailable");
struct TargetCase {
uint32_t width, height, dimension, pitch;
uint64_t size;
Prospero::TileMode tile;
uint32_t levels = 1;
bool float16 = false;
};
const std::array cases{
TargetCase{width, height, 0, pitch, layout.size,
Prospero::TileMode::kRenderTarget},
// PPSA28068: the 1440x720 agreement target must alias its sampled texture.
TargetCase{1440, 720, 1, 1472, 0x40b000, Prospero::TileMode::kLinear},
// PPSA01416: linear RGBA16F chains place the smallest mip first.
TargetCase{128, 128, 1, 128, 0x2be00, Prospero::TileMode::kLinear, 7, true},
// PPSA03541: the single texel still occupies a padded 128x64 block.
TargetCase{1, 1, 1, 128, 0x10000, Prospero::TileMode::kRenderTarget, 1, true},
};
constexpr uint64_t allocation_size = 0x410000;
int64_t direct_offset = -1;
Require(name, "direct allocation",
Libs::LibKernel::Memory::KernelAllocateDirectMemory(
0, Libs::LibKernel::Memory::KernelGetDirectMemorySize(),
allocation_size, allocation_alignment, 0, &direct_offset) == 0,
"color direct-memory allocation failed");
void *mapped = reinterpret_cast<void *>(base);
Require(name, "direct mapping",
Libs::LibKernel::Memory::KernelMapDirectMemory(
&mapped, allocation_size, 0x3, 0x10, direct_offset,
allocation_alignment) == 0 &&
mapped == reinterpret_cast<void *>(base),
"color fixed mapping failed");
std::memset(mapped, 0, allocation_size);
for (const auto &target : cases) {
const auto width = target.width;
const auto height = target.height;
const bool is_1d = target.dimension == 0;
RenderContext context(m_runtime_context);
auto &scheduler = context.GetCommandScheduler();
HW::Context registers{};
HW::UserConfig user_config{};
HW::Shader shaders{};
registers.SetColorBase(0, {.addr = base});
registers.SetColorInfo(
0, {.format = target.float16 ? Prospero::ChannelLayout::k16_16_16_16
: Prospero::ChannelLayout::k8_8_8_8,
.channel_type = target.float16 ? Prospero::ChannelType::kFloat
: Prospero::ChannelType::kSrgb,
.channel_order = Prospero::ChannelOrder::kStandard});
registers.SetColorAttrib2(0, {.height = height - 1, .width = width - 1,
.num_mip_levels = target.levels - 1});
registers.SetColorAttrib3(0,
{.tile_mode = target.tile,
.dimension = target.dimension,
.metadata_pipe_aligned = true});
registers.SetRenderTargetMask(0x0f);
scheduler.Begin(registers, user_config, shaders);
auto &resources = context;
auto &texture_cache = resources.GetTextureCache();
auto &executor = context.GetRenderExecutor();
resources.MapMemory(base, allocation_size);
RenderColorInfo color{};
RenderExecutorTestAccess::ResolveRenderColorTarget(
executor, scheduler.Current(), color, 0);
const auto attachment =
texture_cache.FindRenderTarget(color.image_id, color.desc);
const auto &image = texture_cache.GetImage(color.image_id);
RenderDepthInfo no_depth{};
const auto rendering = RenderExecutorTestAccess::AcquireRenderTargets(
executor, scheduler.Current(), &color, 1, no_depth);
scheduler.Current().BeginRendering(rendering);
scheduler.Current().EndRendering();
Require(name, "captured color target",
color.image_id && attachment != nullptr &&
rendering.color_attachments[0].image_view != nullptr &&
color.desc.info.type == (is_1d ? Prospero::ImageType::kColor1D
: Prospero::ImageType::kColor2D) &&
color.desc.info.extent == vk::Extent3D{width, height, 1} &&
color.desc.info.resources == ImageSubresources{target.levels, 1} &&
color.desc.info.pitch == target.pitch &&
color.desc.info.data.size == target.size &&
color.desc.info.mip_layout[0].offset ==
(target.levels == 1 ? 0 : 0xbe00) &&
color.desc.info.mip_layout[0].size ==
(target.levels == 1 ? target.size : 0x20000) &&
color.desc.view_info.type == (is_1d ? vk::ImageViewType::e1D
: vk::ImageViewType::e2D) &&
image.backing.image_type == (is_1d ? vk::ImageType::e1D
: vk::ImageType::e2D) &&
rendering.width == width && rendering.height == height &&
rendering.num_layers == 1 &&
rendering.num_color_attachments == 1,
"color attachment lost its dimensions or padded guest layout");
if (width == 1 && height == 1) {
Require(name, "single-texel tiled layout",
color.desc.info.mip_layout[0].offset == 0 &&
color.desc.info.mip_layout[0].size == 0x10000 &&
color.desc.info.mip_layout[0].pitch == 128 &&
color.desc.info.mip_layout[0].height == 64,
"the render target lost the expected padded block height");
}
if (target.levels == 7) {
constexpr std::array offsets{0xbe00u, 0x3e00u, 0x1e00u, 0xe00u,
0x600u, 0x200u, 0u};
constexpr std::array pitches{128u, 64u, 32u, 32u, 32u, 32u, 32u};
for (uint32_t mip = 0; mip < target.levels; ++mip) {
const auto &layout = color.desc.info.mip_layout[mip];
Require(name, "linear mip chain", layout.offset == offsets[mip] &&
layout.pitch == pitches[mip] &&
layout.size == pitches[mip] * (height >> mip) * 8,
"linear target mip offsets or padded pitches differ from the expected layout");
}
}
{
vk::ClearValue clear{};
clear.color.float32 = std::array{1.0f, 0.0f, 1.0f, 1.0f};
TextureCacheTestAccess::ClearImage(
texture_cache, scheduler.Current(), color.image_id,
{vk::ImageAspectFlagBits::eColor, 0, target.levels, 0, 1}, clear);
RenderExecutorTestAccess::ResetBindings(executor);
ShaderTextureResource descriptor{{
static_cast<uint32_t>(base >> 8u),
(static_cast<uint32_t>(target.float16
? Prospero::BufferFormat::k16_16_16_16Float
: Prospero::BufferFormat::k8_8_8_8Srgb) << 20u) |
(((width - 1u) & 3u) << 30u),
((width - 1u) >> 2u) | ((height - 1u) << 14u),
DstSel(4, 5, 6, 7) | ((target.levels - 1) << 16u) |
(static_cast<uint32_t>(target.tile) << 20u) |
(static_cast<uint32_t>(is_1d ? Prospero::ImageType::kColor1D
: Prospero::ImageType::kColor2D) << 28u),
0, 0x00700000u | ((target.levels - 1) << 4u), 0, 0}};
ShaderRecompiler::IR::DescriptorValue value{};
value.dword_count = 8;
std::copy_n(descriptor.fields, 8, value.dwords.begin());
ShaderRecompiler::IR::ImageResource resource{};
resource.resource_class = ShaderRecompiler::IR::ImageResourceClass::Sampled;
resource.numeric_class = Prospero::TextureNumericClass::Float;
resource.dimension = is_1d ? ShaderRecompiler::Decoder::ImageDimension::Dim1D
: ShaderRecompiler::Decoder::ImageDimension::Dim2D;
resource.read = true;
const auto sampled =
RenderExecutorTestAccess::ResolveTexture(executor, resource, value);
Require(name, "target sampled alias",
sampled.image_id == color.image_id &&
sampled.desc.info.pitch == target.pitch &&
sampled.desc.info.data.size == target.size &&
sampled.desc.info.mip_layout == color.desc.info.mip_layout &&
texture_cache.FindTexture(sampled.image_id, sampled.desc) != nullptr,
"sampling the target changed its padded layout or created a second image");
const auto expected = target.float16
? std::vector<u32>{0x00003c00u, 0x3c003c00u}
: std::vector<u32>{0xffff00ffu};
for (uint32_t mip = 0; mip < target.levels; ++mip) {
Require(name, "target sampled contents",
ReadCachedTexel(name, context, sampled.image_id,
{static_cast<int32_t>((width >> mip) - 1),
static_cast<int32_t>((height >> mip) - 1), 0},
{1, 1, 1}, 0, mip) == expected,
"the sampled target lost its GPU-written final texel");
}
if (width == 1 && height == 1) {
for (const bool uint_view : {true, false}) {
descriptor.fields[1] = static_cast<uint32_t>(
uint_view ? Prospero::BufferFormat::k16_16_16_16UInt
: Prospero::BufferFormat::k16_16_16_16Float) << 20u;
descriptor.fields[3] = 0x81b00facu; // Captured 1D view, tile27, RGBA swizzle.
std::copy_n(descriptor.fields, 8, value.dwords.begin());
resource.numeric_class = uint_view ? Prospero::TextureNumericClass::Uint
: Prospero::TextureNumericClass::Float;
resource.dimension = ShaderRecompiler::Decoder::ImageDimension::Dim1D;
const auto one_d =
RenderExecutorTestAccess::ResolveTexture(executor, resource, value);
const auto one_d_view = texture_cache.FindTexture(one_d.image_id, one_d.desc);
const auto repeated =
RenderExecutorTestAccess::ResolveTexture(executor, resource, value);
Require(name, "1D sampled backing and reuse",
one_d.image_id != color.image_id && one_d_view != nullptr &&
one_d.desc.view_info.type == vk::ImageViewType::e1D &&
texture_cache.GetImage(one_d.image_id).backing.image_type ==
vk::ImageType::e1D &&
repeated.image_id == one_d.image_id &&
texture_cache.FindTexture(repeated.image_id, repeated.desc) == one_d_view &&
ReadCachedTexel(name, context, one_d.image_id) == expected,
"the 1D alias reused an incompatible backing or lost the rendered texel");
RenderExecutorTestAccess::ResetBindings(executor);
RenderExecutorTestAccess::ResolveRenderColorTarget(
executor, scheduler.Current(), color, 0);
const auto restored = RenderExecutorTestAccess::AcquireRenderTargets(
executor, scheduler.Current(), &color, 1, no_depth);
scheduler.Current().BeginRendering(restored);
scheduler.Current().EndRendering();
Require(name, "2D target restoration",
color.image_id != one_d.image_id &&
restored.color_attachments[0].image_view != nullptr &&
texture_cache.GetImage(color.image_id).backing.image_type ==
vk::ImageType::e2D &&
texture_cache.GetImage(color.image_id).info.pixel_format ==
vk::Format::eR16G16B16A16Sfloat &&
ReadCachedTexel(name, context, color.image_id) == expected,
"restoring the 2D target lost its view type or GPU-written texel");
RenderExecutorTestAccess::ResetBindings(executor);
}
}
}
RenderExecutorTestAccess::ResetBindings(executor);
resources.UnmapMemory(base, allocation_size);
scheduler.Finish();
}
Require(name, "unmap direct backing",
Libs::LibKernel::Memory::KernelMunmap(base, allocation_size) == 0,
"color direct mapping release failed");
Require(name, "release direct backing",
Libs::LibKernel::Memory::KernelReleaseDirectMemory(
direct_offset, allocation_size) == 0,
"color direct-memory allocation release failed");
std::printf("[gpu] %-32s ok\n", name);
}
void CheckRenderExecutorColor1DArrayDiscovery() {
constexpr const char *name = "RenderExecutorColor1DArrayDiscovery";
constexpr uintptr_t base = 0x0000000203a00000ull;
constexpr uint32_t width = 512;
constexpr uint32_t height = 1;
constexpr uint32_t bytes_per_element = 4;
constexpr uint32_t first_slice = 2;
constexpr uint32_t last_slice = 5;
constexpr uint32_t image_layers = last_slice + 1;
constexpr uint32_t view_layers = last_slice - first_slice + 1;
constexpr uint64_t allocation_alignment = 0x10000;
EnsureRuntimeContext();
const auto pitch = TileGetRenderTargetPitch(width, bytes_per_element, 0);
TileSizeAlign layout{};
Require(name, "tile layout",
pitch == width &&
TileGetRenderTargetSize(width, height, pitch,
bytes_per_element, layout, 0) &&
layout.size == 0x40000 &&
layout.align == allocation_alignment,
"1D-array render-target layout is unavailable");
const auto allocation_size = layout.size * image_layers;
int64_t direct_offset = -1;
Require(name, "direct allocation",
Libs::LibKernel::Memory::KernelAllocateDirectMemory(
0, Libs::LibKernel::Memory::KernelGetDirectMemorySize(),
allocation_size, allocation_alignment, 0, &direct_offset) == 0,
"1D-array color direct-memory allocation failed");
void *mapped = reinterpret_cast<void *>(base);
Require(name, "direct mapping",
Libs::LibKernel::Memory::KernelMapDirectMemory(
&mapped, allocation_size, 0x3, 0x10, direct_offset,
allocation_alignment) == 0 &&
mapped == reinterpret_cast<void *>(base),
"1D-array color fixed mapping failed");
std::memset(mapped, 0, allocation_size);
{
RenderContext context(m_runtime_context);
auto &scheduler = context.GetCommandScheduler();
HW::Context registers{};
HW::UserConfig user_config{};
HW::Shader shaders{};
registers.SetColorBase(0, {.addr = base});
registers.SetColorInfo(
0, {.format = Prospero::ChannelLayout::k8_8_8_8,
.channel_type = Prospero::ChannelType::kSrgb,
.channel_order = Prospero::ChannelOrder::kStandard});
registers.SetColorView(
0, {.base_array_slice_index = first_slice,
.last_array_slice_index = last_slice});
registers.SetColorAttrib2(0, {.height = height - 1, .width = width - 1});
registers.SetColorAttrib3(0,
{.tile_mode = Prospero::TileMode::kRenderTarget,
.dimension = 0,
.metadata_pipe_aligned = true});
registers.SetRenderTargetMask(0x0f);
scheduler.Begin(registers, user_config, shaders);
auto &resources = context;
auto &texture_cache = resources.GetTextureCache();
auto &executor = context.GetRenderExecutor();
resources.MapMemory(base, allocation_size);
RenderColorInfo color{};
RenderExecutorTestAccess::ResolveRenderColorTarget(
executor, scheduler.Current(), color, 0);
const auto attachment =
texture_cache.FindRenderTarget(color.image_id, color.desc);
const auto &image = texture_cache.GetImage(color.image_id);
RenderDepthInfo no_depth{};
const auto rendering = RenderExecutorTestAccess::AcquireRenderTargets(
executor, scheduler.Current(), &color, 1, no_depth);
scheduler.Current().BeginRendering(rendering);
scheduler.Current().EndRendering();
Require(
name, "captured 1D-array target",
color.image_id && attachment != nullptr &&
rendering.color_attachments[0].image_view == attachment &&
color.guest_array_layer == first_slice &&
color.desc.info.type == Prospero::ImageType::kColor1D &&
color.desc.info.extent == vk::Extent3D{width, height, 1} &&
color.desc.info.resources == ImageSubresources{1, image_layers} &&
color.desc.info.pitch == pitch &&
color.desc.info.data.size == allocation_size &&
color.desc.info.mip_layout[0].offset == 0 &&
color.desc.info.mip_layout[0].size == allocation_size &&
color.desc.view_info.type == vk::ImageViewType::e1DArray &&
color.desc.view_info.base_layer == first_slice &&
color.desc.view_info.layer_count == view_layers &&
image.backing.image_type == vk::ImageType::e1D &&
image.backing.layers == image_layers &&
rendering.width == width && rendering.height == height &&
rendering.num_layers == view_layers &&
rendering.num_color_attachments == 1,
"dimension=0 slice range did not create a Vulkan 1D-array "
"attachment view");
RenderExecutorTestAccess::ResetBindings(executor);
resources.UnmapMemory(base, allocation_size);
scheduler.Finish();
}
Require(name, "unmap direct backing",
Libs::LibKernel::Memory::KernelMunmap(base, allocation_size) == 0,
"1D-array color direct mapping release failed");
Require(name, "release direct backing",
Libs::LibKernel::Memory::KernelReleaseDirectMemory(
direct_offset, allocation_size) == 0,
"1D-array color direct-memory allocation release failed");
std::printf("[gpu] %-32s ok\n", name);
}
void CheckRenderExecutorColorVolumeDiscovery() {
constexpr const char *name = "RenderExecutorColorVolumeDiscovery";
constexpr uintptr_t base = 0x0000000203e00000ull;
constexpr uint64_t slice_size = 0x10000;
constexpr uint64_t color_size = 0x200000;
constexpr uint64_t metadata_size = 0x20000;
constexpr uint64_t allocation_size = 56 * slice_size;
constexpr uint64_t allocation_alignment = 0x10000;
EnsureRuntimeContext();
int64_t direct_offset = -1;
Require(name, "direct allocation",
Libs::LibKernel::Memory::KernelAllocateDirectMemory(
0, Libs::LibKernel::Memory::KernelGetDirectMemorySize(),
allocation_size, allocation_alignment, 0, &direct_offset) == 0,
"color-volume direct-memory allocation failed");
void *mapped = reinterpret_cast<void *>(base);
Require(name, "direct mapping",
Libs::LibKernel::Memory::KernelMapDirectMemory(
&mapped, allocation_size, 0x3, 0x10, direct_offset,
allocation_alignment) == 0 &&
mapped == reinterpret_cast<void *>(base),
"color-volume fixed mapping failed");
std::memset(mapped, 0x5a, allocation_size);
{
RenderContext context(m_runtime_context);
context.InitializeGpu(nullptr);
LibKernel::Memory::InstallGpuResources(&context);
auto &scheduler = context.GetCommandScheduler();
HW::Context registers{};
HW::UserConfig user_config{};
HW::Shader shaders{};
registers.SetColorBase(0, {.addr = base});
registers.SetColorInfo(
0, {.dcc_compression_enable = true,
.format = Prospero::ChannelLayout::k10_10_10_2,
.channel_type = Prospero::ChannelType::kUNorm,
.channel_order = Prospero::ChannelOrder::kStandard});
registers.SetColorAttrib2(0, {.height = 31, .width = 31});
registers.SetColorAttrib3(0,
{.depth = 31,
.tile_mode = Prospero::TileMode::kRenderTarget,
.dimension = 2,
.metadata_pipe_aligned = true});
constexpr uint64_t dcc_address = base + color_size;
registers.SetColorDccAddr(0, {.addr = dcc_address});
registers.SetColorClearWord0(0, {.word0 = 0});
registers.SetColorClearWord1(0, {.word1 = 0});
registers.SetRenderTargetMask(0x0f);
scheduler.Begin(registers, user_config, shaders);
auto &resources = context;
auto &texture_cache = resources.GetTextureCache();
auto &executor = context.GetRenderExecutor();
resources.MapMemory(base, allocation_size);
WriteMetadata(context, dcc_address, metadata_size, 0);
Require(name, "deferred DCC clear",
!texture_cache.IsMeta(dcc_address),
"a pre-registration DCC clear was not deferred");
RenderColorInfo color{};
RenderExecutorTestAccess::ResolveRenderColorTarget(
executor, scheduler.Current(), color, 0);
const auto attachment =
texture_cache.FindRenderTarget(color.image_id, color.desc);
const auto &image = texture_cache.GetImage(color.image_id);
Require(
name, "captured 3D target",
color.image_id && attachment != nullptr &&
color.desc.info.type == Prospero::ImageType::kColor3D &&
color.desc.info.extent == vk::Extent3D{32, 32, 32} &&
color.desc.info.resources == ImageSubresources{1, 1} &&
color.desc.info.pitch == 128 &&
color.desc.info.data.size == color_size &&
color.desc.info.mip_layout[0].size == 0x10000 &&
color.desc.info.metadata.kind == ImageMetadataKind::Dcc &&
color.desc.info.metadata.range.address == dcc_address &&
!texture_cache.IsMeta(dcc_address) && !texture_cache.ClearMeta(dcc_address) &&
color.desc.view_info.type == vk::ImageViewType::e2D &&
color.desc.view_info.layer_count == 1 &&
image.backing.image_type == vk::ImageType::e3D &&
static_cast<bool>(image.backing.flags &
vk::ImageCreateFlagBits::e2DArrayCompatible) &&
image.usage.render_target && image.IsGpuModified(),
"dimension=2/depth=31 did not create the expected 32x32x32 "
"backing and 2D "
"attachment slice");
constexpr uint64_t metadata_slice_size = metadata_size / 32;
std::vector<uint8_t> metadata_bytes(metadata_size);
resources.GetBufferCache().ReadMemory(dcc_address, metadata_size);
Require(name, "first volume slice expanded metadata",
LibKernel::Memory::TryReadBacking(dcc_address, metadata_bytes.data(),
metadata_bytes.size()) &&
std::all_of(metadata_bytes.begin(),
metadata_bytes.begin() + metadata_slice_size,
[](uint8_t byte) { return byte == 0xff; }) &&
std::all_of(metadata_bytes.begin() + metadata_slice_size,
metadata_bytes.end(), [](uint8_t byte) { return byte == 0; }),
"consuming volume slice 0 changed unrequested native metadata slices");
RenderExecutorTestAccess::ResetBindings(executor);
registers.SetColorView(
0, {.base_array_slice_index = 7, .last_array_slice_index = 7});
RenderColorInfo sliced_color{};
RenderExecutorTestAccess::ResolveRenderColorTarget(
executor, scheduler.Current(), sliced_color, 0);
RenderDepthInfo no_depth{};
const auto sliced_rendering =
RenderExecutorTestAccess::AcquireRenderTargets(
executor, scheduler.Current(), &sliced_color, 1, no_depth);
Require(
name, "3D slice transition",
sliced_color.image_id == color.image_id &&
sliced_rendering.color_attachments[0].image_view != nullptr &&
sliced_color.desc.view_info.base_layer == 7 &&
sliced_rendering.num_color_attachments == 1 &&
sliced_rendering.num_layers == 1,
"a nonzero 3D attachment slice was treated as a Vulkan array layer");
resources.GetBufferCache().ReadMemory(dcc_address, metadata_size);
Require(name, "mixed volume metadata readback",
LibKernel::Memory::TryReadBacking(dcc_address, metadata_bytes.data(),
metadata_bytes.size()),
"mixed volume metadata is unavailable");
for (uint32_t slice = 0; slice < 32; slice++) {
const uint8_t expected = slice == 0 || slice == 7 ? 0xff : 0;
const auto first = metadata_bytes.begin() + slice * metadata_slice_size;
Require(name, "independent volume metadata consumption",
std::all_of(first, first + metadata_slice_size,
[expected](uint8_t byte) { return byte == expected; }),
"mixed DCC metadata lost a pending slice or consumed an unrequested slice");
}
Require(name, "3D slice clear and preservation",
ReadCachedTexel(name, context, color.image_id, {0, 0, 7}) ==
std::vector<u32>{0} &&
ReadCachedTexel(name, context, color.image_id, {0, 0, 31}) ==
std::vector<u32>{0x5a5a5a5a},
"clearing one volume slice changed untouched slice contents");
auto storage_desc = color.desc;
storage_desc.type = BindingType::Storage;
storage_desc.info.metadata = {};
storage_desc.info.guest_format = Prospero::BufferFormat::k10_10_10_2UNorm;
storage_desc.view_info.type = vk::ImageViewType::e3D;
storage_desc.view_info.usage = vk::ImageUsageFlagBits::eStorage;
const auto storage_id = texture_cache.FindImage(storage_desc);
const auto storage_view =
texture_cache.FindTexture(storage_id, storage_desc);
const auto &shared_image = texture_cache.GetImage(storage_id);
Require(name, "storage alias reuse",
storage_id == color.image_id && storage_view != nullptr &&
shared_image.IsGpuModified() &&
shared_image.usage.render_target,
"the matching 3D storage binding did not reuse the live "
"render-target image");
Require(
name, "volume readback queue",
TextureCacheTestAccess::TryDownload(texture_cache, storage_id),
"the 3D render target could not be queued for guest-layout readback");
auto mirror = resources.GetBufferCache().ObtainBuffer(
base, color_size, false, true);
Require(name, "volume mirror", mirror.first != nullptr,
"the 3D render-target readback has no BufferCache owner");
auto slice_probe =
CreateHostBuffer(name, 4, vk::BufferUsageFlagBits::eTransferDst, {0});
const vk::BufferCopy copy{mirror.second + 31 * slice_size, 0, 4};
scheduler.Current().Handle().copyBuffer(mirror.first->Handle(),
slice_probe.buffer, 1, &copy);
vk::BufferMemoryBarrier barrier{};
barrier.sType = vk::StructureType::eBufferMemoryBarrier;
barrier.srcAccessMask = vk::AccessFlagBits::eTransferWrite;
barrier.dstAccessMask = vk::AccessFlagBits::eHostRead;
barrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
barrier.buffer = slice_probe.buffer;
barrier.size = slice_probe.size;
scheduler.Current().Handle().pipelineBarrier(
vk::PipelineStageFlagBits::eTransfer,
vk::PipelineStageFlagBits::eHost, {}, 0, nullptr, 1, &barrier, 0,
nullptr);
scheduler.Finish();
scheduler.DrainPriorityOperations();
Require(name, "volume Z transfer",
ReadBuffer(name, slice_probe, 1) == std::vector<u32>{0x5a5a5a5a},
"render-target upload/readback lost the final Z slice");
DestroyBuffer(&slice_probe);
vk::ClearValue painted{};
painted.color.float32 = std::array{1.0f, 1.0f, 1.0f, 0.0f};
TextureCacheTestAccess::ClearImage(
texture_cache, scheduler.Current(), color.image_id,
{vk::ImageAspectFlagBits::eColor, 0, 1, 0, 1}, painted);
WriteMetadata(context, dcc_address, metadata_size, 0xffffffffu);
color.image_id = texture_cache.FindImage(color.desc);
(void)texture_cache.FindRenderTarget(color.image_id, color.desc);
resources.GetBufferCache().ReadMemory(dcc_address, metadata_size);
Require(name, "expanded volume metadata preserves rendering",
ReadCachedTexel(name, context, color.image_id) ==
std::vector<u32>{0x3fffffffu} &&
LibKernel::Memory::TryReadBacking(dcc_address, metadata_bytes.data(),
metadata_bytes.size()) &&
std::all_of(metadata_bytes.begin(), metadata_bytes.end(),
[](uint8_t byte) { return byte == 0xff; }),
"uncompressed native keys reapplied an old volume clear");
WriteMetadata(context, dcc_address, metadata_size, 0);
color.image_id = texture_cache.FindImage(color.desc);
(void)texture_cache.FindRenderTarget(color.image_id, color.desc);
resources.GetBufferCache().ReadMemory(dcc_address, metadata_size);
Require(name, "native volume re-clear",
ReadCachedTexel(name, context, color.image_id) == std::vector<u32>{0} &&
LibKernel::Memory::TryReadBacking(dcc_address, metadata_bytes.data(),
metadata_bytes.size()) &&
std::all_of(metadata_bytes.begin(),
metadata_bytes.begin() + metadata_slice_size,
[](uint8_t byte) { return byte == 0xff; }) &&
std::all_of(metadata_bytes.begin() + metadata_slice_size,
metadata_bytes.end(), [](uint8_t byte) { return byte == 0; }),
"a native metadata overwrite did not clear only its requested volume slice");
for (const uint32_t base_slice : {0u, 7u}) {
RenderExecutorTestAccess::ResetBindings(executor);
registers.SetColorView(0, {.base_array_slice_index = base_slice,
.last_array_slice_index = 32});
RenderColorInfo volume_color{};
RenderExecutorTestAccess::ResolveRenderColorTarget(
executor, scheduler.Current(), volume_color, 0);
const auto volume_rendering =
RenderExecutorTestAccess::AcquireRenderTargets(
executor, scheduler.Current(), &volume_color, 1, no_depth);
Require(name, "3D export bound exceeds depth",
volume_color.image_id == color.image_id &&
volume_color.desc.info.extent == vk::Extent3D{32, 32, 32} &&
volume_color.desc.info.data.size == color_size &&
volume_color.desc.view_info.type == vk::ImageViewType::e2DArray &&
volume_color.desc.view_info.base_layer == base_slice &&
volume_color.desc.view_info.layer_count == 32 - base_slice &&
volume_rendering.num_layers == 32 - base_slice &&
registers.GetRenderTarget(0).view.last_array_slice_index == 32,
"3D export bounds changed the backing or exceeded its attachment depth");
scheduler.Current().BeginRendering(volume_rendering);
scheduler.Current().EndRendering();
}
resources.GetBufferCache().ReadMemory(dcc_address, metadata_size);
Require(name, "complete volume expansion",
LibKernel::Memory::TryReadBacking(dcc_address, metadata_bytes.data(),
metadata_bytes.size()) &&
std::all_of(metadata_bytes.begin(), metadata_bytes.end(),
[](uint8_t byte) { return byte == 0xff; }),
"full volume acquisition left an unconsumed native clear slice");
TextureCacheTestAccess::ClearImage(
texture_cache, scheduler.Current(), color.image_id,
{vk::ImageAspectFlagBits::eColor, 0, 1, 31, 1}, painted);
auto expanded = storage_desc;
expanded.info.extent.depth = 56;
expanded.info.data.size = allocation_size;
expanded.info.UpdateSize();
const auto expanded_id = texture_cache.FindImage(expanded);
Require(name, "3D storage depth expansion",
expanded_id != color.image_id &&
!TextureCacheTestAccess::Contains(texture_cache, color.image_id) &&
texture_cache.GetImage(expanded_id).backing.extent.depth == 56 &&
texture_cache.GetImage(expanded_id).IsGpuModified(),
"a deeper storage binding reused the smaller native volume");
Require(name, "3D expansion preserves native prefix and guest suffix",
ReadCachedTexel(name, context, expanded_id, {0, 0, 31}) ==
std::vector<u32>{0x3fffffffu} &&
ReadCachedTexel(name, context, expanded_id, {0, 0, 55}) ==
std::vector<u32>{0x5a5a5a5a},
"volume expansion lost GPU writes or copied beyond the old depth");
RenderExecutorTestAccess::ResetBindings(executor);
resources.UnmapMemory(base, allocation_size);
scheduler.Finish();
context.ShutdownGpu();
LibKernel::Memory::InstallGpuResources(nullptr);
}
Require(name, "unmap direct backing",
Libs::LibKernel::Memory::KernelMunmap(base, allocation_size) == 0,
"color-volume direct mapping release failed");
Require(name, "release direct backing",
Libs::LibKernel::Memory::KernelReleaseDirectMemory(
direct_offset, allocation_size) == 0,
"color-volume direct-memory allocation release failed");
std::printf("[gpu] %-32s ok\n", name);
}
static void WriteMetadata(RenderContext &context, uint64_t address,
uint64_t size, uint32_t value) {
Require("DccMetadata", "guest write range", size % sizeof(value) == 0 &&
context.InvalidateMemory(address, size),
"metadata write is outside the mapped allocation");
const std::vector<uint32_t> words(size / sizeof(value), value);
LibKernel::Memory::WriteBacking(address, words.data(), size);
}
static TextureCache::ImageDesc MakeLinearDesc(
uint64_t address, uint64_t size, vk::Format format,
Prospero::BufferFormat guest_format, Prospero::ImageType type,
vk::Extent3D extent, uint32_t layers, uint32_t bytes_per_block,
uint32_t samples) {
TextureCache::ImageDesc desc{};
desc.type = TextureCache::BindingType::Texture;
desc.info.data = {address, size};
desc.info.pixel_format = format;
desc.info.guest_format = guest_format;
desc.info.type = type;
desc.info.extent = extent;
desc.info.resources = {1, layers};
desc.info.pitch = extent.width;
desc.info.bytes_per_block = bytes_per_block;
desc.info.samples = samples;
desc.info.tile_mode = Prospero::TileMode::kLinear;
desc.info.mip_layout[0] = {0, size, extent.width, extent.height};
desc.info.tiling.texel_width = desc.info.IsBlock() ? 4u : 1u;
desc.info.tiling.texel_height = desc.info.IsBlock() ? 4u : 1u;
desc.info.tiling.block.bytes_per_element = bytes_per_block;
desc.info.linear_slice_stride =
size / (type == Prospero::ImageType::kColor3D ? extent.depth : layers);
desc.info.mip_layout[0].pitch =
(extent.width + desc.info.tiling.texel_width - 1u) /
desc.info.tiling.texel_width;
desc.info.mip_layout[0].height =
(extent.height + desc.info.tiling.texel_height - 1u) /
desc.info.tiling.texel_height;
desc.info.mip_layout[0].linear_pitch = desc.info.mip_layout[0].pitch;
desc.info.mip_layout[0].linear_size = desc.info.linear_slice_stride;
desc.view_info.format = format;
desc.view_info.type = type == Prospero::ImageType::kColor3D
? vk::ImageViewType::e3D
: layers > 1 ? vk::ImageViewType::e2DArray
: vk::ImageViewType::e2D;
desc.view_info.aspect = vk::ImageAspectFlagBits::eColor;
desc.view_info.layer_count = layers;
desc.view_info.usage = vk::ImageUsageFlagBits::eSampled;
return desc;
}
std::vector<u32> ReadCachedTexel(const char *name, RenderContext &context,
ImageId id, vk::Offset3D offset = {},
vk::Extent3D extent = {1, 1, 1}, uint32_t layer = 0,
uint32_t mip = 0) {
auto &scheduler = context.GetCommandScheduler();
auto &image = context.GetTextureCache().GetImage(id);
const auto bytes = image.info.bytes_per_block * extent.width *
extent.height * extent.depth;
auto probe = CreateHostBuffer(name, bytes,
vk::BufferUsageFlagBits::eTransferDst, {});
scheduler.Current().EndRendering();
image.Transit(vk::ImageLayout::eTransferSrcOptimal,
vk::AccessFlagBits2::eTransferRead, {},
scheduler.Current().Handle());
vk::BufferImageCopy copy{};
copy.imageSubresource = {image.info.IsDepth() ? vk::ImageAspectFlagBits::eDepth
: vk::ImageAspectFlagBits::eColor, mip, layer, 1};
copy.imageOffset = offset;
copy.imageExtent = extent;
scheduler.Current().Handle().copyImageToBuffer(
image.backing.image, vk::ImageLayout::eTransferSrcOptimal,
probe.buffer, 1, &copy);
vk::BufferMemoryBarrier barrier{};
barrier.srcAccessMask = vk::AccessFlagBits::eTransferWrite;
barrier.dstAccessMask = vk::AccessFlagBits::eHostRead;
barrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
barrier.buffer = probe.buffer;
barrier.size = probe.size;
scheduler.Current().Handle().pipelineBarrier(
vk::PipelineStageFlagBits::eTransfer,
vk::PipelineStageFlagBits::eHost, {}, 0, nullptr, 1, &barrier, 0,
nullptr);
scheduler.Finish();
auto result = ReadBuffer(name, probe, bytes / 4);
DestroyBuffer(&probe);
return result;
}
void CheckNullTextureDescriptors() {
constexpr const char *name = "NullTextureDescriptors";
constexpr uintptr_t base = 0x0000000204900000ull;
constexpr uint64_t size = 0x10000;
std::vector<u32> reader{
EncodeVop2(0x1a, 20, InlineU32(5), 0),
EncodeVop1(0x01, 21, InlineU32(0)),
EncodeVop1(0x01, 22, InlineU32(0)),
EncodeMimg0(0x00, 0xf), EncodeMimg1(1, 21)};
AppendVMovLiteral(&reader, 23, std::bit_cast<u32>(0.5f));
AppendVMovLiteral(&reader, 24, std::bit_cast<u32>(0.5f));
reader.insert(reader.end(), {
EncodeMimg0(0x27, 0xf), EncodeMimg1(5, 23, 0, 2),
0xbf8c3f70u,
EncodeMubuf0(0x1e), EncodeMubuf1(1, 3, 20),
EncodeMubuf0(0x1e, 16), EncodeMubuf1(5, 3, 20)});
AppendEnd(&reader);
std::vector<u32> writer{
EncodeVopc(0xd2, InlineU32(0), 0),
EncodeVop1(0x01, 21, InlineU32(0)),
EncodeVop1(0x01, 22, InlineU32(0))};
for (u32 i = 1; i <= 4; ++i)
AppendVMovLiteral(&writer, i, std::bit_cast<u32>(-1000000.0f));
writer.insert(writer.end(), {EncodeMimg0(0x08, 0xf), EncodeMimg1(1, 21)});
AppendEnd(&writer);
for (const auto *code : {&reader, &writer})
ShaderMapUserData(reinterpret_cast<uint64_t>(code->data()),
{.type = Prospero::ShaderBinaryType::kCs,
.code_size_bytes = static_cast<u32>(code->size() * sizeof(u32))});
EnsureRuntimeContext();
int64_t physical = -1;
Require(name, "allocate", LibKernel::Memory::KernelAllocateDirectMemory(
0, LibKernel::Memory::KernelGetDirectMemorySize(), size, size, 0, &physical) == 0,
"null texture output allocation failed");
void *mapped = reinterpret_cast<void *>(base);
Require(name, "map", LibKernel::Memory::KernelMapDirectMemory(
&mapped, size, 0x3, 0x10, physical, size) == 0 && mapped == reinterpret_cast<void *>(base),
"null texture output mapping failed");
std::memset(mapped, 0xa5, size);
RenderContext context(m_runtime_context);
context.InitializeGpu(nullptr);
LibKernel::Memory::InstallGpuResources(&context);
context.GetGpu().SendCommandSync([&] {
CommandProcessor processor(context, 0);
processor.BufferInit();
context.MapMemory(base, size);
auto &shaders = processor.GetShCtx();
auto &executor = context.GetRenderExecutor();
for (u32 wave_size : {32u, 64u}) {
const auto output_address = [&](u32 phase) {
return base + (wave_size == 64 ? 0x8000u : 0u) + 0x100u + phase * 0x2000u;
};
for (u32 sgpr = 0; sgpr < 16; ++sgpr)
shaders.SetCsUserSgpr(sgpr, 0, HW::UserSgprType::Unknown);
shaders.SetCsUserSgpr(8, 0x92u, HW::UserSgprType::Unknown);
for (u32 phase = 0; phase < 3; ++phase) {
const auto &code = phase == 1 ? writer : reader;
shaders.SetCsShader({.data_addr = reinterpret_cast<uint64_t>(code.data()),
.num_thread_x = wave_size, .num_thread_y = 1, .num_thread_z = 1,
.wave_size = static_cast<uint8_t>(wave_size), .user_sgpr = 16});
ShaderBufferResource output{};
output.UpdateAddress48(output_address(phase));
output.fields[2] = wave_size * 8u * sizeof(u32);
output.fields[3] = DstSel(4, 5, 6, 7) |
(static_cast<u32>(Prospero::BufferFormat::k32UInt) << 12u);
for (u32 i = 0; i < 4; ++i)
shaders.SetCsUserSgpr(12 + i, output.fields[i], HW::UserSgprType::Unknown);
ShaderComputeInputInfo input{};
(void)context.GetPipelineCache().GetComputeProgram(
shaders.GetCs(), processor.GetCtx().GetShaderRegisters(), input);
PreparedBindings bindings;
executor.PrepareBindings(input.stage, bindings);
executor.RebindImages(bindings);
Require(name, "native null image bindings", !bindings.images.empty() &&
std::ranges::all_of(bindings.images, [](const auto &binding) {
return !binding.image_id && binding.image_view == nullptr &&
binding.mip_views.empty() &&
MakeImageInfo(binding).imageView == nullptr &&
binding.layout == vk::ImageLayout::eGeneral;
}), "an all-zero guest descriptor allocated or bound a backing image");
processor.DispatchDirect(1, 1, 1, 0x41u |
(wave_size == 32u ? 1u << Pm4::COMPUTE_DISPATCH_INITIATOR_CS_W32_EN_SHIFT : 0u));
}
for (u32 phase : {0u, 2u}) {
const auto address = output_address(phase);
std::vector<u32> actual(wave_size * 8u);
context.GetBufferCache().ReadMemory(address, actual.size() * sizeof(u32));
Require(name, "null RGBA before and after storage write",
LibKernel::Memory::TryReadBacking(address, actual.data(), actual.size() * sizeof(u32)) &&
std::ranges::all_of(actual, [](u32 value) { return value == 0; }),
"wave" + std::to_string(wave_size) + " phase" + std::to_string(phase) +
" returned nonzero load/sample components from an unbound texture");
}
}
context.UnmapMemory(base, size);
context.GetCommandScheduler().Finish();
});
LibKernel::Memory::InstallGpuResources(nullptr);
context.ShutdownGpu();
Require(name, "release", LibKernel::Memory::KernelCheckedReleaseDirectMemory(physical, size) == 0,
"null texture output release failed");
std::printf("[gpu] %-32s ok\n", name);
}
void CheckScalarLeafCall() {
constexpr const char* name = "ScalarLeafCall";
constexpr uintptr_t base = 0x0000000204800000ull;
constexpr uint64_t size = 0x10000;
constexpr auto table = base + 0x100;
constexpr auto first = base + 0x1000;
constexpr auto second = base + 0x2000;
constexpr auto output = base + 0x4000;
std::vector<u32> code{
EncodeSMovB32(20, InlineU32(0)),
0xbeea03ffu, static_cast<u32>(base), 0xbeeb03ffu, static_cast<u32>(base >> 32u),
0xf4080135u, 0xfa000000u, 0xbf8cc07fu, 0xf4240382u, 0xfa000020u,
0xbf8cc07fu, 0xbf130e80u, 0xbf840004u, 0xf4240402u, 0xfa000028u,
0xbf8cc07fu, 0xbe8e210eu, EncodeVop1(0x01, 1, 20),
EncodeMubuf0(0x1c, 0, false), EncodeMubuf1(1, 0, 0)};
AppendEnd(&code);
ShaderMapUserData(reinterpret_cast<uint64_t>(code.data()),
{.type = Prospero::ShaderBinaryType::kCs,
.code_size_bytes = static_cast<u32>(code.size() * sizeof(u32))});
EnsureRuntimeContext();
int64_t physical = -1;
Require(name, "allocate", LibKernel::Memory::KernelAllocateDirectMemory(
0, LibKernel::Memory::KernelGetDirectMemorySize(), size, size, 0, &physical) == 0,
"call fixture allocation failed");
void* mapped = reinterpret_cast<void*>(base);
Require(name, "map", LibKernel::Memory::KernelMapDirectMemory(
&mapped, size, 0x3, 0x10, physical, size) == 0 && mapped == reinterpret_cast<void*>(base),
"call fixture mapping failed");
std::memset(mapped, 0, size);
const ShaderBufferResource table_desc{{static_cast<u32>(table),
static_cast<u32>(table >> 32u) | (16u << 16u), 8, 0x5204u}};
LibKernel::Memory::WriteBacking(base, table_desc.fields, sizeof(table_desc.fields));
const std::array leaf{EncodeSMovB32(20, 255), 41u, EncodeSop1(0x21, 125, 14)};
for (const auto address : {first, second})
LibKernel::Memory::WriteBacking(address, leaf.data(), sizeof(leaf));
RenderContext context(m_runtime_context);
context.InitializeGpu(nullptr);
LibKernel::Memory::InstallGpuResources(&context);
context.GetGpu().SendCommandSync([&] {
CommandProcessor processor(context, 0);
processor.BufferInit();
context.MapMemory(base, size);
auto& cache = context.GetBufferCache();
auto& shaders = processor.GetShCtx();
shaders.SetCsShader({.data_addr = reinterpret_cast<uint64_t>(code.data()),
.num_thread_x = 1, .num_thread_y = 1, .num_thread_z = 1,
.wave_size = 64, .user_sgpr = 4});
ShaderBufferResource out{};
out.UpdateAddress48(output);
out.fields[2] = 4;
out.fields[3] = DstSel(4, 5, 6, 7) |
(static_cast<u32>(Prospero::BufferFormat::k32UInt) << 12u);
for (u32 i = 0; i < 4; ++i)
shaders.SetCsUserSgpr(i, out.fields[i], HW::UserSgprType::Unknown);
uint64_t original_id = 0;
struct Case { uint64_t address; u32 value; bool update; u32 field = 0; };
for (const auto test : {Case{first, 41, false}, Case{first, 41, false},
Case{second, 41, false}, Case{second, 73, true},
Case{first, 41, false}, Case{first, 6, false, 0x1818},
Case{first, 0, false, 0x0a18}}) {
Require(name, "refresh function pointer", context.InvalidateMemory(table + 0x20, 8),
"function table escaped mapped memory");
const uint64_t target = test.address | 3u;
LibKernel::Memory::WriteBacking(table + 0x20, &target, sizeof(target));
if (test.field != 0) {
Require(name, "refresh hardware-field leaf", context.InvalidateMemory(first, sizeof(leaf)),
"hardware-field leaf escaped mapped memory");
const std::array field_leaf{0xb96a0000u | test.field,
EncodeSMovB32(20, 106), EncodeSop1(0x21, 125, 14)};
LibKernel::Memory::WriteBacking(first, field_leaf.data(), sizeof(field_leaf));
}
if (test.update) {
auto [buffer, offset] = cache.ObtainBuffer(test.address + 4, 4, true);
buffer->Fill(offset, 4, test.value);
Require(name, "GPU-written function code", cache.HasGpuDirtyBytes(test.address + 4, 4),
"code mutation did not remain GPU-owned before refresh");
}
ShaderComputeInputInfo input{};
const auto program = context.GetPipelineCache().GetComputeProgram(
shaders.GetCs(), processor.GetCtx().GetShaderRegisters(), input);
if (original_id == 0) original_id = program.id;
Require(name, "exact function identity",
(program.id == original_id) == (test.value == 41),
"helper bytes were stale or identical relocated/restored code recompiled");
Require(name, "current code dependency",
std::ranges::find(input.stage.resources->specialization_reads,
std::pair<uint64_t, uint64_t>{test.address, sizeof(leaf)}) !=
input.stage.resources->specialization_reads.end(),
"function relocation retained an old code dependency");
processor.DispatchDirect(1, 1, 1, 0x41u);
cache.ReadMemory(output, sizeof(u32));
u32 actual = 0;
Require(name, "scalar leaf execution and return",
LibKernel::Memory::TryReadBacking(output, &actual, sizeof(actual)) && actual == test.value,
"caller did not observe the leaf's scalar result after return");
}
context.UnmapMemory(base, size);
context.GetCommandScheduler().Finish();
});
LibKernel::Memory::InstallGpuResources(nullptr);
context.ShutdownGpu();
Require(name, "release", LibKernel::Memory::KernelCheckedReleaseDirectMemory(physical, size) == 0,
"call fixture release failed");
std::printf("[gpu] %-32s ok\n", name);
}
void CheckNativeIndirectDispatch() {
constexpr const char *name = "NativeIndirectDispatch";
constexpr uintptr_t base = 0x0000000204600000ull;
constexpr uint64_t allocation_size = 0x100000;
constexpr uint64_t case_size = 0x10000;
constexpr uint32_t sentinel = 0xa5a5a5a5u;
struct DispatchCase {
std::array<uint32_t, 3> dimensions;
uint32_t mode;
uint32_t expected_threads;
bool transfer = false;
};
constexpr std::array cases{
DispatchCase{{2, 1, 1}, 0x41u, 8},
DispatchCase{{0, 1, 1}, 0x41u, 0},
DispatchCase{{1, 0, 1}, 0x41u, 0},
DispatchCase{{1, 1, 0}, 0x41u, 0},
DispatchCase{{3, 1, 1}, 0x41u, 12, true},
DispatchCase{{8, 1, 1}, 0x61u, 8},
};
// Separate DWORD descriptors preserve two owners until the indirect argument
// range spans them. The consumer's output aliases the second owner.
std::vector<u32> writer;
for (u32 i = 0; i < 3; i++) {
writer.push_back(EncodeVop1(0x01u, 1, 12u + i));
writer.push_back(EncodeMubuf0(0x1cu, 0, false, false));
writer.push_back(EncodeMubuf1(1, i, 0));
}
AppendEnd(&writer);
std::vector<u32> consumer;
AppendVop3(&consumer, 0x346u, 1, 4, InlineU32(2), Vgpr(0));
consumer.push_back(EncodeVop2(0x25u, 2, InlineU32(1), 1));
consumer.push_back(EncodeVop2(0x1au, 1, InlineU32(2), 1));
consumer.push_back(EncodeMubuf0(0x1cu));
consumer.push_back(EncodeMubuf1(2, 0, 1));
AppendEnd(&consumer);
std::vector<u32> counted_consumer{
EncodeMubuf0(0x0cu, 0, false, false), EncodeMubuf1(3, 1, 0),
EncodeSopp(0x0c, 0), EncodeVop1(0x02, 2, Vgpr(3))};
AppendVop3(&counted_consumer, 0x346u, 1, 8, InlineU32(2), Vgpr(0));
AppendVop3(&counted_consumer, 0x36du, 2, Vgpr(1), Vgpr(3), InlineU32(1));
counted_consumer.push_back(EncodeMubuf0(0x1cu, 0, true, false));
counted_consumer.push_back(EncodeMubuf1(2, 0, 1));
AppendEnd(&counted_consumer);
for (const auto *code : {&writer, &consumer, &counted_consumer}) {
ShaderMapUserData(reinterpret_cast<uint64_t>(code->data()),
{.type = Prospero::ShaderBinaryType::kCs,
.code_size_bytes = static_cast<uint32_t>(code->size() * sizeof(u32))});
}
EnsureRuntimeContext();
int64_t direct_offset = -1;
Require(name, "direct allocation",
Libs::LibKernel::Memory::KernelAllocateDirectMemory(
0, Libs::LibKernel::Memory::KernelGetDirectMemorySize(),
allocation_size, case_size, 0, &direct_offset) == 0,
"indirect argument allocation failed");
void *mapped = reinterpret_cast<void *>(base);
Require(name, "direct mapping",
Libs::LibKernel::Memory::KernelMapDirectMemory(
&mapped, allocation_size, 0x3, 0x10, direct_offset, case_size) == 0 &&
mapped == reinterpret_cast<void *>(base),
"indirect argument mapping failed");
std::memset(mapped, 0xa5, allocation_size);
const auto argument_address = [&](size_t index) {
return base + index * case_size +
(cases[index].transfer ? 0x200u : BufferCache::CACHING_PAGESIZE - 4u);
};
for (size_t i = 0; i < cases.size(); i++) {
std::memset(reinterpret_cast<void *>(argument_address(i)), 0, 12);
}
RenderContext context(m_runtime_context);
context.InitializeGpu(nullptr);
LibKernel::Memory::InstallGpuResources(&context);
context.GetGpu().SendCommandSync([&] {
CommandProcessor processor(context, 0);
processor.BufferInit();
auto &scheduler = context.GetCommandScheduler();
auto &cache = context.GetBufferCache();
auto &shaders = processor.GetShCtx();
context.MapMemory(base, allocation_size);
// Empty CPU-owned arguments must not inspect the shader, even on a GPU-dirty page.
constexpr auto empty_args = base + allocation_size - case_size;
constexpr auto neighbor = empty_args + 0x100u;
constexpr std::array<std::array<u32, 3>, 3> empty_dimensions{{
{0, 1, 1}, {1, 0, 1}, {1, 1, 0}}};
Require(name, "empty arguments",
LibKernel::Memory::TryWriteBacking(empty_args, empty_dimensions.data(),
sizeof(empty_dimensions)),
"empty indirect arguments could not be written");
shaders.SetCsShader({.data_addr = 1});
for (const bool dirty_neighbor : {false, true}) {
if (dirty_neighbor) {
auto [buffer, offset] = cache.ObtainBuffer(neighbor, sizeof(sentinel), true);
buffer->Fill(offset, sizeof(sentinel), sentinel + 1u);
}
Require(name, "exact argument ownership",
!cache.HasGpuDirtyBytes(empty_args, sizeof(empty_dimensions)) &&
cache.IsRegionGpuModified(empty_args, sizeof(empty_dimensions)) == dirty_neighbor,
"empty arguments did not retain independent byte ownership");
const auto tick = scheduler.CurrentTick();
for (size_t i = 0; i < empty_dimensions.size(); ++i) {
processor.DispatchIndirect(empty_args + i * sizeof(empty_dimensions[0]), 0x41u);
}
Require(name, "empty dispatch avoids readback",
scheduler.CurrentTick() == tick &&
cache.HasGpuDirtyBytes(neighbor, sizeof(sentinel)) == dirty_neighbor,
"empty dispatch submitted work or read back neighboring GPU data");
}
const auto set_buffer = [&](u32 sgpr, uint64_t address, u32 records, u32 stride = 0) {
ShaderBufferResource descriptor{};
descriptor.UpdateAddress48(address);
descriptor.fields[1] |= stride << 16u;
descriptor.fields[2] = records;
descriptor.fields[3] = DstSel(4, 5, 6, 7) |
(static_cast<u32>(Prospero::BufferFormat::k32UInt) << 12u);
for (u32 i = 0; i < 4; i++) {
shaders.SetCsUserSgpr(sgpr + i, descriptor.fields[i],
HW::UserSgprType::Unknown);
}
};
for (size_t index = 0; index < cases.size(); index++) {
const auto &test = cases[index];
const auto args = argument_address(index);
const auto output = base + index * case_size + BufferCache::CACHING_PAGESIZE + 0x100u;
if (test.transfer) {
for (u32 i = 0; i < 3; i++) {
auto [buffer, offset] = cache.ObtainBuffer(args + i * 4u, 4, true);
buffer->Fill(offset, 4, test.dimensions[i]);
}
} else {
shaders.SetCsShader({.data_addr = reinterpret_cast<uint64_t>(writer.data()),
.num_thread_x = 1, .num_thread_y = 1, .num_thread_z = 1,
.wave_size = 64, .user_sgpr = 15});
for (u32 i = 0; i < 3; i++) {
set_buffer(i * 4u, args + i * 4u, 4);
shaders.SetCsUserSgpr(12u + i, test.dimensions[i], HW::UserSgprType::Unknown);
}
processor.DispatchDirect(1, 1, 1, 0x41u);
}
std::array<u32, 3> stale{};
Require(name, "GPU-only argument write",
LibKernel::Memory::TryReadBacking(args, stale.data(), sizeof(stale)) &&
stale == std::array<u32, 3>{},
"argument publication changed the CPU backing before indirect dispatch");
const auto first_owner = BufferCacheTestAccess::PageOwner(cache, args);
const auto second_owner = BufferCacheTestAccess::PageOwner(cache, args + 8u);
if (!test.transfer) {
Require(name, "separate argument owners",
first_owner && second_owner && first_owner != second_owner,
"the indirect argument merge fixture already shared one owner");
}
shaders.SetCsShader({.data_addr = reinterpret_cast<uint64_t>(consumer.data()),
.num_thread_x = 4, .num_thread_y = 1, .num_thread_z = 1,
.wave_size = 64, .user_sgpr = 4, .tgid_x_en = true});
set_buffer(0, output, 16u * sizeof(u32));
const auto tick = scheduler.CurrentTick();
if (test.transfer) {
const std::array<u32, 3> packet{static_cast<u32>(args),
static_cast<u32>(args >> 32u), test.mode};
Require(name, "absolute indirect packet",
CpOpDispatchIndirect(processor, 0xc0021600u, packet.data(), 0, 0) == 3,
"the absolute indirect packet was not consumed");
} else {
processor.SetDispatchIndirectArgsBaseAddress(base + index * case_size);
const std::array<u32, 2> packet{
static_cast<u32>(BufferCache::CACHING_PAGESIZE - 4u), test.mode};
Require(name, "offset indirect packet",
CpOpDispatchIndirect(processor, 0xc0011600u, packet.data(), 0, 0) == 2,
"the offset indirect packet was not consumed");
}
if (test.mode == 0x41u) {
Require(name, "asynchronous indirect dispatch", scheduler.CurrentTick() == tick,
"workgroup-count arguments caused a host submission or readback");
if (!test.transfer) {
const auto merged = BufferCacheTestAccess::PageOwner(cache, args);
Require(name, "argument/output owner merge",
merged && merged != first_owner && merged != second_owner &&
merged == BufferCacheTestAccess::PageOwner(cache, output),
"argument discovery did not merge the consumer's buffer owner");
}
}
cache.ReadMemory(output, 16u * sizeof(u32));
std::array<u32, 16> actual{};
Require(name, "output readback",
LibKernel::Memory::TryReadBacking(output, actual.data(), sizeof(actual)),
"consumer output could not be read back");
for (u32 i = 0; i < actual.size(); i++) {
const auto expected = i < test.expected_threads ? i + 1u : sentinel;
Require(name, "indirect invocation coverage", actual[i] == expected,
"case " + std::to_string(index) + " word " + std::to_string(i) +
" expected " + Hex(expected) + ", got " + Hex(actual[i]));
}
}
// The same GPU-written DWORD supplies indirect X and the output descriptor's
// native NUM_RECORDS. Four lanes per group exercise the descriptor's OOB bound.
constexpr auto image_address = base + cases.size() * case_size;
constexpr auto count_args = image_address + 4u * sizeof(u32);
constexpr auto count_output = image_address + 2u * BufferCache::CACHING_PAGESIZE;
constexpr u32 image_value = 0x13579bdfu;
auto &textures = context.GetTextureCache();
auto image_desc = MakeLinearDesc(
image_address, 8u * sizeof(u32), vk::Format::eR32Uint,
Prospero::BufferFormat::k32UInt, Prospero::ImageType::kColor2D,
{8, 1, 1}, 1, 4, 1);
const auto image = textures.FindImage(image_desc);
ShaderProgram previous_program;
for (const auto count : {3u, 7u}) {
cache.FillBuffer(count_output, 16u * sizeof(u32), sentinel, false);
shaders.SetCsShader({.data_addr = reinterpret_cast<uint64_t>(writer.data()),
.num_thread_x = 1, .num_thread_y = 1, .num_thread_z = 1,
.wave_size = 64, .user_sgpr = 15});
for (u32 i = 0; i < 3; i++) {
set_buffer(i * 4u, count_args + i * 4u, 4);
shaders.SetCsUserSgpr(12u + i, i == 0 ? count : 1u,
HW::UserSgprType::Unknown);
}
processor.DispatchDirect(1, 1, 1, 0x41u);
Require(name, "GPU-owned descriptor count", cache.HasGpuDirtyBytes(count_args, 4),
"the descriptor count was already visible to the CPU");
// Raw buffer access keeps its own contents even after a newer overlapping image write.
vk::ClearValue painted{};
painted.color.uint32 = std::array{image_value + count, 0u, 0u, 0u};
TextureCacheTestAccess::ClearImage(
textures, scheduler.Current(), image,
{vk::ImageAspectFlagBits::eColor, 0, 1, 0, 1}, painted);
u32 stale_count = 0;
Require(name, "independent raw buffer and image contents",
textures.GetImage(image).SafeToDownload() &&
cache.HasGpuDirtyBytes(count_args, sizeof(u32)) &&
LibKernel::Memory::TryReadBacking(count_args, &stale_count,
sizeof(stale_count)) &&
stale_count != count,
"the alias fixture did not retain GPU buffer data and a newer independent image");
shaders.SetCsShader({.data_addr = reinterpret_cast<uint64_t>(counted_consumer.data()),
.num_thread_x = 4, .num_thread_y = 1, .num_thread_z = 1,
.wave_size = 64, .user_sgpr = 8, .tgid_x_en = true});
set_buffer(0, count_output, 0, 4);
set_buffer(4, count_args, 1, 4);
const std::array<u32, 3> packet{static_cast<u32>(count_args),
static_cast<u32>(count_args >> 32u), 0x41u};
const auto dirty_tick = scheduler.CurrentTick();
Require(name, "GPU count indirect dispatch",
CpOpDispatchIndirect(processor, 0xc0021600u, packet.data(), 0, 0) == 3 &&
scheduler.CurrentTick() > dirty_tick &&
!cache.HasGpuDirtyBytes(count_args, 4),
"the descriptor dependency did not synchronize its preceding GPU writer");
const auto clean_tick = scheduler.CurrentTick();
ShaderComputeInputInfo input{};
const auto program = context.GetPipelineCache().GetComputeProgram(
shaders.GetCs(), processor.GetCtx().GetShaderRegisters(), input);
if (previous_program.id != 0) {
Require(name, "count permutation reuse", program.id == previous_program.id,
"a changing GPU count compiled a new shader permutation");
}
previous_program = program;
Require(name, "clean count repeat",
CpOpDispatchIndirect(processor, 0xc0021600u, packet.data(), 0, 0) == 3 &&
scheduler.CurrentTick() == clean_tick,
"a clean descriptor dependency caused another GPU drain");
cache.ReadMemory(count_output, 16u * sizeof(u32));
std::array<u32, 16> actual{};
Require(name, "counted output readback",
LibKernel::Memory::TryReadBacking(count_output, actual.data(), sizeof(actual)),
"the counted consumer output could not be read back");
for (u32 i = 0; i < actual.size(); i++) {
Require(name, "native descriptor count bound",
actual[i] == (i < count ? count + i + 1u : sentinel),
"GPU count " + std::to_string(count) + " wrote an incorrect output word " +
std::to_string(i));
}
Require(name, "raw count preserves independent image contents",
ReadCachedTexel(name, context, image, {}, {8, 1, 1}) ==
std::vector<u32>(8, image_value + count),
"raw buffer descriptor materialization overwrote the newer native image");
}
// PPSA24156 loads an image sharp from a 148-byte record selected by
// WorkGroupID.z, then uses the same descriptor words for dimensions.
constexpr auto parameters = base + 0x80000;
constexpr auto table_address = parameters + 0x100;
std::vector<u32> table_shader{
EncodeSop2(0x26, 8, 4, 255), 148,
EncodeSmem0(0x02, 16), EncodeSmem1(0, 125),
EncodeSmem0(0x0b, 32, 8), EncodeSmem1(0, 8),
EncodeSop2(0x1e, 8, 34, InlineU32(2)),
EncodeSop2(0x20, 9, 33, InlineU32(30)),
EncodeSop2(0x0e, 8, 8, 255), 0xfffcu,
EncodeSop2(0x10, 8, 8, 9),
EncodeSop2(0x27, 9, 34, 255), 0x000e000eu,
EncodeSop2(0x02, 8, 8, 9),
EncodeSop2(0x02, 8, 8, InlineU32(2)),
EncodeVop1(0x01, 0, 4), EncodeVop1(0x01, 1, InlineU32(0)),
EncodeVop1(0x01, 2, 8),
EncodeMimg0(0x08, 1), EncodeMimg1(2, 0, 8)};
AppendEnd(&table_shader);
ShaderMapUserData(reinterpret_cast<uint64_t>(table_shader.data()),
{.type = Prospero::ShaderBinaryType::kCs,
.code_size_bytes = static_cast<u32>(table_shader.size() * sizeof(u32))});
shaders.SetCsShader({.data_addr = reinterpret_cast<uint64_t>(table_shader.data()),
.num_thread_x = 1, .num_thread_y = 1, .num_thread_z = 1,
.wave_size = 64, .user_sgpr = 2, .tgid_x_en = true,
.tgid_y_en = true, .tgid_z_en = true});
shaders.SetCsUserSgpr(0, static_cast<u32>(parameters), HW::UserSgprType::Unknown);
shaders.SetCsUserSgpr(1, static_cast<u32>(parameters >> 32u), HW::UserSgprType::Unknown);
const ShaderBufferResource table_buffer{{static_cast<u32>(table_address),
static_cast<u32>(table_address >> 32u) | (148u << 16u), 2, 0x5204u}};
LibKernel::Memory::WriteBacking(parameters, table_buffer.fields, sizeof(table_buffer.fields));
struct TableCase { u32 groups, width, height, image; };
uint64_t table_program_id = 0;
for (const auto test : {TableCase{1, 4, 2, 0}, TableCase{2, 4, 2, 0},
TableCase{2, 8, 3, 1}, TableCase{1, 4, 2, 0}}) {
const auto address = base + 0x90000 + test.image * 0x10000;
const ShaderTextureResource sharp{{static_cast<u32>(address >> 8u),
(static_cast<u32>(Prospero::BufferFormat::k32UInt) << 20u) |
(((test.width - 1u) & 3u) << 30u),
((test.width - 1u) >> 2u) | ((test.height - 1u) << 14u),
DstSel(4, 5, 6, 7) |
(static_cast<u32>(Prospero::TileMode::kLinear) << 20u) |
(static_cast<u32>(Prospero::ImageType::kColor2D) << 28u),
0, 0x00700000u, 0, 0}};
std::array<u32, 74> table{};
for (u32 group = 0; group < test.groups; group++) {
std::copy_n(sharp.fields, 8, table.begin() + group * 37);
}
Require(name, "descriptor table refresh", context.InvalidateMemory(table_address, sizeof(table)),
"the descriptor table escaped its mapped allocation");
LibKernel::Memory::WriteBacking(table_address, table.data(), sizeof(table));
ShaderComputeInputInfo input{};
input.workgroup_counts[0] = input.workgroup_counts[1] = 1;
input.workgroup_counts[2] = test.groups;
const auto program = context.GetPipelineCache().GetComputeProgram(
shaders.GetCs(), processor.GetCtx().GetShaderRegisters(), input);
Require(name, "workgroup table refresh and coalescing",
input.workgroup_counts[2] == test.groups &&
input.stage.program->info.images.size() == 1 &&
input.stage.program->info.images[0].indirect_resources.empty() &&
input.stage.resources->images.size() == 1 &&
std::equal(std::begin(sharp.fields), std::end(sharp.fields),
input.stage.resources->images[0].dwords.begin()) &&
!input.stage.program->info.buffers.empty(),
"workgroup bounds, refreshed sharp, or live descriptor dimension reads were lost");
Require(name, "workgroup table program reuse",
table_program_id == 0 || program.id == table_program_id,
"changing the workgroup count or identical sharp values recompiled the shader");
table_program_id = program.id;
const auto *snapshot = input.stage.resources;
const auto *image_storage = snapshot->images.data();
const auto flattened_capacity = snapshot->flattened_srt.capacity();
const auto repeated = context.GetPipelineCache().GetComputeProgram(
shaders.GetCs(), processor.GetCtx().GetShaderRegisters(), input);
Require(name, "workgroup table cache-hit storage reuse",
repeated.id == program.id && input.stage.resources == snapshot &&
snapshot->images.data() == image_storage &&
snapshot->flattened_srt.capacity() == flattened_capacity,
"a repeated descriptor table hit replaced its snapshot or storage");
auto binding = RenderExecutorTestAccess::ResolveTexture(
context.GetRenderExecutor(), input.stage.program->info.images[0],
input.stage.resources->images[0]);
const auto target = textures.FindImage(binding.desc);
TextureCacheTestAccess::ClearImage(textures, scheduler.Current(), target,
{vk::ImageAspectFlagBits::eColor, 0, 1, 0, 1}, {});
processor.DispatchDirect(1, 1, test.groups, 0x41u);
const auto pixels = ReadCachedTexel(name, context, target, {},
{test.width, test.height, 1});
for (u32 pixel = 0; pixel < pixels.size(); pixel++) {
const auto expected = pixel < test.groups ? test.width + test.height : 0u;
Require(name, "workgroup table GPU dimension reads", pixels[pixel] == expected,
"a retained descriptor load used stale dimensions or the wrong record stride");
}
}
// GPU-written dispatch counts bound the stride12 selector prefix; only
// those keys select the two real descriptor members in each stride192 row.
constexpr auto record_args = base + 0xd0000;
constexpr auto record_keys = base + 0xd1000;
constexpr auto record_table = base + 0xd2000;
constexpr auto record_output = base + 0xd7000;
for (const bool based : {false, true}) {
std::vector<u32> record_shader;
if (based) {
// The later captured shader starts its prefix at a clean SRT DWORD.
record_shader = {EncodeSmem0(0x00, 18, 2), EncodeSmem1(0x234, 125), EncodeSopp(0x0c, 0),
EncodeSop2(0x02, 18, 18, 13),
EncodeSop2(0x1e, 16, 18, InlineU32(3)),
EncodeSop2(0x02, 16, 16, InlineU32(4))};
} else record_shader = {EncodeSop2(0x26, 16, 13, InlineU32(12))};
record_shader.insert(record_shader.end(), {
EncodeSmem0(0x08, 16, 0), EncodeSmem1(0, 16), EncodeSopp(0x0c, 0)});
if (based) record_shader.insert(record_shader.end(), {
EncodeSop2(0x27, 16, 16, 255), (5u << 16u) | 16u});
record_shader.insert(record_shader.end(), {
EncodeSop2(0x26, 17, 16, 255), 192u,
EncodeSmem0(based ? 0x02 : 0x0a, 20, 2), EncodeSmem1(160, 17),
EncodeSmem0(based ? 0x02 : 0x0a, 24, 2), EncodeSmem1(176, 17), EncodeSopp(0x0c, 0),
EncodeMubuf0(0x83, 0, true, false), EncodeMubuf1(4, 5, 0),
EncodeSopp(0x0c, 0)});
AppendVop3(&record_shader, 0x346, 8, 13, 12, Vgpr(0));
record_shader.insert(record_shader.end(), {
EncodeMubuf0(0x1d, 0, true, false), EncodeMubuf1(4, 2, 8)});
AppendVMovLiteral(&record_shader, 10, 0x5au);
AppendSMovLiteral(&record_shader, 126, 0x55555555u);
AppendSMovLiteral(&record_shader, 127, 0x55555555u);
record_shader.insert(record_shader.end(), {
EncodeMubuf0(0x18, 0, true, false, true), EncodeMubuf1(10, 6, 0)});
AppendEnd(&record_shader);
ShaderMapUserData(reinterpret_cast<uint64_t>(record_shader.data()),
{.type = Prospero::ShaderBinaryType::kCs,
.code_size_bytes = static_cast<u32>(record_shader.size() * sizeof(u32))});
std::array<u32, 96> records;
records.fill(0xdeadbeefu);
const std::array<u32, 4> numeric_descriptor{0x640u, 3u, 0x3ba3d70au, 0x3a23d70au};
std::copy(numeric_descriptor.begin(), numeric_descriptor.end(), records.begin() + 76u);
for (u32 row = 0; row < 2u; ++row) {
const auto payload = base + 0xd3000 + row * 0x1000;
const auto target = base + 0xd5000 + row * 0x1000;
const std::array<u32, 8> descriptors{
static_cast<u32>(payload), static_cast<u32>(payload >> 32u) | (8u << 16u), 4u,
(u32(Prospero::BufferFormat::k16_16_16_16Float) << 12u) | DstSel(4, 5, 6, 7),
static_cast<u32>(target), static_cast<u32>(target >> 32u) | (1u << 16u), 4u, 0x5204u};
std::copy(descriptors.begin(), descriptors.end(), records.begin() + row * 48u + 40u);
const std::array<u32, 8> data{0x40003c00u + row, 0xc0003800u + row,
0x44004200u + row, 0xc400bc00u + row, 0x48004600u + row, 0xc800c600u + row,
0x4c004a00u + row, 0xcc00ca00u + row};
LibKernel::Memory::WriteBacking(payload, data.data(), sizeof(data));
}
LibKernel::Memory::WriteBacking(record_table, records.data(), sizeof(records));
const auto produce = [&](uint64_t address, u32 stride, std::array<u32, 3> values) {
shaders.SetCsShader({.data_addr = reinterpret_cast<uint64_t>(writer.data()),
.num_thread_x = 1, .num_thread_y = 1, .num_thread_z = 1,
.wave_size = 64, .user_sgpr = 15});
for (u32 word = 0; word < 3u; ++word) {
set_buffer(word * 4u, address + word * stride, 4u);
shaders.SetCsUserSgpr(12u + word, values[word], HW::UserSgprType::Unknown);
}
processor.DispatchDirect(1, 1, 1, 0x41u);
};
for (const u32 wave : {32u, 64u}) {
u32 first = based ? 2u : 0u;
for (const auto dimensions : {std::array{2u, 1u, 1u}, std::array{0u, 1u, 1u},
std::array{2u, 0u, 1u}, std::array{2u, 1u, 0u},
std::array{1u, 1u, 1u}, std::array{2u, 1u, 1u}}) {
const bool empty = std::ranges::find(dimensions, 0u) != dimensions.end();
const std::array<u32, 3> stale_args{};
std::array<u32, 10> stale_keys;
stale_keys.fill(sentinel);
Require(name, "record fixture reset", context.InvalidateMemory(record_args, 0x8000),
"record fixture escaped its mapped allocation");
LibKernel::Memory::WriteBacking(record_args, stale_args.data(), sizeof(stale_args));
LibKernel::Memory::WriteBacking(record_keys, stale_keys.data(), sizeof(stale_keys));
LibKernel::Memory::WriteBacking(record_table + 0x234, &first, sizeof(first));
std::vector<u32> expected_output(wave * 4u, sentinel);
LibKernel::Memory::WriteBacking(record_output, expected_output.data(), expected_output.size() * 4u);
const std::array<u32, 4> untouched{sentinel, sentinel, sentinel, sentinel};
for (u32 row = 0; row < 2u; ++row)
LibKernel::Memory::WriteBacking(base + 0xd5000 + row * 0x1000, untouched.data(), sizeof(untouched));
produce(record_args, 4u, dimensions);
produce(record_keys + (based ? first * 8u + 4u : 0u), based ? 8u : 12u,
based ? std::array{0x1beefu, 0xbeefu, 0x1fbeefu}
: std::array{1u, 0u, 22369622u});
std::array<u32, 3> cpu_args{};
std::array<u32, 10> cpu_keys{};
Require(name, "GPU record dependencies remain pending",
cache.HasGpuDirtyBytes(record_args, 12u) && cache.HasGpuDirtyBytes(record_keys, sizeof(stale_keys)) &&
LibKernel::Memory::TryReadBacking(record_args, cpu_args.data(), sizeof(cpu_args)) &&
LibKernel::Memory::TryReadBacking(record_keys, cpu_keys.data(), sizeof(cpu_keys)) &&
cpu_args == stale_args && cpu_keys == stale_keys,
"the producer did not leave dispatch counts and selectors GPU-owned");
shaders.SetCsShader({.data_addr = reinterpret_cast<uint64_t>(record_shader.data()),
.num_thread_x = wave, .num_thread_y = 1, .num_thread_z = 1,
.user_sgpr = 13, .tgid_x_en = true});
set_buffer(0, record_keys, based ? 5u : 3u, based ? 8u : 12u);
set_buffer(4, record_table, 2u, 192u);
if (based) shaders.SetCsUserSgpr(5, record_table >> 32u, HW::UserSgprType::Unknown);
set_buffer(8, record_output, wave * 2u, 8u);
shaders.SetCsUserSgpr(12, std::countr_zero(wave), HW::UserSgprType::Unknown);
processor.DispatchIndirect(record_args, 0x41u |
(wave == 32u ? 1u << Pm4::COMPUTE_DISPATCH_INITIATOR_CS_W32_EN_SHIFT : 0u));
cache.ReadMemory(record_output, expected_output.size() * 4u);
std::vector<u32> actual_output(expected_output.size());
Require(name, "record output readback", LibKernel::Memory::TryReadBacking(
record_output, actual_output.data(), actual_output.size() * 4u),
"indirect record output could not be read back");
if (!empty)
for (u32 group = 0; group < dimensions[0]; ++group) {
const auto row = 1u - group;
for (u32 lane = 0; lane < wave; ++lane) {
const auto out = (group * wave + lane) * 2u;
expected_output[out] = expected_output[out + 1u] = 0u;
if (lane < 4u)
Require(name, "unchanged formatted input", LibKernel::Memory::TryReadBacking(
base + 0xd3000 + row * 0x1000 + lane * 8u, expected_output.data() + out, 8u),
"the formatted source escaped its mapped allocation");
}
}
Require(name, "coherent exact workgroup selector prefix", actual_output == expected_output,
"GPU counts or selectors were stale, over-enumerated, or selected the wrong record");
for (u32 row = 0; row < 2u; ++row) {
const auto target = base + 0xd5000 + row * 0x1000;
cache.ReadMemory(target, sizeof(untouched));
auto expected = untouched;
if (!empty && (dimensions[0] == 2u || row == 1u)) expected[0] = 0xa55aa55au;
std::array<u32, 4> actual{};
Require(name, "selected byte store and canaries", LibKernel::Memory::TryReadBacking(
target, actual.data(), sizeof(actual)) && actual == expected,
"masked/OOB record stores changed an inactive row or adjacent bytes");
}
if (based) first ^= 2u;
}
}
}
// The captured material dispatch reaches 132 distinct writable records.
// Keep their null row and GPU-only keys on the native renderer/cache path.
constexpr u32 material_count = 132, material_rows = material_count + 1u;
constexpr u32 material_groups = material_rows + 1u, material_slot = 256;
constexpr auto material_keys = base;
constexpr auto material_table = base + 0x1000;
constexpr auto material_output = base + 0x20000;
const auto material_prefix = [](u32 row) { return row % 127u + 1u; };
std::vector<u32> material_producer;
AppendVop3(&material_producer, 0x346, 1, 4, InlineU32(6), Vgpr(0));
material_producer.insert(material_producer.end(), {
EncodeVop2(0x1a, 2, InlineU32(16), 1),
EncodeMubuf0(0x1c, 0, true, false), EncodeMubuf1(2, 0, 1)});
AppendEnd(&material_producer);
ShaderMapUserData(reinterpret_cast<uint64_t>(material_producer.data()),
{.type = Prospero::ShaderBinaryType::kCs,
.code_size_bytes = static_cast<u32>(material_producer.size() * sizeof(u32))});
for (const u32 wave : {32u, 64u}) {
Require(name, "large material fixture reset", context.InvalidateMemory(base, 0x50000),
"material fixtures escaped the existing mapped allocation");
std::vector<u32> records(material_rows * 592u / 4u);
std::vector<u32> expected(material_count * material_slot / 4u, sentinel);
auto expected_bytes = std::as_writable_bytes(std::span(expected));
for (u32 row = 1; row < material_rows; ++row) {
const auto target = material_output + (row - 1u) * material_slot + material_prefix(row);
const std::array<u32, 4> descriptor{static_cast<u32>(target),
static_cast<u32>(target >> 32u) | (1u << 16u), 20u,
(u32(Prospero::BufferFormat::k8SInt) << 12u) | DstSel(4, 0, 0, 0)};
std::copy(descriptor.begin(), descriptor.end(), records.begin() + (row * 592u + 448u) / 4u);
for (u32 lane = 0; lane < 20u; lane += 2u)
expected_bytes[(row - 1u) * material_slot + material_prefix(row) + lane] =
std::byte(u32(std::clamp(int32_t(lane * 32u) - 200, -128, 127)) & 255u);
}
const std::vector<u32> untouched(expected.size(), sentinel);
const std::vector<u32> stale_keys(material_groups, sentinel);
LibKernel::Memory::WriteBacking(material_keys, stale_keys.data(), stale_keys.size() * 4u);
LibKernel::Memory::WriteBacking(material_table, records.data(), records.size() * 4u);
LibKernel::Memory::WriteBacking(material_output, untouched.data(), untouched.size() * 4u);
shaders.SetCsShader({.data_addr = reinterpret_cast<uint64_t>(material_producer.data()),
.num_thread_x = 64, .num_thread_y = 1, .num_thread_z = 1,
.wave_size = 64, .user_sgpr = 4, .tgid_x_en = true});
set_buffer(0, material_keys, material_groups, 4u);
processor.DispatchDirect(3, 1, 1, 0x41u);
std::vector<u32> cpu_keys(material_groups);
Require(name, "GPU-only material keys",
cache.HasGpuDirtyBytes(material_keys, stale_keys.size() * 4u) &&
LibKernel::Memory::TryReadBacking(material_keys, cpu_keys.data(), cpu_keys.size() * 4u) &&
cpu_keys == stale_keys,
"material selection did not remain GPU-owned before native compilation");
std::vector<u32> material_shader{
EncodeVop1(0x01, 2, 8),
EncodeMubuf0(0x0c, 0, true, false), EncodeMubuf1(4, 0, 2), EncodeSopp(0x0c, 0),
EncodeVop2(0x16, 6, InlineU32(16), 4)};
AppendVop3(&material_shader, 0x360, 12, Vgpr(6), InlineU32(0));
material_shader.insert(material_shader.end(), {
EncodeSop2(0x26, 13, 12, 255), 592u,
EncodeSmem0(0x0a, 16, 2), EncodeSmem1(448, 13), EncodeSopp(0x0c, 0),
EncodeVop2(0x1a, 10, InlineU32(5), 0),
EncodeVop2(0x25, 10, 255, 10), u32(-200)});
AppendSMovLiteral(&material_shader, 126, 0x55555555u);
AppendSMovLiteral(&material_shader, 127, wave == 64u ? 0x55555555u : 0u);
material_shader.insert(material_shader.end(), {
EncodeMubuf0(0x04, 0, true, false, true), EncodeMubuf1(10, 4, 0)});
AppendEnd(&material_shader);
ShaderMapUserData(reinterpret_cast<uint64_t>(material_shader.data()),
{.type = Prospero::ShaderBinaryType::kCs,
.code_size_bytes = static_cast<u32>(material_shader.size() * sizeof(u32))});
shaders.SetCsShader({.data_addr = reinterpret_cast<uint64_t>(material_shader.data()),
.num_thread_x = wave, .num_thread_y = 1, .num_thread_z = 1,
.wave_size = static_cast<uint8_t>(wave), .user_sgpr = 8, .tgid_x_en = true});
set_buffer(0, material_keys, material_groups, 4u);
set_buffer(4, material_table, material_rows, 592u);
ShaderComputeInputInfo input{};
input.workgroup_counts[0] = material_groups;
input.workgroup_counts[1] = input.workgroup_counts[2] = 1u;
(void)context.GetPipelineCache().GetComputeProgram(
shaders.GetCs(), processor.GetCtx().GetShaderRegisters(), input);
const auto &info = input.stage.program->info;
const auto root = std::ranges::find_if(info.buffers, [](const auto &buffer) {
return buffer.indirect_resources.size() == material_rows;
});
const auto &bindings = input.stage.program->bindings;
const auto native_count = bindings.descriptor_counts[
static_cast<size_t>(ShaderRecompiler::IR::DescriptorBindingKind::Buffers)];
Require(name, "native material descriptor array",
root != info.buffers.end() && native_count > 64u && root->packed_stride == 0u &&
std::ranges::all_of(root->indirect_resources,
[&](u32 index) { return info.buffers[index].live; }),
"material candidates were truncated, left unbound, or lost their null root");
Require(name, "large material shader-data buffer", !bindings.UsesPushData() &&
bindings.descriptor_counts[static_cast<size_t>(
ShaderRecompiler::IR::DescriptorBindingKind::ShaderData)] == 1u,
"large binding-prefix data did not use the existing shader-data buffer");
processor.DispatchDirect(material_groups, 1, 1, 0x41u |
(wave == 32u ? 1u << Pm4::COMPUTE_DISPATCH_INITIATOR_CS_W32_EN_SHIFT : 0u));
for (u32 row = 1; row < material_rows; ++row)
Require(name, "every selected material becomes GPU-owned",
cache.HasGpuDirtyBytes(material_output + (row - 1u) * material_slot + material_prefix(row), 20u),
"a writable material beyond the original descriptor capacity was not tracked");
cache.ReadMemory(material_output, expected.size() * 4u);
std::vector<u32> actual(expected.size());
Require(name, "132 material writes, null/OOB keys, masks and byte canaries",
LibKernel::Memory::TryReadBacking(material_output, actual.data(), actual.size() * 4u) &&
actual == expected,
"native material selection or per-binding byte offsets corrupted a target or neighboring byte");
}
// Exchange values across two waves in the last 512 bytes of each
// workgroup's 64-KiB LDS, on a host restricted to 48-KiB native shared memory.
constexpr auto lds_output = base + 0xb0000;
constexpr auto lds_args = base + 0xc0000;
constexpr u32 lds_words = 8 * 128;
std::vector<u32> lds_shader{
EncodeSop2(0x1e, 8, 5, InlineU32(1)),
EncodeSop2(0x1e, 9, 6, InlineU32(2)),
EncodeSop2(0x02, 8, 8, 9), EncodeSop2(0x02, 8, 8, 4)};
AppendVop3(&lds_shader, 0x346, 2, 8, InlineU32(7), Vgpr(0));
lds_shader.push_back(EncodeVop2(0x1a, 5, InlineU32(2), 2));
lds_shader.push_back(EncodeVop2(0x25, 2, InlineU32(1), 2));
lds_shader.push_back(EncodeVop2(0x1a, 1, InlineU32(2), 0));
AppendVMovLiteral(&lds_shader, 7, 64 * 1024 - 128 * 4);
lds_shader.insert(lds_shader.end(), {
EncodeVop2(0x25, 1, Vgpr(7), 1),
EncodeDs0(0x0d), EncodeDs1(0, 2, 1), EncodeSopp(0x0a, 0),
EncodeVop2(0x1d, 3, InlineU32(64), 0),
EncodeVop2(0x1a, 3, InlineU32(2), 3), EncodeVop2(0x25, 3, Vgpr(7), 3),
EncodeDs0(0x36), EncodeDs1(4, 0, 3),
EncodeMubuf0(0x1c), EncodeMubuf1(4, 0, 5)});
AppendEnd(&lds_shader);
ShaderMapUserData(reinterpret_cast<uint64_t>(lds_shader.data()),
{.type = Prospero::ShaderBinaryType::kCs,
.code_size_bytes = static_cast<u32>(lds_shader.size() * sizeof(u32))});
auto &host_lds_limit = context.GetGraphics().physical_device_properties.limits.maxComputeSharedMemorySize;
const auto saved_lds_limit = host_lds_limit;
host_lds_limit = std::min(host_lds_limit, 48u * 1024u);
for (u32 dispatch = 0; dispatch < 4; dispatch++) {
cache.FillBuffer(lds_output, lds_words * sizeof(u32), sentinel, false);
const bool indirect = dispatch == 1 || dispatch == 2;
const bool empty = dispatch == 2;
if (indirect) {
const auto groups = empty ? std::array{4096u, 0u, 1u} : std::array{2u, 2u, 2u};
shaders.SetCsShader({.data_addr = reinterpret_cast<uint64_t>(writer.data()),
.num_thread_x = 1, .num_thread_y = 1, .num_thread_z = 1,
.wave_size = 64, .user_sgpr = 15});
for (u32 axis = 0; axis < 3; axis++) {
set_buffer(axis * 4, lds_args + axis * sizeof(u32), sizeof(u32));
shaders.SetCsUserSgpr(12 + axis, groups[axis], HW::UserSgprType::Unknown);
}
processor.DispatchDirect(1, 1, 1, 0x41u);
Require(name, "GPU-authored LDS dispatch arguments",
cache.HasGpuDirtyBytes(lds_args, 3 * sizeof(u32)),
"indirect LDS dimensions were already visible on the CPU");
}
shaders.SetCsShader({.data_addr = reinterpret_cast<uint64_t>(lds_shader.data()),
.num_thread_x = 128, .num_thread_y = 1, .num_thread_z = 1,
.wave_size = 64, .user_sgpr = 4, .tgid_x_en = true,
.tgid_y_en = true, .tgid_z_en = true, .lds_size = 128});
set_buffer(0, lds_output, lds_words * sizeof(u32));
if (indirect) {
const std::array<u32, 3> packet{static_cast<u32>(lds_args),
static_cast<u32>(lds_args >> 32u), 0x41u};
Require(name, "indirect storage LDS dispatch",
CpOpDispatchIndirect(processor, 0xc0021600u, packet.data(), 0, 0) == 3,
"the GPU-authored LDS dispatch was not consumed");
} else {
processor.DispatchDirect(2, 2, 2, 0x41u);
}
cache.ReadMemory(lds_output, lds_words * sizeof(u32));
std::array<u32, lds_words> actual{};
Require(name, "storage LDS readback",
LibKernel::Memory::TryReadBacking(lds_output, actual.data(), sizeof(actual)),
"LDS result buffer could not be read back");
for (u32 word = 0; word < actual.size(); word++) {
const auto expected = empty ? sentinel : (word ^ 64u) + 1u;
Require(name, "storage LDS upper range, isolation and barrier", actual[word] == expected,
"64-KiB LDS lost upper bytes, workgroup isolation, or cross-wave writes");
}
}
host_lds_limit = saved_lds_limit;
context.UnmapMemory(base, allocation_size);
scheduler.Finish();
});
LibKernel::Memory::InstallGpuResources(nullptr);
context.ShutdownGpu();
Require(name, "unmap direct backing",
Libs::LibKernel::Memory::KernelMunmap(base, allocation_size) == 0,
"indirect argument mapping release failed");
Require(name, "release direct backing",
Libs::LibKernel::Memory::KernelReleaseDirectMemory(
direct_offset, allocation_size) == 0,
"indirect argument allocation release failed");
std::printf("[gpu] %-32s ok\n", name);
}
void CheckRenderExecutorColorMetadataClear() {
constexpr const char *name = "RenderExecutorColorMetadataClear";
struct PackedCase {
vk::Format format;
uint64_t word;
std::array<float, 4> channels;
};
const std::array packed_cases{
PackedCase{vk::Format::eR32G32Sfloat, 0xbf8000003f800000ull, {1.0f, -1.0f}},
PackedCase{vk::Format::eR32G32Sfloat, 0x7f80000080000000ull,
{-0.0f, std::numeric_limits<float>::infinity()}},
PackedCase{vk::Format::eB10G11R11UfloatPack32, 0u, {}},
PackedCase{vk::Format::eB10G11R11UfloatPack32, 0x801c03c0u, {1.0f, 0.5f, 2.0f}},
PackedCase{vk::Format::eB10G11R11UfloatPack32, 1u | (0x7bfu << 11u) | (1u << 22u),
{0x1p-20f, 65024.0f, 0x1p-19f}},
PackedCase{vk::Format::eB10G11R11UfloatPack32, 0x7c0u | (0x7c0u << 11u) | (0x3e0u << 22u),
{std::numeric_limits<float>::infinity(), std::numeric_limits<float>::infinity(),
std::numeric_limits<float>::infinity()}},
PackedCase{vk::Format::eR16Sfloat, 0x7bff7bffu, {65504.0f}},
PackedCase{vk::Format::eR16G16Sfloat, 0xbc003c00u, {1.0f, -1.0f}},
PackedCase{vk::Format::eR16G16B16A16Sfloat, 0x40003800bc003c00ull,
{1.0f, -1.0f, 0.5f, 2.0f}},
PackedCase{vk::Format::eR16Sfloat, 0xffff0001u, {0x1p-24f}},
PackedCase{vk::Format::eR16Sfloat, 0x00008001u, {-0x1p-24f}},
PackedCase{vk::Format::eR16Sfloat, 0x000003ffu, {1023.0f * 0x1p-24f}},
PackedCase{vk::Format::eR16Sfloat, 0x00000400u, {0x1p-14f}},
PackedCase{vk::Format::eR16G16Sfloat, 0x80000000u, {0.0f, -0.0f}},
PackedCase{vk::Format::eR16G16Sfloat, 0xfc007c00u,
{std::numeric_limits<float>::infinity(),
-std::numeric_limits<float>::infinity()}},
PackedCase{vk::Format::eR16Unorm, 0x0000ffffu, {1.0f}},
PackedCase{vk::Format::eR16G16Unorm, 0x0000ffffu, {1.0f, 0.0f}},
PackedCase{vk::Format::eR16G16Unorm, 0x80000001u,
{1.0f / 65535.0f, 32768.0f / 65535.0f}},
};
for (const auto &test : packed_cases) {
vk::ClearColorValue clear{};
Require(name, "packed color clear",
DecodePackedColorClear(test.format, test.word, clear) &&
std::bit_cast<std::array<uint32_t, 4>>(clear.float32) ==
std::bit_cast<std::array<uint32_t, 4>>(test.channels),
"packed clear lost channel order, sign, range or unused-channel zeros");
}
vk::ClearColorValue clear{};
Require(name, "half NaN clear",
DecodePackedColorClear(vk::Format::eR16Sfloat, 0x7e00u, clear) &&
std::isnan(clear.float32[0]),
"half NaN did not remain NaN");
Require(name, "packed unsigned NaN clear",
DecodePackedColorClear(vk::Format::eB10G11R11UfloatPack32,
0x7c1u | (0x7c1u << 11u) | (0x3e1u << 22u), clear) &&
std::isnan(clear.float32[0]) && std::isnan(clear.float32[1]) &&
std::isnan(clear.float32[2]),
"packed unsigned NaNs did not remain NaN");
for (const auto format : {vk::Format::eR16Sfloat, vk::Format::eR16Unorm}) {
Require(name, "R16 DWORD fill",
!DecodeColorDwordFill(format, 0xbc003c00u, clear) &&
DecodeColorDwordFill(format, 0x3c003c00u, clear),
"alternating R16 pixels were treated as a uniform clear");
}
Require(name, "RG32 DWORD fill",
DecodeColorDwordFill(vk::Format::eR32G32Sfloat, 0x3f800000u, clear) &&
clear.float32[0] == 1.0f && clear.float32[1] == 1.0f,
"a repeated DWORD fill was decoded as a two-register clear");
constexpr uintptr_t base = 0x0000000204100000ull;
constexpr uint64_t allocation_size = 0x400000;
constexpr uint64_t allocation_alignment = 0x10000;
struct FillCase {
uint32_t fill;
std::array<uint32_t, 2> texel;
bool reuse_unorm = false;
bool cmask = false;
bool reuse_mips = false;
Prospero::ChannelLayout layout = Prospero::ChannelLayout::k16_16_16_16;
Prospero::ChannelType type = Prospero::ChannelType::kFloat;
uint32_t clear_word = 0;
bool eliminate_fast_clear = false;
};
constexpr std::array cases{
FillCase{0x40404040u, {0, 0x3c000000u}},
FillCase{0x80808080u, {0x3c003c00u, 0x00003c00u}},
FillCase{0x40404040u, {0, 0x3c000000u}, true},
FillCase{.fill = 0, .texel = {0x804020ffu, 0}, .cmask = true,
.layout = Prospero::ChannelLayout::k8_8_8_8,
.type = Prospero::ChannelType::kUNorm},
FillCase{.fill = 0, .texel = {0, 0}, .cmask = true,
.eliminate_fast_clear = true},
FillCase{.fill = 0, .texel = {0, 0x3c000000u}, .cmask = true,
.eliminate_fast_clear = true},
FillCase{.fill = 0, .texel = {0, 0}, .cmask = true,
.layout = Prospero::ChannelLayout::k11_11_10, .eliminate_fast_clear = true},
FillCase{.fill = 0, .texel = {0x801c03c0u, 0}, .cmask = true,
.layout = Prospero::ChannelLayout::k11_11_10, .eliminate_fast_clear = true},
FillCase{.fill = 0, .texel = {0, 0}, .cmask = true,
.layout = Prospero::ChannelLayout::k32_32, .eliminate_fast_clear = true},
FillCase{.fill = 0, .texel = {0x3f800000u, 0xbf800000u}, .cmask = true,
.layout = Prospero::ChannelLayout::k32_32, .eliminate_fast_clear = true},
FillCase{0x40404040u, {0, 0x3c000000u}, false, false, true},
FillCase{.fill = 0x20202020u, .texel = {0x7bff7bffu},
.layout = Prospero::ChannelLayout::k16, .clear_word = 0x7bff7bffu},
FillCase{.fill = 0x20202020u, .texel = {0xbc003c00u},
.layout = Prospero::ChannelLayout::k16_16, .clear_word = 0xbc003c00u},
FillCase{.fill = 0x20202020u, .texel = {0x80008000u},
.layout = Prospero::ChannelLayout::k16,
.type = Prospero::ChannelType::kUNorm, .clear_word = 0x80008000u},
FillCase{.fill = 0x20202020u, .texel = {0x0000ffffu},
.layout = Prospero::ChannelLayout::k16_16,
.type = Prospero::ChannelType::kUNorm, .clear_word = 0x0000ffffu},
};
EnsureRuntimeContext();
// Astro's generic metadata fill, through S_ENDPGM; trailing debug data is omitted.
static constexpr std::array<uint32_t, 69> native_fill{
0xbfa00003u, 0xd7460002u, 0x04010c0au, 0x7da80408u,
0xbf88003fu, 0x7e000c09u, 0xbf070980u, 0x858a807eu,
0x7e005700u, 0x100000ffu, 0x4f800000u, 0x7e060f00u,
0xd5766a00u, 0x02020609u, 0x7d8a0280u, 0x4c020080u,
0x02000101u, 0xd56a0001u, 0x00020700u, 0x4c000303u,
0x4a020303u, 0x02000101u, 0xd56a0000u, 0x00020500u,
0xd5690001u, 0x00020009u, 0x4c060302u, 0x7d860609u,
0x7d8c02f9u, 0x06068c02u, 0x87ea6a0cu, 0x50000080u,
0xd5286a00u, 0x003200c1u, 0xd5010000u, 0x002a00c1u,
0xd5690000u, 0x00020009u, 0x4c000102u, 0x7d0a0080u,
0xbe88246au, 0xbf880015u, 0x7d0a0081u, 0xbe8a246au,
0xbf88000cu, 0x7d0a0082u, 0xbeea246au, 0xbf880003u,
0x7e000207u, 0xe0102000u, 0x80000002u, 0x8afe7e6au,
0xbf880003u, 0x7e000206u, 0xe0102000u, 0x80000002u,
0xbefe046au, 0x8afe7e0au, 0xbf880003u, 0x7e000205u,
0xe0102000u, 0x80000002u, 0xbefe040au, 0x8afe7e08u,
0xbf880003u, 0x7e000204u, 0xe0102000u, 0x80000002u,
0xbf810000u,
};
static constexpr auto raw_fill = [] {
auto words = native_fill;
for (auto &word : words) {
if (word == 0xe0102000u) {
word = 0xe0702000u; // BUFFER_STORE_DWORD in place of BUFFER_STORE_FORMAT_X.
}
}
return words;
}();
for (const auto *shader : {&native_fill, &raw_fill}) {
ShaderMapUserData(reinterpret_cast<uint64_t>(shader->data()),
{.type = Prospero::ShaderBinaryType::kCs,
.code_size_bytes = sizeof(native_fill)});
}
int64_t direct_offset = -1;
Require(name, "direct allocation",
Libs::LibKernel::Memory::KernelAllocateDirectMemory(
0, Libs::LibKernel::Memory::KernelGetDirectMemorySize(),
allocation_size, allocation_alignment, 0, &direct_offset) == 0,
"fixed-clear direct-memory allocation failed");
void *mapped = reinterpret_cast<void *>(base);
Require(name, "direct mapping",
Libs::LibKernel::Memory::KernelMapDirectMemory(
&mapped, allocation_size, 0x3, 0x10, direct_offset,
allocation_alignment) == 0 &&
mapped == reinterpret_cast<void *>(base),
"fixed-clear direct mapping failed");
std::memset(mapped, 0, allocation_size);
for (const auto &fill_case : cases) {
const auto layout = fill_case.layout;
const auto type = fill_case.type;
const auto bytes_per_pixel = TextureGetRenderTargetFormat(
layout, type, Prospero::ChannelOrder::kStandard).bytes_per_element;
const uint32_t probe_width = bytes_per_pixel == 2 ? 2 : 1;
TileSizeAlign dcc_size{};
(void)TileGetDccSize(512, 256, 1, bytes_per_pixel, 1,
Prospero::TileMode::kRenderTarget, dcc_size, 0);
const uint64_t dcc_address = base + (fill_case.eliminate_fast_clear ? 0x200000 :
fill_case.reuse_mips ? 0x160000 : 0x100000);
const uint32_t selected_layer = fill_case.cmask ? 1 : 0;
const uint64_t metadata_size = fill_case.cmask ? 0x2000 : dcc_size.size;
const std::vector<u32> expected(
fill_case.texel.begin(), fill_case.texel.begin() + std::max(bytes_per_pixel / 4u, 1u));
RenderContext context(m_runtime_context);
HW::Context registers{};
HW::UserConfig user_config{};
HW::Shader shaders{};
context.InitializeGpu(nullptr);
LibKernel::Memory::InstallGpuResources(&context);
context.GetGpu().SendCommandSync([&] {
auto &scheduler = context.GetCommandScheduler();
registers.SetColorBase(0, {.addr = base});
registers.SetColorInfo(
0, {.cmask_fast_clear_enable = !fill_case.reuse_unorm,
.dcc_compression_enable = !fill_case.cmask,
.format = layout,
.channel_type = type,
.channel_order = Prospero::ChannelOrder::kStandard});
registers.SetColorView(0, {.base_array_slice_index = selected_layer,
.last_array_slice_index = selected_layer});
registers.SetColorAttrib2(0, {.height = 255, .width = 511});
registers.SetColorAttrib3(0,
{.tile_mode = Prospero::TileMode::kRenderTarget,
.dimension = 1,
.metadata_pipe_aligned = true});
registers.SetColorDccAddr(0, {.addr = dcc_address});
registers.SetColorCmask(0, {.addr = dcc_address});
registers.SetColorClearWord0(0, {.word0 = fill_case.cmask ? expected[0] : fill_case.clear_word});
registers.SetColorClearWord1(0, {.word1 = fill_case.cmask && expected.size() > 1u ? expected[1] : 0});
registers.SetRenderTargetMask(0x0f);
scheduler.Begin(registers, user_config, shaders);
auto &resources = context;
auto &texture_cache = resources.GetTextureCache();
auto &executor = context.GetRenderExecutor();
resources.MapMemory(base, allocation_size);
ImageId preexisting_id{};
RenderColorInfo color{};
if (fill_case.reuse_unorm || fill_case.reuse_mips || fill_case.eliminate_fast_clear) {
const auto float_info = registers.GetRenderTarget(0).info;
const auto single_mip = registers.GetRenderTarget(0).attrib2;
auto initial_info = float_info;
initial_info.dcc_compression_enable = false;
initial_info.cmask_fast_clear_enable = false;
if (fill_case.reuse_unorm) {
initial_info.channel_type = Prospero::ChannelType::kUNorm;
}
auto initial_mips = single_mip;
initial_mips.num_mip_levels = fill_case.reuse_mips ? 1 : 0;
registers.SetColorInfo(0, initial_info);
registers.SetColorAttrib2(0, initial_mips);
RenderColorInfo preexisting{};
RenderExecutorTestAccess::ResolveRenderColorTarget(
executor, scheduler.Current(), preexisting, 0);
preexisting_id = preexisting.image_id;
color = preexisting;
(void)texture_cache.FindRenderTarget(preexisting_id, preexisting.desc);
if (fill_case.reuse_mips) {
Require(name, "mip chain excludes metadata",
preexisting.desc.info.data.address + preexisting.desc.info.data.size <=
dcc_address,
"the mip preservation fixture overlaps its color metadata");
vk::ClearValue clear{};
clear.color.float32 = std::array{1.0f, 0.0f, 1.0f, 0.0f};
TextureCacheTestAccess::ClearImage(
texture_cache, scheduler.Current(), preexisting_id,
{vk::ImageAspectFlagBits::eColor, 0, 2, 0, 1}, clear);
}
if (fill_case.eliminate_fast_clear) {
vk::ClearValue stale{};
stale.color.float32 = std::array{1.0f, 0.0f, 1.0f, 1.0f};
TextureCacheTestAccess::ClearImage(
texture_cache, scheduler.Current(), preexisting_id,
{vk::ImageAspectFlagBits::eColor, 0, 1, 0, 2}, stale);
}
RenderExecutorTestAccess::ResetBindings(executor);
registers.SetColorInfo(0, float_info);
registers.SetColorAttrib2(0, single_mip);
}
const auto fill_metadata = [&](uint32_t count, bool raw = false) {
const auto *shader = &native_fill;
if (raw) {
shader = &raw_fill;
}
shaders.SetCsShader({.data_addr = reinterpret_cast<uint64_t>(shader->data()),
.num_thread_x = 64, .num_thread_y = 1, .num_thread_z = 1,
.wave_size = 64, .user_sgpr = 10, .tgid_x_en = true});
const std::array<uint32_t, 10> user_data{
static_cast<uint32_t>(dcc_address),
static_cast<uint32_t>(dcc_address >> 32u) | (4u << 16u),
count, 0x00014004u, fill_case.fill, 0, 0x200, 0, count, 1};
for (uint32_t i = 0; i < user_data.size(); i++) {
shaders.SetCsUserSgpr(i, user_data[i], HW::UserSgprType::Unknown);
}
executor.DispatchDirect(0, scheduler.Current(), (count + 63) / 64, 1, 1, 0x41u, false);
};
const auto metadata_words = static_cast<uint32_t>(metadata_size / 4);
fill_metadata(metadata_words);
Require(name, "metadata before descriptor",
!texture_cache.IsMeta(dcc_address),
"the native fill guessed an unbound metadata allocation's type");
const auto check_expanded_metadata = [&] {
context.GetBufferCache().ReadMemory(dcc_address, metadata_size, false);
std::vector<uint8_t> bytes(metadata_size);
Require(name, "expanded metadata readback",
LibKernel::Memory::TryReadBacking(dcc_address, bytes.data(), bytes.size()),
"native color metadata could not be read back");
const auto selected = bytes.begin() + (fill_case.cmask ? 0x1000 : 0);
Require(name, "selected metadata slice expansion",
std::all_of(bytes.begin(), selected, [](uint8_t v) { return v == 0; }) &&
std::all_of(selected, bytes.end(), [](uint8_t v) { return v == 0xff; }),
"materialization retained a clear key or changed an unselected slice");
};
const auto paint = [&] {
vk::ClearValue clear{};
clear.color.float32 = std::array{1.0f, 0.0f, 1.0f, 1.0f};
TextureCacheTestAccess::ClearImage(
texture_cache, scheduler.Current(), color.image_id,
{vk::ImageAspectFlagBits::eColor, 0, 1, selected_layer, 1}, clear);
};
if (fill_case.eliminate_fast_clear) {
registers.SetRenderTargetMask(0);
registers.SetColorControl({.mode = 2});
RenderExecutorTestAccess::DrawAuto(executor, scheduler.Current(),
{.vertex_count = 3, .instance_count = 1});
auto sampled = color.desc;
sampled.type = BindingType::Texture;
sampled.view_info.usage = vk::ImageUsageFlagBits::eSampled;
color.image_id = texture_cache.FindImage(sampled);
Require(name, "FCE sampled image reuses stale target",
color.image_id == preexisting_id &&
texture_cache.FindTexture(color.image_id, sampled) != nullptr,
"FCE failed to retain the sampled allocation");
const std::vector<u32> stale = layout == Prospero::ChannelLayout::k11_11_10
? std::vector<u32>{0x780003c0u} : layout == Prospero::ChannelLayout::k32_32
? std::vector<u32>{0x3f800000u, 0u} : std::vector<u32>{0x00003c00u, 0x3c003c00u};
Require(name, "FCE before graphics shader or target binding",
ReadCachedTexel(name, context, color.image_id, {}, {1, 1, 1}, 1) == expected &&
ReadCachedTexel(name, context, color.image_id, {511, 255, 0},
{1, 1, 1}, 1) == expected &&
ReadCachedTexel(name, context, color.image_id, {}, {1, 1, 1}, 0) == stale,
"FCE lost a clear register or changed the unselected layer");
check_expanded_metadata();
paint();
RenderExecutorTestAccess::DrawIndex(executor, scheduler.Current(),
{.index_count = 3, .instance_count = 1});
(void)texture_cache.FindTexture(color.image_id, sampled);
Require(name, "repeated indexed FCE preserves expanded color",
ReadCachedTexel(name, context, color.image_id, {}, {1, 1, 1}, 1) == stale,
"FCE reapplied the consumed clear after rendering");
WriteMetadata(context, dcc_address, metadata_size, 0);
RenderExecutorTestAccess::DrawIndex(executor, scheduler.Current(),
{.index_count = 3, .instance_count = 1});
(void)texture_cache.FindTexture(color.image_id, sampled);
Require(name, "indexed FCE consumes a new clear",
ReadCachedTexel(name, context, color.image_id, {}, {1, 1, 1}, 1) == expected,
"indexed FCE skipped a new metadata clear");
resources.UnmapMemory(base, allocation_size);
scheduler.Finish();
return;
}
RenderExecutorTestAccess::ResolveRenderColorTarget(
executor, scheduler.Current(), color, 0);
RenderDepthInfo no_depth{};
const auto rendering = RenderExecutorTestAccess::AcquireRenderTargets(
executor, scheduler.Current(), &color, 1, no_depth);
if (fill_case.reuse_unorm) {
Require(name, "FLOAT clear reuses UNORM image",
color.image_id == preexisting_id &&
texture_cache.GetImage(preexisting_id).backing.format ==
vk::Format::eR16G16B16A16Unorm,
"the aliased clear did not reuse its existing UNORM allocation");
}
Require(name, "color clear metadata discovery",
color.image_id &&
color.desc.info.metadata.kind ==
(fill_case.cmask ? ImageMetadataKind::Cmask : ImageMetadataKind::Dcc) &&
color.desc.info.metadata.range.address == dcc_address &&
color.desc.info.metadata.range.size == metadata_size &&
rendering.num_color_attachments == 1 &&
!texture_cache.IsMeta(dcc_address),
"color metadata did not retain its type and full native allocation");
const auto read_texel = [&] {
return ReadCachedTexel(name, context, color.image_id, {}, {probe_width, 1, 1}, selected_layer);
};
Require(name, "GPU color clear value",
read_texel() == expected &&
ReadCachedTexel(name, context, color.image_id,
{static_cast<int32_t>(512 - probe_width), 255, 0},
{probe_width, 1, 1}, selected_layer) == expected,
"native metadata did not materialize the expected color");
if (fill_case.reuse_mips) {
const auto &image = texture_cache.GetImage(color.image_id);
Require(name, "single-mip clear reuses mip chain",
color.image_id == preexisting_id && image.info.resources.levels == 2 &&
color.desc.info.resources.levels == 1 &&
color.desc.view_info.base_level == 0 && color.desc.view_info.level_count == 1,
"the single-mip target did not retain its cached image and view scope");
Require(name, "single-mip clear preserves higher mip",
ReadCachedTexel(name, context, color.image_id, {255, 127, 0},
{1, 1, 1}, 0, 1) ==
std::vector<u32>{0x00003c00u, 0x00003c00u},
"clearing mip zero changed its higher mip");
}
check_expanded_metadata();
const auto bind = [&] {
RenderExecutorTestAccess::ResetBindings(executor);
color.image_id = texture_cache.FindImage(color.desc);
(void)texture_cache.FindRenderTarget(color.image_id, color.desc);
};
std::vector<u32> painted{0x00003c00u, 0x3c003c00u};
if (fill_case.reuse_unorm) {
painted = {0x0000ffffu, 0xffffffffu};
} else if (fill_case.cmask) {
painted = {0xffff00ffu};
} else if (bytes_per_pixel <= 4) {
const uint32_t one = type == Prospero::ChannelType::kFloat ? 0x3c00u : 0xffffu;
painted = {bytes_per_pixel == 2 ? one | (one << 16u) : one};
}
const auto retains_painted_texel = [&] {
const auto texels = ReadCachedTexel(name, context, color.image_id, {},
{512, 256, 1}, selected_layer);
for (size_t i = 0; i < texels.size(); i += painted.size()) {
if (std::equal(painted.begin(), painted.end(), texels.begin() + i)) {
return true;
}
}
return false;
};
paint();
bind();
Require(name, "unchanged metadata preserves rendering",
read_texel() == painted,
"rebinding unchanged color metadata reapplied an old clear");
if (fill_case.cmask) {
WriteMetadata(context, dcc_address, metadata_size, 0x40404040u);
bind();
Require(name, "nonzero CMASK is not a DCC clear", read_texel() == painted,
"CMASK bytes were decoded as a DCC fixed-color key");
}
WriteMetadata(context, dcc_address, metadata_size, fill_case.fill);
bind();
Require(name, "CPU metadata re-clear", read_texel() == expected,
"CPU metadata writes did not clear an already drawn target");
paint();
fill_metadata(metadata_words - 1);
bind();
Require(name, "partial same-value metadata fill",
retains_painted_texel(),
"a partial same-value write reused stale clear bytes after rendering");
fill_metadata(metadata_words, true);
bind();
Require(name, "same-value raw native re-clear",
read_texel() == expected,
"a repeated metadata fill did not clear an already drawn target");
paint();
WriteMetadata(context, dcc_address, metadata_size, UINT32_MAX);
fill_metadata(metadata_words - 1);
bind();
Require(name, "partial native metadata fill",
retains_painted_texel(),
"a nonuniform metadata allocation cleared the whole target");
fill_metadata(metadata_words);
bind();
Require(name, "complete fill after partial fill",
read_texel() == expected,
"completing the metadata overwrite did not restore its clear");
// Consuming a clear key updates metadata backing without invalidating
// a pooled image whose first texel precedes the metadata range.
constexpr uint32_t alias_value = 0x13579bdfu;
constexpr uint32_t expanded_words = 0x1000 / sizeof(uint32_t);
const auto selected_metadata = dcc_address + metadata_size - 0x1000;
ImageDesc metadata_alias{};
metadata_alias.info.data = {dcc_address - 0x1000, 0x4000};
metadata_alias.info.pixel_format = vk::Format::eR32Uint;
metadata_alias.info.guest_format = Prospero::BufferFormat::k32UInt;
metadata_alias.info.extent = {4096, 1, 1};
metadata_alias.info.pitch = 4096;
metadata_alias.info.bytes_per_block = sizeof(uint32_t);
metadata_alias.info.UpdateSize();
metadata_alias.view_info.format = vk::Format::eR32Uint;
const auto metadata_alias_id = texture_cache.FindImage(metadata_alias);
vk::ClearValue alias_paint{};
alias_paint.color.uint32[0] = alias_value;
TextureCacheTestAccess::ClearImage(
texture_cache, scheduler.Current(), metadata_alias_id,
{vk::ImageAspectFlagBits::eColor, 0, 1, 0, 1}, alias_paint);
paint();
fill_metadata(metadata_words);
bind();
Require(name, "pooled metadata clear target", read_texel() == expected,
"the overlapping native image prevented the metadata clear");
auto metadata_source = resources.GetBufferCache().ObtainBuffer(
selected_metadata, 0x1000, false);
Require(name, "expanded metadata buffer source", metadata_source.first != nullptr,
"expanded metadata did not produce a GPU-readable buffer");
auto metadata_readback = CreateHostBuffer(
name, 0x1000, vk::BufferUsageFlagBits::eTransferDst, {});
vk::MemoryBarrier metadata_barrier{};
metadata_barrier.srcAccessMask = vk::AccessFlagBits::eMemoryWrite;
metadata_barrier.dstAccessMask = vk::AccessFlagBits::eTransferRead;
scheduler.Current().Handle().pipelineBarrier(
vk::PipelineStageFlagBits::eAllCommands, vk::PipelineStageFlagBits::eTransfer,
{}, 1, &metadata_barrier, 0, nullptr, 0, nullptr);
const vk::BufferCopy metadata_copy{metadata_source.second, 0, 0x1000};
scheduler.Current().Handle().copyBuffer(metadata_source.first->Handle(),
metadata_readback.buffer, 1, &metadata_copy);
metadata_barrier.srcAccessMask = vk::AccessFlagBits::eTransferWrite;
metadata_barrier.dstAccessMask = vk::AccessFlagBits::eHostRead;
scheduler.Current().Handle().pipelineBarrier(
vk::PipelineStageFlagBits::eTransfer, vk::PipelineStageFlagBits::eHost,
{}, 1, &metadata_barrier, 0, nullptr, 0, nullptr);
scheduler.Finish();
Require(name, "expanded metadata GPU reader",
ReadBuffer(name, metadata_readback, expanded_words) ==
std::vector<u32>(expanded_words, UINT32_MAX),
"a later GPU buffer read retained the consumed clear key");
DestroyBuffer(&metadata_readback);
(void)texture_cache.FindTexture(metadata_alias_id, metadata_alias);
Require(name, "metadata consumption preserves pooled image",
ReadCachedTexel(name, context, metadata_alias_id) ==
std::vector<u32>{alias_value},
"internal metadata publication replaced an untouched GPU image texel");
ImageDesc previous_depth{};
previous_depth.type = BindingType::DepthTarget;
previous_depth.info.data = {base + 0x180000, 0x10000};
previous_depth.info.pixel_format = vk::Format::eD32Sfloat;
previous_depth.info.guest_format = Prospero::BufferFormat::k32Float;
previous_depth.info.type = Prospero::ImageType::kColor2D;
previous_depth.info.extent = {128, 128, 1};
previous_depth.info.resources = {1, 1};
previous_depth.info.pitch = 128;
previous_depth.info.bytes_per_block = 4;
previous_depth.info.samples = 1;
previous_depth.info.tile_mode = Prospero::TileMode::kLinear;
previous_depth.info.UpdateSize();
previous_depth.info.metadata.kind = ImageMetadataKind::Htile;
previous_depth.info.metadata.range = {dcc_address, metadata_size};
previous_depth.info.htile_initial_clear = false;
previous_depth.view_info.format = vk::Format::eD32Sfloat;
previous_depth.view_info.type = vk::ImageViewType::e2D;
previous_depth.view_info.aspect = vk::ImageAspectFlagBits::eDepth;
previous_depth.view_info.usage = vk::ImageUsageFlagBits::eDepthStencilAttachment;
const auto previous_depth_id = texture_cache.FindImage(previous_depth);
Require(name, "old HTile interpretation",
texture_cache.FindDepthTarget(previous_depth_id, previous_depth) != nullptr &&
texture_cache.IsMeta(dcc_address),
"the old depth target did not register its metadata allocation");
paint();
bind();
Require(name, "HTile allocation reused as expanded color metadata",
!texture_cache.IsMeta(dcc_address) && !texture_cache.ClearMeta(dcc_address) &&
read_texel() == painted,
"color discovery retained the old HTile classification or erased rendering");
fill_metadata(metadata_words);
bind();
Require(name, "native color clear after HTile reuse", read_texel() == expected,
"the former HTile entry swallowed the native color metadata fill");
check_expanded_metadata();
RenderExecutorTestAccess::ResetBindings(executor);
resources.UnmapMemory(base, allocation_size);
scheduler.Finish();
});
context.ShutdownGpu();
LibKernel::Memory::InstallGpuResources(nullptr);
}
Require(name, "unmap direct backing",
Libs::LibKernel::Memory::KernelMunmap(base, allocation_size) == 0,
"fixed-clear direct mapping release failed");
Require(name, "release direct backing",
Libs::LibKernel::Memory::KernelReleaseDirectMemory(
direct_offset, allocation_size) == 0,
"fixed-clear direct-memory allocation release failed");
std::printf("[gpu] %-32s ok\n", name);
}
void CheckSampledDccClear() {
constexpr const char *name = "SampledDccClear";
constexpr uintptr_t base = 0x0000000205000000ull;
constexpr uint64_t color_size = 0x3fc0000;
constexpr uint64_t metadata_size = 0x48000;
constexpr uint64_t dcc_address = base + color_size;
constexpr uint64_t allocation_size = 0x4020000;
constexpr uint64_t alignment = 0x10000;
EnsureRuntimeContext();
int64_t direct_offset = -1;
Require(name, "direct allocation",
Libs::LibKernel::Memory::KernelAllocateDirectMemory(
0, Libs::LibKernel::Memory::KernelGetDirectMemorySize(),
allocation_size, alignment, 0, &direct_offset) == 0,
"captured scene allocation failed");
void *mapped = reinterpret_cast<void *>(base);
Require(name, "direct mapping",
Libs::LibKernel::Memory::KernelMapDirectMemory(
&mapped, allocation_size, 0x3, 0x10, direct_offset, alignment) == 0 &&
mapped == reinterpret_cast<void *>(base),
"captured scene mapping failed");
for (const auto initial_size : {metadata_size - 0x1000, metadata_size}) {
std::memset(mapped, 0x5a, color_size);
std::memset(reinterpret_cast<void *>(dcc_address), 0xff, metadata_size);
RenderContext context(m_runtime_context);
context.InitializeGpu(nullptr);
LibKernel::Memory::InstallGpuResources(&context);
auto &scheduler = context.GetCommandScheduler();
HW::Context registers{};
HW::UserConfig user_config{};
HW::Shader shaders{};
scheduler.Begin(registers, user_config, shaders);
auto &resources = context;
auto &cache = resources.GetTextureCache();
auto &executor = context.GetRenderExecutor();
resources.MapMemory(base, allocation_size);
WriteMetadata(context, dcc_address, initial_size, 0x40404040u);
Require(name, "deferred metadata fill",
!cache.IsMeta(dcc_address),
"unknown metadata was consumed before descriptor discovery");
// PPSA19244's first sampled scene descriptor, with relocated allocations.
ShaderTextureResource descriptor{{static_cast<uint32_t>(base >> 8u),
0xc4700000u, 0x021bc3bfu, 0x91b00facu, 0, 0x00700000u,
0x006b0000u | (static_cast<uint32_t>(dcc_address >> 8u) << 24u),
static_cast<uint32_t>(dcc_address >> 16u)}};
ShaderRecompiler::IR::DescriptorValue value{};
value.dword_count = 8;
std::copy_n(descriptor.fields, 8, value.dwords.begin());
ShaderRecompiler::IR::ImageResource resource{};
resource.resource_class = ShaderRecompiler::IR::ImageResourceClass::Sampled;
resource.numeric_class = Prospero::TextureNumericClass::Float;
resource.dimension = ShaderRecompiler::Decoder::ImageDimension::Dim2D;
resource.read = true;
auto binding = RenderExecutorTestAccess::ResolveTexture(executor, resource, value);
Require(name, "sampled metadata discovery",
binding.desc.info.extent == vk::Extent3D{3840, 2160, 1} &&
binding.desc.info.data.size == color_size &&
binding.desc.info.metadata.kind == ImageMetadataKind::Dcc &&
binding.desc.info.metadata.range.address == dcc_address &&
binding.desc.info.metadata.range.size == metadata_size,
"captured sampled descriptor lost its DCC allocation");
Require(name, "sampled view", cache.FindTexture(binding.image_id, binding.desc) != nullptr,
"captured scene did not produce a sampled view");
if (initial_size != metadata_size) {
Require(name, "partial metadata coverage",
ReadCachedTexel(name, context, binding.image_id) ==
std::vector<u32>{0x5a5a5a5au, 0x5a5a5a5au},
"an incomplete metadata fill cleared the whole scene");
WriteMetadata(context, dcc_address, metadata_size, 0x40404040u);
binding.image_id = cache.FindImage(binding.desc);
(void)cache.FindTexture(binding.image_id, binding.desc);
}
Require(name, "opaque black before first sample",
ReadCachedTexel(name, context, binding.image_id, {3839, 2159, 0}) ==
std::vector<u32>{0, 0x3c000000u},
"sampling read stale color bytes instead of the DCC opaque black clear");
context.GetBufferCache().ReadMemory(dcc_address, metadata_size);
std::vector<uint8_t> expanded_metadata(metadata_size);
Require(name, "sampled clear expands native metadata",
LibKernel::Memory::TryReadBacking(dcc_address, expanded_metadata.data(),
expanded_metadata.size()) &&
std::all_of(expanded_metadata.begin(), expanded_metadata.end(),
[](uint8_t byte) { return byte == 0xff; }),
"the sampled clear retained native clear keys after materialization");
WriteMetadata(context, dcc_address, metadata_size, 0x80808080u);
binding.image_id = cache.FindImage(binding.desc);
(void)cache.FindTexture(binding.image_id, binding.desc);
Require(name, "later sampled clear",
ReadCachedTexel(name, context, binding.image_id) ==
std::vector<u32>{0x3c003c00u, 0x00003c00u},
"an existing sampled image retained the previous DCC clear");
resources.UnmapMemory(base, allocation_size);
scheduler.Finish();
context.ShutdownGpu();
LibKernel::Memory::InstallGpuResources(nullptr);
}
{
// PPSA01593 samples a DCC R32 color surface through a comparison sampler.
constexpr uint64_t comparison_dcc = base + 0x10000;
std::memset(mapped, 0, 0x10000);
std::memset(reinterpret_cast<void *>(comparison_dcc), 0xff, 0x1000);
RenderContext context(m_runtime_context);
context.InitializeGpu(nullptr);
LibKernel::Memory::InstallGpuResources(&context);
auto &scheduler = context.GetCommandScheduler();
HW::Context registers{};
HW::UserConfig user_config{};
HW::Shader shaders{};
scheduler.Begin(registers, user_config, shaders);
context.MapMemory(base, allocation_size);
auto &cache = context.GetTextureCache();
auto &executor = context.GetRenderExecutor();
const ShaderTextureResource descriptor{{static_cast<uint32_t>(base >> 8u),
0xc1600000u, 0x000fc00fu, 0x91b00204u, 0, 0x00700000u,
0x002b0000u | (static_cast<uint32_t>(comparison_dcc >> 8u) << 24u),
static_cast<uint32_t>(comparison_dcc >> 16u)}};
ShaderRecompiler::IR::DescriptorValue value{};
value.dword_count = 8;
std::copy_n(descriptor.fields, 8, value.dwords.begin());
ShaderRecompiler::IR::ImageResource resource{};
resource.resource_class = ShaderRecompiler::IR::ImageResourceClass::Storage;
resource.numeric_class = Prospero::TextureNumericClass::Float;
resource.dimension = ShaderRecompiler::Decoder::ImageDimension::Dim2D;
resource.written = true;
const auto color = RenderExecutorTestAccess::ResolveTexture(executor, resource, value);
(void)cache.FindTexture(color.image_id, color.desc);
vk::ClearValue paint{};
paint.color.float32 = std::array{0.375f, 0.f, 0.f, 0.f};
TextureCacheTestAccess::ClearImage(cache, scheduler.Current(), color.image_id,
{vk::ImageAspectFlagBits::eColor, 0, 1, 0, 1}, paint);
resource.resource_class = ShaderRecompiler::IR::ImageResourceClass::Sampled;
resource.written = false;
resource.read = true;
resource.depth_compare = true;
auto comparison = RenderExecutorTestAccess::ResolveTexture(executor, resource, value);
(void)cache.FindTexture(comparison.image_id, comparison.desc);
const auto &depth = cache.GetImage(comparison.image_id);
Require(name, "expanded DCC comparison alias",
comparison.image_id != color.image_id &&
depth.info.pixel_format == vk::Format::eD32Sfloat &&
depth.info.guest_format == Prospero::BufferFormat::k32Float &&
depth.info.tile_mode == Prospero::TileMode::kRenderTarget &&
depth.info.extent == vk::Extent3D{64, 64, 1} &&
depth.info.metadata.kind == ImageMetadataKind::Dcc &&
depth.info.metadata.range == GuestRange{comparison_dcc, 0x1000} &&
ReadCachedTexel(name, context, comparison.image_id) ==
std::vector<u32>{std::bit_cast<u32>(0.375f)},
"comparison promotion lost native color data or guest DCC layout");
for (const u32 code : {0x40404040u, 0x80808080u}) {
WriteMetadata(context, comparison_dcc, 0x1000, code);
comparison = RenderExecutorTestAccess::ResolveTexture(executor, resource, value);
(void)cache.FindTexture(comparison.image_id, comparison.desc);
Require(name, "DCC clear on a comparison backing",
ReadCachedTexel(name, context, comparison.image_id, {63, 63, 0}) ==
std::vector<u32>{code == 0x40404040u ? 0u : 0x3f800000u},
"DCC clear used the native depth format or color aspect");
std::array<uint8_t, 0x1000> metadata{};
Require(name, "comparison clear expands metadata",
LibKernel::Memory::TryReadBacking(comparison_dcc, metadata.data(),
metadata.size()) &&
std::ranges::all_of(metadata, [](uint8_t byte) { return byte == 0xff; }),
"comparison clear retained the native DCC clear key");
}
context.UnmapMemory(base, allocation_size);
scheduler.Finish();
context.ShutdownGpu();
LibKernel::Memory::InstallGpuResources(nullptr);
}
Require(name, "unmap",
Libs::LibKernel::Memory::KernelMunmap(base, allocation_size) == 0,
"captured scene mapping release failed");
Require(name, "release",
Libs::LibKernel::Memory::KernelReleaseDirectMemory(
direct_offset, allocation_size) == 0,
"captured scene allocation release failed");
std::printf("[gpu] %-32s ok\n", name);
}
void CheckPackedTextureComponents() {
constexpr const char *name = "PackedTextureComponents";
constexpr uintptr_t base = 0x0000000204200000ull;
constexpr uint64_t allocation_size = 0x10000;
struct FormatCase {
Prospero::BufferFormat format;
std::array<uint16_t, 4> words;
std::array<std::array<uint32_t, 4>, 4> channels;
std::array<uint32_t, 4> maxima;
};
// Packed format channels start at the least significant bit.
// PPSA24515's blue, white, and masked black exposed the 5551 reversal.
constexpr std::array cases{
FormatCase{Prospero::BufferFormat::k5_5_5_1UNorm,
{0x7c00, 0x7fff, 0x8000, 0x8c41},
{{{0, 0, 31, 0}, {31, 31, 31, 0}, {0, 0, 0, 1}, {1, 2, 3, 1}}},
{31, 31, 31, 1}},
FormatCase{Prospero::BufferFormat::k4_4_4_4UNorm,
{0x000f, 0x00f0, 0x0f00, 0x4321},
{{{15, 0, 0, 0}, {0, 15, 0, 0}, {0, 0, 15, 0}, {1, 2, 3, 4}}},
{15, 15, 15, 15}},
};
const auto argb = TextureGetComponentMapping(DstSel(5, 1, 7, 0),
Prospero::ColorMappingArgb);
Require(name, "inverse mapping",
argb == vk::ComponentMapping{vk::ComponentSwizzle::eB,
vk::ComponentSwizzle::eOne, vk::ComponentSwizzle::eR,
vk::ComponentSwizzle::eZero},
"cyclic host mapping reordered output slots or descriptor constants");
const auto target = TextureGetRenderTargetFormat(
Prospero::ChannelLayout::k1_5_5_5, Prospero::ChannelType::kUNorm,
Prospero::ChannelOrder::kStandard);
Require(name, "1555 render target",
target.format == vk::Format::eR5G5B5A1UnormPack16 &&
target.export_mapping == Prospero::ColorMappingAbgr,
"shared format resolution changed the existing 1555 render target");
for (const auto native : {target, TextureGetRenderTargetFormat(
Prospero::ChannelLayout::k8,
Prospero::ChannelType::kSNorm,
Prospero::ChannelOrder::kStandard)}) {
const auto upload =
MakeTilingImage(native.guest_format, 4, 1, 1, 1,
Prospero::TileMode::kLinear, 256, false);
Require(
name, "render-target guest upload layout",
upload.pitch * native.bytes_per_element == 256 &&
upload.mip_layout[0].linear_size == 256,
"retaining the render-target format lost its physical byte width");
}
EnsureRuntimeContext();
int64_t direct_offset = -1;
Require(name, "allocation",
Libs::LibKernel::Memory::KernelAllocateDirectMemory(
0, Libs::LibKernel::Memory::KernelGetDirectMemorySize(),
allocation_size, allocation_size, 0, &direct_offset) == 0,
"packed texture allocation failed");
void *mapped = reinterpret_cast<void *>(base);
Require(name, "mapping",
Libs::LibKernel::Memory::KernelMapDirectMemory(
&mapped, allocation_size, 0x3, 0x10, direct_offset,
allocation_size) == 0 && mapped == reinterpret_cast<void *>(base),
"packed texture mapping failed");
for (const auto &format : cases) {
std::memset(mapped, 0, allocation_size);
std::memcpy(mapped, format.words.data(), sizeof(format.words));
RenderContext context(m_runtime_context);
auto &scheduler = context.GetCommandScheduler();
HW::Context registers{};
HW::UserConfig user_config{};
HW::Shader shaders{};
scheduler.Begin(registers, user_config, shaders);
auto &resources = context;
auto &cache = resources.GetTextureCache();
auto &executor = context.GetRenderExecutor();
resources.MapMemory(base, allocation_size);
for (const auto swizzle : {DstSel(4, 5, 6, 7), DstSel(7, 1, 4, 0)}) {
ShaderTextureResource descriptor{{static_cast<uint32_t>(base >> 8u),
(static_cast<uint32_t>(format.format) << 20u) | (3u << 30u), 0,
swizzle | (static_cast<uint32_t>(Prospero::ImageType::kColor2D) << 28u),
0, 0x00700000u, 0, 0}};
TestCase test;
test.name = name;
test.has_user_data = true;
test.image_descriptor_swizzle = swizzle;
std::copy_n(descriptor.fields, 8, test.user_data.begin());
test.user_data[50] = 16 * sizeof(uint32_t);
test.opcodes = {ShaderOpcode::V_MOV_B32, ShaderOpcode::IMAGE_LOAD,
ShaderOpcode::BUFFER_STORE_DWORD, ShaderOpcode::S_ENDPGM};
for (uint32_t pixel = 0; pixel < format.words.size(); ++pixel) {
AppendVMovU32(&test.code, 20, pixel);
AppendVMovU32(&test.code, 21, 0);
test.code.push_back(EncodeMimg0(0x00, 0xf));
test.code.push_back(EncodeMimg1(0, 20));
for (uint32_t channel = 0; channel < 4; ++channel) {
AppendStoreVgpr(&test.code, channel, pixel * 4 + channel);
}
}
AppendEnd(&test.code);
const auto compiled = CompileCase(test, SubgroupSize());
ShaderRecompiler::IR::DescriptorValue value{};
value.dword_count = 8;
std::copy_n(descriptor.fields, 8, value.dwords.begin());
const auto binding = RenderExecutorTestAccess::ResolveTexture(
executor, compiled.program.info.images.at(0), value);
Image sampled;
sampled.view = cache.FindTexture(binding.image_id, binding.desc);
sampled.layout = cache.GetImage(binding.image_id).backing.state.layout;
scheduler.Finish();
auto output = CreateStorageBuffer(name, {}, 16);
Dispatch(test, compiled, output, nullptr, &sampled);
const auto actual = ReadBuffer(name, output, 16);
for (uint32_t pixel = 0; pixel < format.words.size(); ++pixel) {
for (uint32_t channel = 0; channel < 4; ++channel) {
const auto selector = GetDstSel(swizzle, channel);
const auto maximum = selector < 4 ? 1u : format.maxima[selector - 4];
const auto expected = selector < 4 ? selector : format.channels[pixel][selector - 4];
const float observed = std::bit_cast<float>(actual[pixel * 4 + channel]);
// Recover the source bits without testing the host's UNORM rounding precision.
const bool matches = maximum == 1 ? observed == expected
: observed >= 0.0f && observed <= 1.0f &&
std::round(observed * maximum) == expected;
if (!matches) {
std::ostringstream message;
message << "format=" << static_cast<uint32_t>(format.format)
<< " swizzle=" << swizzle << " word=" << format.words[pixel]
<< " channel=" << channel << " expected=" << expected
<< " actual=" << observed;
Fail(name, "GPU components", message.str());
}
}
}
DestroyBuffer(&output);
}
RenderExecutorTestAccess::ResetBindings(executor);
resources.UnmapMemory(base, allocation_size);
scheduler.Finish();
}
for (const auto tile : {Prospero::TileMode::kLinear, Prospero::TileMode::kRenderTarget}) {
constexpr std::array<u32, 4> words{0x883005e1u, 0x7c0e81c0u,
0x800f0000u, 0x98120280u};
constexpr std::array<std::array<float, 4>, 4> channels{{
{1.03125f, 2.03125f, 4.0625f, 1.f}, {0.5f, 0.75f, 1.5f, 1.f},
{0.f, 1.f, 2.f, 1.f}, {32.f, 8.f, 16.f, 1.f}}};
const auto target = TextureGetRenderTargetFormat(
Prospero::ChannelLayout::k10_11_11, Prospero::ChannelType::kFloat,
Prospero::ChannelOrder::kReversed);
Require(name, "10_11_11 reversed render target",
target.format == vk::Format::eB10G11R11UfloatPack32 &&
target.guest_format == Prospero::BufferFormat::k10_11_11Float &&
target.export_mapping == Prospero::ColorMappingRgba,
"PPSA24156 reversed packed-float target lost its physical encoding");
std::memset(mapped, 0, allocation_size);
std::array<u32, 4> offsets{0, 4, 8, 12};
if (tile == Prospero::TileMode::kRenderTarget) {
TileBlockLayout block{};
u32 block_xor = 0;
Require(name, "10_11_11 tile layout",
TileGetBlockLayout(TileBlockFamily::RenderTarget64KB, 4, block) &&
TileGetBlockXor(block, 0, 0, 0, block_xor),
"packed-float render-target layout is unavailable");
for (u32 pixel = 0; pixel < words.size(); ++pixel) {
Require(name, "10_11_11 tile offset",
TileGetBlockOffset(block, pixel, 0, 0, offsets[pixel]),
"packed-float texel offset is unavailable");
offsets[pixel] ^= block_xor;
}
}
for (u32 pixel = 0; pixel < words.size(); ++pixel) {
std::memcpy(static_cast<uint8_t *>(mapped) + offsets[pixel], &words[pixel], sizeof(u32));
}
RenderContext context(m_runtime_context);
auto &scheduler = context.GetCommandScheduler();
HW::Context registers{};
HW::UserConfig user_config{};
HW::Shader shaders{};
scheduler.Begin(registers, user_config, shaders);
context.MapMemory(base, allocation_size);
auto &cache = context.GetTextureCache();
auto &executor = context.GetRenderExecutor();
constexpr u32 swizzle = DstSel(4, 5, 6, 1);
ShaderTextureResource descriptor{{static_cast<u32>(base >> 8u),
(static_cast<u32>(Prospero::BufferFormat::k10_11_11Float) << 20u) |
(3u << 30u), 0,
swizzle | (static_cast<u32>(tile) << 20u) |
(static_cast<u32>(Prospero::ImageType::kColor2D) << 28u),
0, 0x00700000u, 0, 0}};
TestCase test;
test.name = name;
test.has_user_data = true;
test.image_descriptor_swizzle = swizzle;
std::copy_n(descriptor.fields, 8, test.user_data.begin());
test.user_data[50] = 16 * sizeof(u32);
for (u32 pixel = 0; pixel < words.size(); ++pixel) {
AppendVMovU32(&test.code, 20, pixel);
AppendVMovU32(&test.code, 21, 0);
test.code.push_back(EncodeMimg0(0x00, 0xf));
test.code.push_back(EncodeMimg1(0, 20));
for (u32 channel = 0; channel < 4; ++channel) {
AppendStoreVgpr(&test.code, channel, pixel * 4 + channel);
test.expected.push_back(std::bit_cast<u32>(channels[pixel][channel]));
}
}
AppendEnd(&test.code);
const auto compiled = CompileCase(test, SubgroupSize());
ShaderRecompiler::IR::DescriptorValue value{};
value.dword_count = 8;
std::copy_n(descriptor.fields, 8, value.dwords.begin());
const auto binding = RenderExecutorTestAccess::ResolveTexture(
executor, compiled.program.info.images.at(0), value);
Image sampled;
sampled.view = cache.FindTexture(binding.image_id, binding.desc);
sampled.layout = cache.GetImage(binding.image_id).backing.state.layout;
scheduler.Finish();
auto output = CreateStorageBuffer(name, {}, 16);
Dispatch(test, compiled, output, nullptr, &sampled);
Require(name, "10_11_11 uploaded components",
ReadBuffer(name, output, 16) == test.expected,
"packed-float upload or sample changed a component's width or order");
DestroyBuffer(&output);
const auto clear = [&](std::array<float, 4> channels) {
auto &native = cache.GetImage(binding.image_id);
native.Transit(vk::ImageLayout::eTransferDstOptimal,
vk::AccessFlagBits2::eTransferWrite, {},
scheduler.Current().Handle());
const vk::ImageSubresourceRange range{vk::ImageAspectFlagBits::eColor, 0, 1, 0, 1};
scheduler.Current().Handle().clearColorImage(native.backing.image,
vk::ImageLayout::eTransferDstOptimal, vk::ClearColorValue{channels}, range);
cache.MarkGpuWritten(binding.image_id);
};
clear({8.f, 4.f, 2.f, 1.f});
Require(name, "10_11_11 guest download",
TextureCacheTestAccess::TryDownload(cache, binding.image_id),
"packed-float native contents could not be downloaded");
scheduler.Finish();
scheduler.DrainPriorityOperations();
std::array<u32, 4> guest{};
for (u32 pixel = 0; pixel < words.size(); ++pixel) {
Require(name, "10_11_11 guest texel",
Libs::LibKernel::Memory::TryReadBacking(base + offsets[pixel],
&guest[pixel], sizeof(u32)),
"packed-float guest texel is inaccessible");
}
Require(name, "10_11_11 exact guest words",
guest == std::array<u32, 4>{0x90110200u, 0x90110200u, 0x90110200u, 0x90110200u},
"native packed-float write lost the guest 10/11/11 bit layout");
clear({16.f, 8.f, 4.f, 1.f});
auto alias_desc = binding.desc;
alias_desc.info.guest_format = Prospero::BufferFormat::k11_11_10Float;
alias_desc.view_info.mapping = {};
Require(name, "same-format transform compatibility",
!alias_desc.info.IsCompatible(binding.desc.info),
"equal Vulkan formats hid incompatible guest color transforms");
const auto alias = cache.FindImage(alias_desc);
(void)cache.FindTexture(alias, alias_desc);
Require(name, "same-format transformed alias",
alias != binding.image_id &&
ReadCachedTexel(name, context, alias, {}, {4, 1, 1}) ==
std::vector<u32>(4, 0x98120220u),
"packed-float alias reused a transformed image or bypassed guest bytes");
RenderExecutorTestAccess::ResetBindings(executor);
context.UnmapMemory(base, allocation_size);
scheduler.Finish();
}
Require(name, "release",
Libs::LibKernel::Memory::KernelMunmap(base, allocation_size) == 0 &&
Libs::LibKernel::Memory::KernelReleaseDirectMemory(
direct_offset, allocation_size) == 0,
"packed texture backing release failed");
std::printf("[gpu] %-32s ok\n", name);
}
void CheckSamplerBorderColors() {
constexpr const char *name = "SamplerBorderColors";
auto &context = RuntimeRenderer();
constexpr std::array<std::array<float, 2>, 5> coordinates{{
{{0.5f, -1.f}}, {{-1.f, 0.5f}}, {{2.f, 0.5f}},
{{0.5f, 2.f}}, {{0.5f, 0.5f}}}};
std::array<vk::Sampler, 3> float_samplers{};
for (const bool integer : {false, true}) {
auto image = CreateImage2D(name, 2, 2,
integer ? vk::Format::eR8G8B8A8Uint : vk::Format::eR8G8B8A8Unorm,
vk::ImageUsageFlagBits::eSampled, std::vector<u32>(4, 0xffffffffu), 1,
vk::ImageLayout::eShaderReadOnlyOptimal);
for (const auto border : {Prospero::SamplerBorderColor::kOpaqueWhite,
Prospero::SamplerBorderColor::kOpaqueBlack,
Prospero::SamplerBorderColor::kTransBlack}) {
ShaderSamplerResource sampler_descriptor{};
const auto clamp = static_cast<u32>(Prospero::SamplerClampMode::kClampBorder);
sampler_descriptor.fields[0] = clamp | (clamp << 3u);
sampler_descriptor.fields[2] = 1u << 24u;
sampler_descriptor.fields[3] = static_cast<u32>(border) << 30u;
TestCase test;
test.name = name;
test.has_user_data = true;
test.user_data = MakeSampledTextureData(integer ? Prospero::BufferFormat::k8_8_8_8UInt
: Prospero::BufferFormat::k8_8_8_8UNorm);
test.user_data[2] = 1u | (1u << 14u);
std::copy_n(sampler_descriptor.fields, 4, test.user_data.begin() + 8);
test.user_data[50] = coordinates.size() * 4u * sizeof(u32);
test.user_data[51] = 3u << 28u;
test.opcodes = {ShaderOpcode::V_MOV_B32, ShaderOpcode::IMAGE_SAMPLE,
ShaderOpcode::BUFFER_STORE_DWORD, ShaderOpcode::S_ENDPGM};
for (u32 sample = 0; sample < coordinates.size(); ++sample) {
AppendVMovLiteral(&test.code, 20, std::bit_cast<u32>(coordinates[sample][0]));
AppendVMovLiteral(&test.code, 21, std::bit_cast<u32>(coordinates[sample][1]));
test.code.push_back(EncodeMimg0(0x20, 0xf));
test.code.push_back(EncodeMimg1(0, 20, 0, 2));
for (u32 channel = 0; channel < 4; ++channel) {
AppendStoreVgpr(&test.code, channel, sample * 4u + channel);
const bool inside = sample == coordinates.size() - 1u;
const bool one = inside || border == Prospero::SamplerBorderColor::kOpaqueWhite ||
(border == Prospero::SamplerBorderColor::kOpaqueBlack && channel == 3u);
test.expected.push_back(one ? (integer ? (inside ? 255u : 1u) : 0x3f800000u) : 0u);
}
}
AppendEnd(&test.code);
const auto compiled = CompileCase(test, SubgroupSize());
const auto &sampler_info = compiled.program.info.samplers[0];
Require(name, "sampler specialization", sampler_info.integer_border == integer,
"sampled image numeric type did not select its sampler border class");
auto output = CreateStorageBuffer(name, {}, test.expected.size());
const auto sampler = context.GetSamplerCache().GetSampler(
sampler_descriptor, sampler_info.integer_border);
Require(name, "sampler reuse",
sampler == context.GetSamplerCache().GetSampler(sampler_descriptor, integer),
"identical typed sampler descriptors did not reuse the cached sampler");
if (integer) {
Require(name, "typed border cache key", sampler != float_samplers[static_cast<u32>(border)],
"integer image reused the floating-point border sampler");
} else {
float_samplers[static_cast<u32>(border)] = sampler;
}
Dispatch(test, compiled, output, nullptr, &image, nullptr, nullptr, sampler);
const auto actual = ReadBuffer(name, output, test.expected.size());
for (u32 i = 0; i < actual.size(); ++i) {
if (actual[i] != test.expected[i]) {
Fail(name, "native border sample", "integer=" + std::to_string(integer) +
" border=" + std::to_string(static_cast<u32>(border)) + " word=" + std::to_string(i) +
" expected=" + std::to_string(test.expected[i]) + " actual=" + std::to_string(actual[i]));
}
}
DestroyBuffer(&output);
}
DestroyImage(&image);
}
std::printf("[gpu] %-32s ok\n", name);
}
void CheckComparisonDepthTexture() {
constexpr const char *name = "ComparisonDepthTexture";
constexpr uintptr_t base = 0x0000000204200000ull;
constexpr uint64_t allocation_size = 0x20000;
constexpr uint64_t ordinary_address = base + 0x10000;
constexpr std::array<uint16_t, 6> depths{0, 13107, 26214, 39321, 52428, 65535};
constexpr std::array<float, 5> references{0.1f, 0.3f, 0.5f, 0.7f, 0.9f};
EnsureRuntimeContext();
int64_t direct_offset = -1;
Require(name, "direct allocation",
Libs::LibKernel::Memory::KernelAllocateDirectMemory(
0, Libs::LibKernel::Memory::KernelGetDirectMemorySize(),
allocation_size, allocation_size, 0, &direct_offset) == 0,
"comparison texture allocation failed");
void *mapped = reinterpret_cast<void *>(base);
Require(name, "direct mapping",
Libs::LibKernel::Memory::KernelMapDirectMemory(
&mapped, allocation_size, 0x3, 0x10, direct_offset,
allocation_size) == 0 && mapped == reinterpret_cast<void *>(base),
"comparison texture mapping failed");
// Sea of Stars' 1D comparison texture and the existing six-face cube.
const auto check_layout = [&](bool cube, uint32_t swizzle) {
const uint32_t layers = cube ? depths.size() : 1;
const auto depth_at = [&](uint32_t layer, bool reversed) {
const auto index = cube ? layer : 2u;
return depths[reversed ? depths.size() - 1u - index : index];
};
std::memset(mapped, 0, allocation_size);
for (uint32_t face = 0; face < layers; face++) {
const auto depth = depth_at(face, false);
for (const auto address : {static_cast<uint64_t>(base), ordinary_address}) {
std::memcpy(reinterpret_cast<void *>(address + face * 0x100),
&depth, sizeof(depth));
}
}
RenderContext context(m_runtime_context);
auto &scheduler = context.GetCommandScheduler();
HW::Context registers{};
HW::UserConfig user_config{};
HW::Shader shaders{};
scheduler.Begin(registers, user_config, shaders);
auto &resources = context;
auto &cache = resources.GetTextureCache();
auto &executor = context.GetRenderExecutor();
resources.MapMemory(base, allocation_size);
ShaderTextureResource descriptor{{static_cast<uint32_t>(base >> 8u),
0x00700000u, 0, (cube ? 0xb0100000u : 0x81800000u) | swizzle,
layers - 1u, 0x00700000u, 0, 0}};
ShaderSamplerResource sampler_descriptor{{1u << 12u, 0, 1u << 24u, 0}};
TestCase test;
test.name = name;
test.has_user_data = true;
test.image_descriptor_swizzle = descriptor.DstSelXYZW();
std::copy_n(descriptor.fields, 8, test.user_data.begin());
std::copy_n(sampler_descriptor.fields, 4, test.user_data.begin() + 8);
test.user_data[50] = layers * references.size() * sizeof(uint32_t);
test.opcodes = {ShaderOpcode::V_MOV_B32, ShaderOpcode::IMAGE_SAMPLE,
ShaderOpcode::BUFFER_STORE_DWORD, ShaderOpcode::S_ENDPGM};
test.required_spirv = {"OpImageSampleDrefExplicitLod"};
std::vector<uint32_t> promoted_expected;
for (uint32_t face = 0; face < layers; face++) {
for (const auto reference : references) {
AppendVMovLiteral(&test.code, 20, std::bit_cast<uint32_t>(reference));
AppendVMovLiteral(&test.code, 21,
std::bit_cast<uint32_t>(cube ? 1.5f : 0.5f));
if (cube) {
AppendVMovLiteral(&test.code, 22, std::bit_cast<uint32_t>(1.5f));
AppendVMovLiteral(&test.code, 23,
std::bit_cast<uint32_t>(static_cast<float>(face)));
}
test.code.push_back(EncodeMimg0(0x2f, 1, 0, false, cube ? 3 : 0));
test.code.push_back(EncodeMimg1(0, 20, 0, 2));
AppendStoreVgpr(&test.code, 0, static_cast<uint32_t>(test.expected.size()));
test.expected.push_back(
reference < static_cast<float>(depth_at(face, false)) / 65535.f
? 0x3f800000u : 0u);
promoted_expected.push_back(
reference < static_cast<float>(depth_at(face, true)) / 65535.f
? 0x3f800000u : 0u);
}
}
AppendEnd(&test.code);
const auto compiled = CompileCase(test, SubgroupSize());
Require(name, "comparison resource", compiled.program.info.images.size() == 1 &&
compiled.program.info.images[0].depth_compare &&
compiled.program.info.images[0].cube == cube &&
compiled.program.info.images[0].dimension ==
(cube ? ShaderRecompiler::Decoder::ImageDimension::Dim2DArray
: ShaderRecompiler::Decoder::ImageDimension::Dim1D),
"native comparison did not retain its resource semantics");
ShaderRecompiler::IR::DescriptorValue value{};
value.dword_count = 8;
std::copy_n(descriptor.fields, 8, value.dwords.begin());
auto binding = RenderExecutorTestAccess::ResolveTexture(
executor, compiled.program.info.images[0], value);
const auto view = cache.FindTexture(binding.image_id, binding.desc);
auto &image = cache.GetImage(binding.image_id);
if (swizzle == DstSel(4, 4, 4, 1)) {
const vk::ComponentMapping mapping{vk::ComponentSwizzle::eR, vk::ComponentSwizzle::eR,
vk::ComponentSwizzle::eR, vk::ComponentSwizzle::eOne};
Require(name, "RRR1 depth view", std::ranges::any_of(image.views, [&](const auto &cached) {
return cached.view == view && cached.info.mapping == mapping;
}), "comparison depth view lost its replicated depth or constant alpha");
}
Require(name, "fresh depth backing", view != nullptr &&
image.info.pixel_format == vk::Format::eD16Unorm &&
image.info.guest_format == Prospero::BufferFormat::k16UNorm &&
image.info.tile_mode == descriptor.TileMode() &&
image.info.type == (cube ? Prospero::ImageType::kColor2D
: Prospero::ImageType::kColor1D) &&
image.info.data.size == (cube ? 0x600u : 0x10000u) &&
image.info.resources.layers == layers,
"comparison texture lost its guest layout or depth representation");
auto ordinary_resource = compiled.program.info.images[0];
ordinary_resource.depth_compare = false;
value.dwords[0] = static_cast<uint32_t>(ordinary_address >> 8u);
auto ordinary = RenderExecutorTestAccess::ResolveTexture(executor, ordinary_resource, value);
(void)cache.FindTexture(ordinary.image_id, ordinary.desc);
auto &color = cache.GetImage(ordinary.image_id);
Require(name, "ordinary R16", color.info.pixel_format == vk::Format::eR16Unorm,
"ordinary R16 sampling unexpectedly acquired a depth format");
scheduler.Finish();
auto output = CreateStorageBuffer(name, {}, test.expected.size());
const auto sampler = context.GetSamplerCache().GetSampler(
sampler_descriptor, compiled.program.info.samplers[0].integer_border);
Image sampled;
sampled.view = view;
sampled.layout = image.backing.state.layout;
Dispatch(test, compiled, output, nullptr, &sampled, nullptr, nullptr, sampler);
Require(name, "native comparisons",
ReadBuffer(name, output, test.expected.size()) == test.expected,
"comparison samples lost a depth value or guest layer stride");
image.Transit(vk::ImageLayout::eTransferDstOptimal, vk::AccessFlagBits2::eTransferWrite,
{}, scheduler.Current().Handle());
color.Transit(vk::ImageLayout::eTransferDstOptimal, vk::AccessFlagBits2::eTransferWrite,
{}, scheduler.Current().Handle());
for (uint32_t face = 0; face < layers; face++) {
const vk::ImageSubresourceRange depth_range{vk::ImageAspectFlagBits::eDepth, 0, 1, face, 1};
const vk::ClearDepthStencilValue clear_depth{
static_cast<float>(depth_at(face, true)) / 65535.f, 0};
scheduler.Current().Handle().clearDepthStencilImage(image.backing.image,
vk::ImageLayout::eTransferDstOptimal, &clear_depth, 1, &depth_range);
const vk::ImageSubresourceRange color_range{vk::ImageAspectFlagBits::eColor, 0, 1, face, 1};
const vk::ClearColorValue clear_color{std::array<float, 4>{
static_cast<float>(depth_at(face, true)) / 65535.f, 0.f, 0.f, 0.f}};
scheduler.Current().Handle().clearColorImage(color.backing.image,
vk::ImageLayout::eTransferDstOptimal, &clear_color, 1, &color_range);
}
cache.MarkGpuWritten(binding.image_id);
cache.MarkGpuWritten(ordinary.image_id);
Require(name, "depth readback", TextureCacheTestAccess::TryDownload(cache, binding.image_id),
"ordinary tiled depth readback did not use the shared transfer path");
scheduler.Finish();
scheduler.DrainPriorityOperations();
for (uint32_t face = 0; face < layers; face++) {
uint16_t actual = 0;
std::memcpy(&actual, reinterpret_cast<const void *>(base + face * 0x100), sizeof(actual));
Require(name, "depth guest roundtrip", actual == depth_at(face, true),
"GPU depth clear was not written to the correct guest face");
}
auto promoted = RenderExecutorTestAccess::ResolveTexture(
executor, compiled.program.info.images[0], value);
sampled.view = cache.FindTexture(promoted.image_id, promoted.desc);
auto &promoted_image = cache.GetImage(promoted.image_id);
Require(name, "GPU color owner promotion", promoted.image_id != ordinary.image_id &&
promoted_image.info.pixel_format == vk::Format::eD16Unorm &&
promoted_image.IsGpuModified(),
"comparison binding did not preserve the GPU-current color owner");
sampled.layout = promoted_image.backing.state.layout;
scheduler.Finish();
Dispatch(test, compiled, output, nullptr, &sampled, nullptr, nullptr, sampler);
Require(name, "promoted comparisons",
ReadBuffer(name, output, promoted_expected.size()) == promoted_expected,
"color-to-depth ownership conversion changed comparison results");
DestroyBuffer(&output);
RenderExecutorTestAccess::ResetBindings(executor);
resources.UnmapMemory(base, allocation_size);
scheduler.Finish();
};
check_layout(false, DstSel(4, 4, 4, 4));
check_layout(false, DstSel(4, 4, 4, 1));
check_layout(true, DstSel(4, 0, 0, 1));
for (const u32 wave : {32u, 64u}) {
TestCase test;
test.name = wave == 32u ? "ImageGatherCompareLod32" : "ImageGatherCompareLod64";
test.has_user_data = test.has_compute_info = true;
test.compute_info.wave_size = wave;
test.compute_info.threads_num[0] = wave;
test.compute_info.threads_num[1] = test.compute_info.threads_num[2] = 1u;
test.compute_info.thread_ids_num = 1u;
test.compute_info.lds_size_dwords = 0;
ShaderTextureResource descriptor{{static_cast<u32>(base >> 8u),
(u32(Prospero::BufferFormat::k32Float) << 20u) | (3u << 30u), 3u << 14u,
(u32(Prospero::ImageType::kColor2D) << 28u) |
(u32(Prospero::TileMode::kStandard4KB) << 20u) | (1u << 16u) | DstSel(4, 4, 4, 4),
0, 0x00700000u | (1u << 4u), 0, 0}};
ShaderSamplerResource sampler_descriptor{{0x92u | (1u << 12u), 0xfff000u, 1u << 26u, 0}};
std::copy_n(descriptor.fields, 8u, test.user_data.begin() + 12u);
std::copy_n(sampler_descriptor.fields, 4u, test.user_data.begin() + 20u);
test.code.push_back(EncodeVop1(0x01, 40, Vgpr(0)));
AppendVMovLiteral(&test.code, 25, std::bit_cast<u32>(0.5f));
AppendVMovLiteral(&test.code, 26, std::bit_cast<u32>(0.5f));
constexpr std::array mip_depths{0.1f, 0.9f, 0.7f, 0.3f};
constexpr std::array gather_order{2u, 3u, 1u, 0u};
for (u32 mip = 0; mip < 2u; ++mip) {
AppendVMovLiteral(&test.code, 22, std::bit_cast<u32>(float(mip)));
for (const float reference : {0.2f, 0.5f, 0.8f}) {
AppendVMovLiteral(&test.code, 19, std::bit_cast<u32>(reference));
// Exact captured NSA order: reference v19, coordinates v25/v26, LOD v22.
test.code.insert(test.code.end(), {0xf130010au, 0x00a30013u, 0x00161a19u});
for (u32 component = 0; component < 4u; ++component) {
AppendStoreVgprAtLaneDwordOffset(&test.code, component, 40, test.expected.size());
const float depth = mip == 0u ? 0.6f : mip_depths[gather_order[component]];
test.expected.insert(test.expected.end(), wave, reference < depth ? 0x3f800000u : 0u);
}
}
}
AppendEnd(&test.code);
test.user_data[50] = test.expected.size() * sizeof(u32);
test.user_data[51] = 3u << 28u;
test.initial.assign(test.expected.size(), 0xdeadbeefu);
test.expected_mip_descriptors = 2u;
test.required_spirv = {"OpImageDrefGather", "OpSwitch"};
test.decoded_counts = {{"IMAGE_GATHER4_C_L", 6u}};
const auto compiled = CompileCase(test, SubgroupSize());
Require(test.name, "comparison mip resource", compiled.program.info.images.size() == 1u &&
compiled.program.info.images[0].depth_compare,
"explicit comparison gather lost its depth resource");
std::memset(mapped, 0, allocation_size);
RenderContext context(m_runtime_context);
auto &scheduler = context.GetCommandScheduler();
HW::Context registers{};
HW::UserConfig user_config{};
HW::Shader shaders{};
scheduler.Begin(registers, user_config, shaders);
context.MapMemory(base, allocation_size);
auto &executor = context.GetRenderExecutor();
auto binding = RenderExecutorTestAccess::ResolveTexture(
executor, compiled.program.info.images[0], compiled.resources.images[0]);
auto &cache = context.GetTextureCache();
Image sampled;
sampled.view = cache.FindTexture(binding.image_id, binding.desc);
auto &image = cache.GetImage(binding.image_id);
Require(test.name, "native depth mip backing",
image.info.pixel_format == vk::Format::eD32Sfloat && image.backing.mip_levels == 2u,
"comparison gather did not allocate both depth mips");
std::vector<u32> pixels(16u, std::bit_cast<u32>(0.6f));
for (const auto depth : mip_depths) pixels.push_back(std::bit_cast<u32>(depth));
auto staging = CreateHostBuffer(test.name, pixels.size() * sizeof(u32),
vk::BufferUsageFlagBits::eTransferSrc, pixels);
image.Transit(vk::ImageLayout::eTransferDstOptimal, vk::AccessFlagBits2::eTransferWrite,
{}, scheduler.Current().Handle());
std::array<vk::BufferImageCopy, 2> copies{};
for (u32 mip = 0; mip < 2u; ++mip) {
copies[mip].bufferOffset = mip == 0u ? 0u : 16u * sizeof(u32);
copies[mip].imageSubresource = {vk::ImageAspectFlagBits::eDepth, mip, 0, 1};
copies[mip].imageExtent = {4u >> mip, 4u >> mip, 1u};
}
scheduler.Current().Handle().copyBufferToImage(staging.buffer, image.backing.image,
vk::ImageLayout::eTransferDstOptimal, copies.size(), copies.data());
image.Transit(vk::ImageLayout::eShaderReadOnlyOptimal, vk::AccessFlagBits2::eShaderRead,
{}, scheduler.Current().Handle());
scheduler.Finish();
DestroyBuffer(&staging);
sampled.image = image.backing.image;
sampled.format = image.info.pixel_format;
sampled.mip_levels = 2u;
sampled.layout = image.backing.state.layout;
const auto sampler = context.GetSamplerCache().GetSampler(sampler_descriptor, false);
auto output = CreateStorageBuffer(test.name, test.initial, test.initial.size());
Dispatch(test, compiled, output, nullptr, &sampled, nullptr, nullptr, sampler);
Require(test.name, "explicit LOD, comparison reference and gather component order",
ReadBuffer(test.name, output, test.expected.size()) == test.expected,
"comparison gather selected the wrong mip, reference or component order");
DestroyBuffer(&output);
RenderExecutorTestAccess::ResetBindings(executor);
context.UnmapMemory(base, allocation_size);
scheduler.Finish();
}
Require(name, "unmap", Libs::LibKernel::Memory::KernelMunmap(base, allocation_size) == 0,
"comparison texture unmap failed");
Require(name, "release", Libs::LibKernel::Memory::KernelReleaseDirectMemory(
direct_offset, allocation_size) == 0,
"comparison texture release failed");
std::printf("[gpu] %-32s ok\n", name);
}
void CheckCubeFaceStorageExpansion() {
constexpr const char *name = "CubeFaceStorageExpansion";
constexpr uintptr_t base = 0x0000000204700000ull;
constexpr uint64_t allocation_size = 0x10000;
constexpr std::array<uint32_t, 6> half_red{0x3c00, 0x4000, 0x4200,
0x4400, 0x4500, 0x4600};
EnsureRuntimeContext();
int64_t direct_offset = -1;
Require(name, "direct allocation",
Libs::LibKernel::Memory::KernelAllocateDirectMemory(
0, Libs::LibKernel::Memory::KernelGetDirectMemorySize(),
allocation_size, allocation_size, 0, &direct_offset) == 0,
"cube-face allocation failed");
void *mapped = reinterpret_cast<void *>(base);
Require(name, "direct mapping",
Libs::LibKernel::Memory::KernelMapDirectMemory(
&mapped, allocation_size, 0x3, 0x10, direct_offset,
allocation_size) == 0 && mapped == reinterpret_cast<void *>(base),
"cube-face mapping failed");
std::memset(mapped, 0, allocation_size);
{
RenderContext context(m_runtime_context);
auto &scheduler = context.GetCommandScheduler();
HW::Context registers{};
HW::UserConfig user_config{};
HW::Shader shaders{};
scheduler.Begin(registers, user_config, shaders);
context.MapMemory(base, allocation_size);
auto &cache = context.GetTextureCache();
auto &executor = context.GetRenderExecutor();
for (uint32_t face = 0; face < half_red.size(); ++face) {
// PPSA25380 clears six single-face cube views at one guest address.
ShaderTextureResource descriptor{{static_cast<uint32_t>(base >> 8u),
0xc4700000u, 0x0001c001u, 0xb0000facu,
face | (face << 16u), 0x00700000u, 0, 0}};
TestCase test;
test.name = name;
test.has_user_data = true;
std::copy_n(descriptor.fields, 8, test.user_data.begin());
test.has_compute_info = true;
test.compute_info.threads_num[0] = test.compute_info.threads_num[1] = 8;
test.compute_info.threads_num[2] = 1;
test.compute_info.thread_ids_num = 2;
test.opcodes = {ShaderOpcode::V_MOV_B32, ShaderOpcode::IMAGE_STORE,
ShaderOpcode::S_ENDPGM};
test.code = {EncodeVop1(0x01, 20, 256), EncodeVop1(0x01, 21, 257)};
AppendVMovU32(&test.code, 22, 0);
AppendVMovLiteral(&test.code, 0,
std::bit_cast<uint32_t>(static_cast<float>(face + 1)));
AppendVMovLiteral(&test.code, 1, std::bit_cast<uint32_t>(0.5f));
AppendVMovLiteral(&test.code, 2, std::bit_cast<uint32_t>(0.25f));
AppendVMovLiteral(&test.code, 3, std::bit_cast<uint32_t>(1.0f));
test.code.push_back(EncodeMimg0(0x08, 0xf, 0, false, 5));
test.code.push_back(EncodeMimg1(0, 20));
AppendEnd(&test.code);
const auto compiled = CompileCase(test, SubgroupSize());
const auto &resource = compiled.program.info.images.at(0);
Require(name, "cube storage specialization", resource.cube &&
resource.written && resource.dimension ==
ShaderRecompiler::Decoder::ImageDimension::Dim2DArray,
"single-face cube store lost its array coordinates");
ShaderRecompiler::IR::DescriptorValue value{};
value.dword_count = 8;
std::copy_n(descriptor.fields, 8, value.dwords.begin());
const auto binding = RenderExecutorTestAccess::ResolveTexture(executor, resource, value);
Image storage;
storage.view = cache.FindTexture(binding.image_id, binding.desc);
auto &image = cache.GetImage(binding.image_id);
Require(name, "face view and backing", storage.view != nullptr &&
image.info.pixel_format == vk::Format::eR16G16B16A16Sfloat &&
image.info.bytes_per_block == 8 &&
image.info.resources.layers == face + 1 &&
image.backing.layers == face + 1 &&
binding.desc.view_info.type == vk::ImageViewType::e2DArray &&
binding.desc.view_info.base_layer == face &&
binding.desc.view_info.layer_count == 1,
"cube face did not expand the array or select its own layer");
image.Transit(vk::ImageLayout::eGeneral, vk::AccessFlagBits2::eShaderWrite,
{}, scheduler.Current().Handle());
storage.layout = image.backing.state.layout;
scheduler.Finish();
Dispatch(test, compiled, {}, nullptr, nullptr, &storage);
cache.MarkGpuWritten(binding.image_id);
for (uint32_t prior = 0; prior <= face; ++prior) {
const auto pixels = ReadCachedTexel(name, context, binding.image_id,
{}, {8, 8, 1}, prior);
for (uint32_t pixel = 0; pixel < 64; ++pixel) {
Require(name, "stored and preserved faces",
pixels[pixel * 2] == (0x38000000u | half_red[prior]) &&
pixels[pixel * 2 + 1] == 0x3c003400u,
"face " + std::to_string(prior) + " pixel " +
std::to_string(pixel) + " changed after writing face " +
std::to_string(face));
}
}
}
RenderExecutorTestAccess::ResetBindings(executor);
context.UnmapMemory(base, allocation_size);
scheduler.Finish();
}
Require(name, "release",
Libs::LibKernel::Memory::KernelMunmap(base, allocation_size) == 0 &&
Libs::LibKernel::Memory::KernelReleaseDirectMemory(
direct_offset, allocation_size) == 0,
"cube-face backing release failed");
std::printf("[gpu] %-32s ok\n", name);
}
void CheckRenderExecutorColorStandardTileDiscovery() {
constexpr const char *name = "RenderExecutorColorStandardTile";
constexpr uintptr_t base = 0x0000000203b00000ull;
constexpr uint64_t allocation_size = 0x190000;
constexpr uint64_t allocation_alignment = 0x10000;
constexpr auto format = Prospero::BufferFormat::k32Float;
struct TileCase {
Prospero::TileMode tile;
uint64_t target_offset;
uint64_t alignment;
};
constexpr std::array cases{
TileCase{Prospero::TileMode::kStandard4KB, 0x4000, 0x1000},
TileCase{Prospero::TileMode::kStandard64KB, 0x10000, 0x10000},
};
EnsureRuntimeContext();
const auto odd_pitch =
TileGetTexturePitch(format, 257, Prospero::TileMode::kStandard4KB);
TileSizeAlign odd_slice{};
TileGetTextureSize(format, 257, 131, 1, Prospero::TileMode::kStandard4KB,
&odd_slice, nullptr, nullptr);
Require(name, "tile layout",
odd_pitch == 288 && odd_slice.size == 0x2d000 &&
odd_slice.align == 0x1000,
"Standard4KB pitch, footprint, or block alignment is incorrect");
int64_t direct_offset = -1;
Require(name, "direct allocation",
Libs::LibKernel::Memory::KernelAllocateDirectMemory(
0, Libs::LibKernel::Memory::KernelGetDirectMemorySize(),
allocation_size, allocation_alignment, 0, &direct_offset) == 0,
"standard-tiled color allocation failed");
void *mapped = reinterpret_cast<void *>(base);
Require(name, "direct mapping",
Libs::LibKernel::Memory::KernelMapDirectMemory(
&mapped, allocation_size, 0x3, 0x10, direct_offset,
allocation_alignment) == 0 &&
mapped == reinterpret_cast<void *>(base),
"standard-tiled color mapping failed");
std::memset(mapped, 0, allocation_size);
for (const auto &tile_case : cases) {
const auto pitch = TileGetTexturePitch(format, 256, tile_case.tile);
TileSizeAlign slice{};
TileGetTextureSize(format, 256, 256, 1, tile_case.tile, &slice, nullptr,
nullptr);
Require(
name, "tile layout",
pitch == 256 && slice.size == 0x40000 &&
slice.align == tile_case.alignment,
"standard tile pitch, footprint, or block alignment is incorrect");
const auto target_address = base + tile_case.target_offset;
const auto target_size = slice.size * 6ull;
RenderContext context(m_runtime_context);
auto &scheduler = context.GetCommandScheduler();
HW::Context registers{};
HW::UserConfig user_config{};
HW::Shader shaders{};
registers.SetColorBase(0, {.addr = target_address});
registers.SetColorInfo(
0, {.format = Prospero::ChannelLayout::k32,
.channel_type = Prospero::ChannelType::kFloat,
.channel_order = Prospero::ChannelOrder::kStandard});
registers.SetColorView(
0, {.base_array_slice_index = 5, .last_array_slice_index = 5});
registers.SetColorAttrib2(0, {.height = 255, .width = 255});
registers.SetColorAttrib3(0, {.tile_mode = tile_case.tile,
.dimension = 1,
.metadata_pipe_aligned = true});
registers.SetRenderTargetMask(0x0f);
scheduler.Begin(registers, user_config, shaders);
auto &resources = context;
auto &texture_cache = resources.GetTextureCache();
auto &executor = context.GetRenderExecutor();
resources.MapMemory(base, allocation_size);
RenderColorInfo color{};
RenderExecutorTestAccess::ResolveRenderColorTarget(
executor, scheduler.Current(), color, 0);
const auto attachment =
texture_cache.FindRenderTarget(color.image_id, color.desc);
const auto &image = texture_cache.GetImage(color.image_id);
Require(name, "captured target",
color.image_id && attachment != nullptr &&
color.guest_array_layer == 5 &&
color.desc.view_info.base_layer == 5 &&
color.desc.info.data.address == target_address &&
color.desc.info.data.size == target_size &&
color.desc.info.tile_mode == tile_case.tile &&
color.desc.info.extent == vk::Extent3D{256, 256, 1} &&
color.desc.info.resources == ImageSubresources{1, 6} &&
color.desc.info.pitch == pitch &&
color.desc.info.mip_layout[0].offset == 0 &&
color.desc.info.mip_layout[0].size == target_size &&
color.desc.view_info.type == vk::ImageViewType::e2D &&
color.desc.view_info.base_layer == 5 &&
color.desc.view_info.layer_count == 1 &&
(target_address & (tile_case.alignment - 1)) == 0 &&
image.usage.render_target && image.IsGpuModified(),
"slice 5 standard-tiled render target was not resolved");
RenderExecutorTestAccess::ResetBindings(executor);
resources.UnmapMemory(base, allocation_size);
scheduler.Finish();
}
Require(name, "unmap direct backing",
Libs::LibKernel::Memory::KernelMunmap(base, allocation_size) == 0,
"standard-tiled color mapping release failed");
Require(name, "release direct backing",
Libs::LibKernel::Memory::KernelReleaseDirectMemory(
direct_offset, allocation_size) == 0,
"standard-tiled color allocation release failed");
std::printf("[gpu] %-32s ok\n", name);
}
void CheckRenderExecutorColorDepthTileDiscovery() {
constexpr const char *name = "RenderExecutorColorDepthTile";
constexpr uintptr_t base = 0x0000000203d00000ull;
constexpr uint64_t allocation_size = 0x10000;
constexpr uint64_t allocation_alignment = 0x10000;
EnsureRuntimeContext();
int64_t direct_offset = -1;
Require(name, "direct allocation",
Libs::LibKernel::Memory::KernelAllocateDirectMemory(
0, Libs::LibKernel::Memory::KernelGetDirectMemorySize(),
allocation_size, allocation_alignment, 0, &direct_offset) == 0,
"depth-tiled color allocation failed");
void *mapped = reinterpret_cast<void *>(base);
Require(name, "direct mapping",
Libs::LibKernel::Memory::KernelMapDirectMemory(
&mapped, allocation_size, 0x3, 0x10, direct_offset,
allocation_alignment) == 0 &&
mapped == reinterpret_cast<void *>(base),
"depth-tiled color mapping failed");
constexpr uint32_t width = 64;
constexpr uint32_t height = 64;
TileTextureBlockLayout tile_layout{};
Require(name, "tile layout",
TileGetTextureBlockLayout(Prospero::BufferFormat::k32Float,
Prospero::TileMode::kDepth, false,
tile_layout),
"depth-tiled color block layout is unavailable");
uint32_t block_xor = 0;
Require(name, "block xor",
TileGetBlockXor(tile_layout.block, 0, 0, 0, block_xor),
"depth-tiled color block XOR is unavailable");
std::vector<uint32_t> expected(allocation_size / sizeof(uint32_t));
for (uint32_t y = 0; y < height; ++y) {
for (uint32_t x = 0; x < width; ++x) {
uint32_t offset = 0;
Require(name, "texel offset",
TileGetBlockOffset(tile_layout.block, x, y, 0, offset),
"depth-tiled color texel offset is unavailable");
const auto index = (offset ^ block_xor) / sizeof(uint32_t);
Require(name, "texel address", index < expected.size(),
"depth-tiled color texel address exceeds its backing");
expected[index] = 0x3f000000u + y * width + x;
}
}
std::memcpy(mapped, expected.data(), allocation_size);
{
RenderContext context(m_runtime_context);
auto &scheduler = context.GetCommandScheduler();
HW::Context registers{};
HW::UserConfig user_config{};
HW::Shader shaders{};
registers.SetColorBase(0, {.addr = base});
registers.SetColorInfo(
0, {.format = Prospero::ChannelLayout::k32,
.channel_type = Prospero::ChannelType::kFloat,
.channel_order = Prospero::ChannelOrder::kStandard});
registers.SetColorAttrib2(0, {.height = 63, .width = 63});
registers.SetColorAttrib3(0, {.tile_mode = Prospero::TileMode::kDepth,
.dimension = 1,
.metadata_pipe_aligned = true});
registers.SetRenderTargetMask(0x0f);
scheduler.Begin(registers, user_config, shaders);
auto &resources = context;
auto &texture_cache = resources.GetTextureCache();
auto &executor = context.GetRenderExecutor();
resources.MapMemory(base, allocation_size);
RenderColorInfo color{};
RenderExecutorTestAccess::ResolveRenderColorTarget(
executor, scheduler.Current(), color, 0);
const auto attachment =
texture_cache.FindRenderTarget(color.image_id, color.desc);
const auto &image = texture_cache.GetImage(color.image_id);
Require(name, "captured target",
color.image_id && attachment != nullptr &&
color.desc.info.tile_mode == Prospero::TileMode::kDepth &&
color.desc.info.pitch == 128 &&
color.desc.info.data.size == allocation_size &&
color.desc.info.mip_layout[0].size == allocation_size &&
image.usage.render_target && image.IsGpuModified(),
"supported depth-tiled color target was not resolved");
scheduler.Finish();
scheduler.DrainPriorityOperations();
std::vector<uint8_t> cleared(allocation_size);
Require(name, "clear guest backing",
Libs::LibKernel::Memory::TryWriteBacking(base, cleared.data(),
allocation_size),
"depth-tiled color backing could not be cleared");
Require(
name, "readback queue",
TextureCacheTestAccess::TryDownload(texture_cache, color.image_id),
"depth-tiled color target could not be queued for readback");
RenderExecutorTestAccess::ResetBindings(executor);
scheduler.Finish();
scheduler.DrainPriorityOperations();
std::vector<uint32_t> observed(expected.size());
Require(name, "readback backing",
Libs::LibKernel::Memory::TryReadBacking(base, observed.data(),
allocation_size),
"depth-tiled color readback is unavailable");
bool recovered = true;
for (uint32_t y = 0; y < height; ++y) {
for (uint32_t x = 0; x < width; ++x) {
uint32_t offset = 0;
const auto valid =
TileGetBlockOffset(tile_layout.block, x, y, 0, offset);
const auto index = (offset ^ block_xor) / sizeof(uint32_t);
recovered &= valid && index < observed.size() &&
observed[index] == expected[index];
}
}
Require(name, "round trip", recovered,
"depth-tiled color upload/readback changed active texels");
resources.UnmapMemory(base, allocation_size);
scheduler.Finish();
}
Require(name, "unmap direct backing",
Libs::LibKernel::Memory::KernelMunmap(base, allocation_size) == 0,
"depth-tiled color mapping release failed");
Require(name, "release direct backing",
Libs::LibKernel::Memory::KernelReleaseDirectMemory(
direct_offset, allocation_size) == 0,
"depth-tiled color allocation release failed");
std::printf("[gpu] %-32s ok\n", name);
}
void CheckRenderExecutorStencilBindingDiscovery() {
constexpr const char *name = "RenderExecutorStencilBindingDiscovery";
constexpr uintptr_t base = 0x0000000203600000ull;
constexpr uint64_t allocation_size = 0x1000000;
constexpr uint64_t allocation_alignment = 0x10000;
constexpr uint64_t depth_address = base + 0x40000;
constexpr uint64_t stencil_address = base + 0x70000;
constexpr uint64_t mipped_storage_address = base + 0x100000;
constexpr uint32_t tail_backing_value = 0x2468ace0u;
constexpr uint32_t tail_gpu_value = 0x13579bdfu;
EnsureRuntimeContext();
int64_t direct_offset = -1;
Require(name, "direct allocation",
Libs::LibKernel::Memory::KernelAllocateDirectMemory(
0, Libs::LibKernel::Memory::KernelGetDirectMemorySize(),
allocation_size, allocation_alignment, 0, &direct_offset) == 0,
"descriptor discovery direct-memory allocation failed");
void *mapped = reinterpret_cast<void *>(base);
Require(name, "direct mapping",
Libs::LibKernel::Memory::KernelMapDirectMemory(
&mapped, allocation_size, 0x3, 0x10, direct_offset,
allocation_alignment) == 0 &&
mapped == reinterpret_cast<void *>(base),
"descriptor discovery fixed mapping failed");
std::memset(mapped, 0, allocation_size);
std::fill_n(reinterpret_cast<uint32_t *>(mipped_storage_address),
0x10000 / 4, tail_backing_value);
{
RenderContext context(m_runtime_context);
auto &scheduler = context.GetCommandScheduler();
HW::Context registers{};
HW::UserConfig user_config{};
HW::Shader shaders{};
context.InitializeGpu(nullptr);
scheduler.Begin(registers, user_config, shaders);
auto &resources = context;
auto &texture_cache = resources.GetTextureCache();
auto &executor = context.GetRenderExecutor();
resources.MapMemory(base, allocation_size);
std::vector<PipelineCache::Pipeline> descriptor_pipelines;
// The virtual-texture atlas is written as raw BC3 blocks, then sampled as BC3.
constexpr uint64_t block_alias_address = base + 0xf0000;
constexpr std::array<uint32_t, 4> block_alias_data{
0x01234567u, 0x89abcdefu, 0xfedcba98u, 0x76543210u};
const auto resolve_block_alias = [&](bool compressed) {
const uint32_t side = compressed ? 256 : 64;
const auto format = compressed ? Prospero::BufferFormat::kBc3UNorm
: Prospero::BufferFormat::k32_32_32_32UInt;
ShaderRecompiler::IR::DescriptorValue value{};
value.dword_count = 8;
value.dwords = {static_cast<uint32_t>(block_alias_address >> 8u),
(static_cast<uint32_t>(format) << 20u) | (3u << 30u),
((side - 1u) >> 2u) | ((side - 1u) << 14u),
0x90900facu, 0, 0x00700000u, 0, 0};
ShaderRecompiler::IR::ImageResource resource{};
resource.resource_class = compressed
? ShaderRecompiler::IR::ImageResourceClass::Sampled
: ShaderRecompiler::IR::ImageResourceClass::Storage;
resource.numeric_class = compressed ? Prospero::TextureNumericClass::Float
: Prospero::TextureNumericClass::Uint;
resource.dimension = ShaderRecompiler::Decoder::ImageDimension::Dim2D;
resource.read = compressed;
resource.written = !compressed;
return RenderExecutorTestAccess::ResolveTexture(executor, resource, value);
};
auto raw_blocks = resolve_block_alias(false);
(void)texture_cache.FindTexture(raw_blocks.image_id, raw_blocks.desc);
auto &raw_blocks_native = texture_cache.GetImage(raw_blocks.image_id);
raw_blocks_native.Transit(vk::ImageLayout::eTransferDstOptimal,
vk::AccessFlagBits2::eTransferWrite, {},
scheduler.Current().Handle());
vk::ClearColorValue block_clear{};
std::copy(block_alias_data.begin(), block_alias_data.end(),
block_clear.uint32.begin());
const vk::ImageSubresourceRange block_range{
vk::ImageAspectFlagBits::eColor, 0, 1, 0, 1};
scheduler.Current().Handle().clearColorImage(
raw_blocks_native.backing.image, vk::ImageLayout::eTransferDstOptimal,
&block_clear, 1, &block_range);
texture_cache.MarkGpuWritten(raw_blocks.image_id);
auto compressed_blocks = resolve_block_alias(true);
(void)texture_cache.FindTexture(compressed_blocks.image_id,
compressed_blocks.desc);
auto raw_blocks_again = resolve_block_alias(false);
Require(name, "compressed atlas storage reuse",
compressed_blocks.image_id != raw_blocks.image_id &&
raw_blocks_again.image_id == compressed_blocks.image_id &&
texture_cache.FindTexture(raw_blocks_again.image_id,
raw_blocks_again.desc) != nullptr,
"BC3 sampling replaced the GPU atlas on its next raw-block write");
Require(name, "compressed atlas download",
TextureCacheTestAccess::TryDownload(texture_cache,
raw_blocks_again.image_id),
"the retained BC3 atlas could not publish its native contents");
scheduler.Finish();
scheduler.DrainPriorityOperations();
const auto *block_words = reinterpret_cast<const uint32_t *>(block_alias_address);
Require(name, "compressed atlas GPU contents",
std::equal(block_alias_data.begin(), block_alias_data.end(), block_words) &&
std::equal(block_alias_data.begin(), block_alias_data.end(),
block_words + 0x10000 / sizeof(uint32_t) - 4),
"compressed atlas aliases reloaded stale CPU bytes over GPU-written blocks");
const auto set_image_bindings = [](ShaderRecompiler::IR::Program &program) {
for (auto &image : program.info.images) {
const auto kind = *ShaderRecompiler::IR::DescriptorBindingForImage(image);
const auto index = static_cast<size_t>(kind);
image.descriptor_index = program.bindings.descriptor_counts[index];
program.bindings.descriptor_counts[index] += image.mip_count;
program.bindings.descriptor_mask |= uint64_t{1} << index;
}
};
const auto make_buffer_program =
[](ShaderType stage, ShaderRecompiler::IR::BufferResource resource) {
ShaderRecompiler::IR::CompiledShaderInfo program{};
program.stage = stage;
resource.descriptor_index = 0;
program.info.buffers.push_back(resource);
program.bindings.descriptor_counts[0] = 1;
program.bindings.descriptor_mask = 1;
program.bindings.push_data_start_dword = 0;
return program;
};
{
const auto buffer_program = make_buffer_program(ShaderType::Compute, {.read = true});
constexpr uint64_t buffer_address = base + allocation_size - 0x5000;
ShaderBufferResource buffer_descriptor{};
buffer_descriptor.UpdateAddress48(buffer_address);
buffer_descriptor.fields[2] = 0x40000000; // Only oversized descriptors are clamped.
ShaderRecompiler::IR::ResourceSnapshot buffer_snapshot;
ShaderStageRuntime buffer_runtime{&buffer_program, &buffer_snapshot};
auto &value = buffer_snapshot.buffers.emplace_back();
std::memcpy(value.dwords.data(), buffer_descriptor.fields,
sizeof(buffer_descriptor.fields));
value.dword_count = 4;
PreparedBindings buffer_bindings;
executor.PrepareBindings(buffer_runtime, buffer_bindings);
std::array<PreparedBindings *, 1> buffer_stages{&buffer_bindings};
executor.FindBuffers(buffer_stages);
const auto original_id = buffer_bindings.buffer_sources[0].id;
auto &buffer_cache = resources.GetBufferCache();
const auto merged_id =
buffer_cache.FindBuffer(base + allocation_size - 0x10000, 0x10000);
Require(name, "buffer discovery invalidation", merged_id != original_id,
"overlapping buffer discovery did not replace its smaller owner");
executor.RebindBuffers(buffer_bindings);
const auto &binding = buffer_bindings.buffers[0];
const auto &owner = buffer_cache.GetBuffer(merged_id);
Require(name, "clamped buffer rebind",
binding.buffer == owner.Handle() &&
binding.offset == owner.Offset(buffer_address) &&
binding.range == 0x5000,
"descriptor rebind lost its clamped guest range or retained a stale host owner");
}
{
const auto buffer_program = make_buffer_program(ShaderType::Compute, {.read = true, .written = true});
constexpr uint64_t buffer_address = base + 0xe0002;
ShaderBufferResource descriptor{};
descriptor.UpdateAddress48(buffer_address);
descriptor.fields[1] |= 6u << 16u;
descriptor.fields[2] = 2;
descriptor.fields[3] = 0x5204;
ShaderRecompiler::IR::ResourceSnapshot snapshot;
auto &value = snapshot.buffers.emplace_back();
std::copy_n(descriptor.fields, 4, value.dwords.begin());
value.dword_count = 4;
ShaderStageRuntime runtime{&buffer_program, &snapshot};
PreparedBindings bindings;
executor.PrepareBindings(runtime, bindings);
std::array<PreparedBindings *, 1> stages{&bindings};
executor.FindBuffers(stages);
executor.RebindBuffers(bindings);
const auto &owner = resources.GetBufferCache().GetBuffer(bindings.buffer_sources[0].id);
Require(name, "halfword buffer base and exact tail",
bindings.buffers[0].buffer == owner.Handle() &&
bindings.buffers[0].offset + 2 == owner.Offset(buffer_address) &&
bindings.buffers[0].range == 14 &&
bindings.shader_data[buffer_program.bindings.memory_offset_dword] == 2,
"native binding lost the byte adjustment or rounded the final halfword range");
}
{
constexpr uint64_t address = base + allocation_size - 0x4000 + 12;
constexpr uint64_t mapped_tail = base + allocation_size - address;
auto writer = make_buffer_program(ShaderType::Vertex, {.written = true});
ShaderRecompiler::IR::CompiledShaderInfo reader{};
reader.stage = ShaderType::Pixel;
ShaderRecompiler::IR::ResourceSnapshot writer_snapshot, reader_snapshot;
ShaderBufferResource descriptor{};
descriptor.UpdateAddress48(address);
descriptor.fields[2] = UINT32_MAX;
auto &value = writer_snapshot.buffers.emplace_back();
std::copy_n(descriptor.fields, 4, value.dwords.begin());
value.dword_count = 4;
ShaderStageRuntime writer_runtime{&writer, &writer_snapshot};
ShaderStageRuntime reader_runtime{&reader, &reader_snapshot};
PreparedBindings writer_bindings, reader_bindings;
const auto check_range = [&](uint64_t expected, const char *description) {
executor.PrepareBindings(writer_runtime, writer_bindings);
executor.PrepareBindings(reader_runtime, reader_bindings);
std::array<PreparedBindings *, 2> stages{&reader_bindings, &writer_bindings};
executor.FindBuffers(stages);
executor.RebindBuffers(writer_bindings);
const auto &source = writer_bindings.buffer_sources[0];
const auto &binding = writer_bindings.buffers[0];
const auto &owner = resources.GetBufferCache().GetBuffer(source.id);
Require(name, description, source.address == address && source.size == expected &&
binding.buffer == owner.Handle() && binding.offset <= owner.Offset(address) &&
binding.range == expected + owner.Offset(address) - binding.offset,
"descriptor discovery or rebind changed the write boundary or base adjustment");
};
writer_snapshot.specialization_reads = {{address + 0x500, 4},
{address - 0x40, 0x40}, {address + 0x80, 0}};
reader_snapshot.specialization_reads = {{address + 0x300, 4}, {address + 0x100, 4}};
check_range(0x100, "unbounded write stops before nearest cross-stage read");
reader_snapshot.specialization_reads.clear();
check_range(0x500, "unbounded write stops before same-stage read");
writer_snapshot.specialization_reads = {{address - 0x40, 0x40},
{address + mapped_tail + 0x100, 4}};
check_range(mapped_tail, "earlier and unmapped reads preserve mapped tail");
writer_snapshot.specialization_reads.clear();
check_range(mapped_tail, "unbounded write without reads preserves mapped tail");
reader_snapshot.specialization_reads = {{address + 0x100, 4}};
value.dwords[2] = 0x300;
check_range(0x300, "finite write preserves its declared range");
value.dwords[2] = UINT32_MAX;
writer.info.buffers[0].written = false;
writer.info.buffers[0].read = true;
check_range(mapped_tail, "read-only sentinel preserves mapped tail");
auto &graphics = context.GetGraphics();
auto &storage_limit = graphics.physical_device_properties.limits.maxStorageBufferRange;
const auto original_limit = storage_limit;
storage_limit = 0x2000;
value.dwords[1] |= 16u << 16u;
check_range(storage_limit - (graphics.StorageMinAlignment() - 1),
"read-only structured sentinel reserves native alignment prefix");
Require(name, "read-only native range fits device", writer_bindings.buffers[0].range <= storage_limit,
"read-only cap omitted the native descriptor alignment prefix");
value.dwords[2] = 0x300; // A finite 12 KiB descriptor remains finite despite the mock limit.
std::array<PreparedBindings *, 1> stages{&writer_bindings};
executor.FindBuffers(stages);
Require(name, "finite read exceeds mock cap unchanged", writer_bindings.buffer_sources[0].size == 0x3000,
"read-only window changed a finite descriptor's declared range");
storage_limit = original_limit;
check_range(0x3000, "finite structured read preserves native range");
value.dwords[2] = UINT32_MAX;
check_range(mapped_tail, "read-only structured sentinel preserves smaller mapping");
}
for (const auto format : {Prospero::BufferFormat::k8UNorm,
Prospero::BufferFormat::k16UNorm}) {
const uint64_t alias_address = base + 0xe00000 +
Prospero::NumBytesPerElement(format) * 0x10000;
constexpr uint8_t texel = 0x7f;
Libs::LibKernel::Memory::WriteBacking(alias_address, &texel, sizeof(texel));
u32 expected = texel;
if (format == Prospero::BufferFormat::k16UNorm) {
auto depth_desc = MakeLinearDesc(alias_address, 256, vk::Format::eD16Unorm,
format, Prospero::ImageType::kColor2D, {1, 1, 1}, 1, 2, 1);
depth_desc.info.pitch = 128;
depth_desc.info.UpdateSize();
depth_desc.type = BindingType::DepthTarget;
depth_desc.view_info.aspect = vk::ImageAspectFlagBits::eDepth;
depth_desc.view_info.usage = vk::ImageUsageFlagBits::eDepthStencilAttachment;
const auto depth_id = texture_cache.FindImage(depth_desc);
auto &depth = texture_cache.GetImage(depth_id);
depth.Transit(vk::ImageLayout::eTransferDstOptimal,
vk::AccessFlagBits2::eTransferWrite, {}, scheduler.Current().Handle());
const vk::ImageSubresourceRange range{vk::ImageAspectFlagBits::eDepth, 0, 1, 0, 1};
scheduler.Current().Handle().clearDepthStencilImage(depth.backing.image,
vk::ImageLayout::eTransferDstOptimal, vk::ClearDepthStencilValue{1.f, 0}, range);
texture_cache.MarkGpuWritten(depth_id);
expected = 0xffff;
}
ShaderRecompiler::IR::CompiledShaderInfo program{};
program.stage = ShaderType::Compute;
ShaderRecompiler::IR::ResourceSnapshot snapshot;
// PPSA24156 binds one-texel R8/R16 allocations as 3D, then 2D.
for (const auto type : {Prospero::ImageType::kColor3D,
Prospero::ImageType::kColor2D}) {
ShaderRecompiler::IR::ImageResource resource{};
resource.resource_class = ShaderRecompiler::IR::ImageResourceClass::Sampled;
resource.read = true;
resource.numeric_class = Prospero::TextureNumericClass::Float;
resource.dimension = ShaderRecompiler::Decoder::ImageDimension::Dim2D;
if (type == Prospero::ImageType::kColor3D) {
resource.dimension = ShaderRecompiler::Decoder::ImageDimension::Dim3D;
}
program.info.images.push_back(resource);
auto &value = snapshot.images.emplace_back();
value.dword_count = 8;
value.dwords = {static_cast<u32>(alias_address >> 8u),
static_cast<u32>(format) << 20u, 0,
DstSel(4, 4, 4, 1) | (static_cast<u32>(type) << 28u),
0, 0x00700000u, 0, 0};
}
ShaderStageRuntime runtime{&program, &snapshot};
PreparedBindings bindings;
executor.PrepareBindings(runtime, bindings);
Require(name, "overlapping dimension discovery",
!texture_cache.GetImage(bindings.images[0].image_id).registered,
"the 2D descriptor did not replace its earlier 3D alias");
executor.RebindImages(bindings);
auto output = CreateHostBuffer(name, 8, vk::BufferUsageFlagBits::eTransferDst, {0, 0});
constexpr std::array image_types{vk::ImageType::e3D, vk::ImageType::e2D};
for (u32 i = 0; i < bindings.images.size(); ++i) {
const auto &binding = bindings.images[i];
auto &image = texture_cache.GetImage(binding.image_id);
Require(name, "overlapping dimension views", binding.image_view != nullptr &&
image.backing.image_type == image_types[i],
"rediscovery lost a live view or the descriptor's image dimension");
image.Transit(vk::ImageLayout::eTransferSrcOptimal,
vk::AccessFlagBits2::eTransferRead, {}, scheduler.Current().Handle());
vk::BufferImageCopy copy{};
copy.bufferOffset = i * sizeof(u32);
copy.imageSubresource = {vk::ImageAspectFlagBits::eColor, 0, 0, 1};
copy.imageExtent = {1, 1, 1};
scheduler.Current().Handle().copyImageToBuffer(image.backing.image,
vk::ImageLayout::eTransferSrcOptimal, output.buffer, 1, &copy);
}
vk::MemoryBarrier barrier{};
barrier.srcAccessMask = vk::AccessFlagBits::eTransferWrite;
barrier.dstAccessMask = vk::AccessFlagBits::eHostRead;
scheduler.Current().Handle().pipelineBarrier(vk::PipelineStageFlagBits::eTransfer,
vk::PipelineStageFlagBits::eHost, {}, 1, &barrier, 0, nullptr, 0, nullptr);
RenderExecutorTestAccess::ResetBindings(executor);
scheduler.Finish();
Require(name, "overlapping dimension GPU contents",
ReadBuffer(name, output, 2) == std::vector<u32>{expected, expected},
"an alias replacement retired image contents before queued GPU reads");
DestroyBuffer(&output);
}
for (const bool volume_first : {false, true}) {
// PPSA24156 shares a six-face R8 allocation with a smaller 3D placeholder.
const auto address = base + 0xe30000 + volume_first * 0x10000;
auto array_desc = MakeLinearDesc(address, 1536, vk::Format::eR8Unorm,
Prospero::BufferFormat::k8UNorm, Prospero::ImageType::kColor2D,
{1, 1, 1}, 6, 1, 1);
array_desc.info.pitch = 256;
array_desc.info.UpdateSize();
auto volume_desc = array_desc;
volume_desc.info.type = Prospero::ImageType::kColor3D;
volume_desc.info.data.size = 256;
volume_desc.info.resources.layers = 1;
volume_desc.info.mip_layout[0].size = 256;
volume_desc.view_info.type = vk::ImageViewType::e3D;
volume_desc.view_info.layer_count = 1;
const auto first_id = texture_cache.FindImage(volume_first ? volume_desc : array_desc);
const auto second_id = texture_cache.FindImage(volume_first ? array_desc : volume_desc);
const auto array_id = volume_first ? second_id : first_id;
const auto volume_id = volume_first ? first_id : second_id;
Require(name, "smaller volume native image", volume_id != array_id &&
texture_cache.GetImage(volume_id).backing.image_type == vk::ImageType::e3D,
"a smaller 3D descriptor reused the six-layer 2D owner");
(void)texture_cache.FindTexture(array_id, array_desc);
(void)texture_cache.FindTexture(volume_id, volume_desc);
Require(name, "retained dimensional aliases",
texture_cache.FindImage(array_desc) == array_id &&
texture_cache.FindImage(volume_desc) == volume_id &&
texture_cache.GetImage(array_id).info.resources.layers == 6,
"dimensional alias lookup discarded the array owner or lost its faces");
auto subarray_desc = array_desc;
subarray_desc.info.data.size = 1280;
subarray_desc.info.resources.layers = 5;
subarray_desc.info.mip_layout[0].size = 1280;
subarray_desc.view_info.layer_count = 5;
const auto subarray_id = texture_cache.FindImage(subarray_desc);
Require(name, "preserved dimensional overlap selection", subarray_id == array_id &&
texture_cache.GetImage(array_id).info.resources.layers == 6,
"a smaller volume alias displaced the selected array owner");
(void)texture_cache.FindTexture(subarray_id, subarray_desc);
}
constexpr auto stencil_format = Prospero::BufferFormat::k8UInt;
constexpr auto linear = Prospero::TileMode::kLinear;
TileSizeAlign stencil_layout{};
TileGetTextureTotalSize(stencil_format, 1, 1, 1, 1, linear, false,
stencil_layout);
Require(name, "stencil footprint",
stencil_layout.size != 0 && stencil_layout.align != 0 &&
(stencil_address & (stencil_layout.align - 1u)) == 0,
"linear stencil descriptor produced an invalid footprint");
ShaderRecompiler::IR::Program null_program{};
null_program.stage = ShaderType::Compute;
null_program.resource_tracking_complete = true;
ShaderRecompiler::IR::ImageResource null_resource{};
null_resource.resource_class =
ShaderRecompiler::IR::ImageResourceClass::Sampled;
null_resource.numeric_class = Prospero::TextureNumericClass::Uint;
null_resource.dimension =
ShaderRecompiler::Decoder::ImageDimension::Dim2D;
null_resource.read = true;
null_program.info.images.push_back(null_resource);
auto null_volume = null_resource;
null_volume.dimension = ShaderRecompiler::Decoder::ImageDimension::Dim3D;
null_program.info.images.push_back(null_volume);
auto null_storage = null_resource;
null_storage.resource_class =
ShaderRecompiler::IR::ImageResourceClass::Storage;
null_storage.read = false;
null_storage.written = true;
null_program.info.images.push_back(null_storage);
ShaderRecompiler::IR::ResourceSnapshot null_snapshot{};
ShaderRecompiler::IR::DescriptorValue null_descriptor{};
null_descriptor.dword_count = 8;
null_snapshot.images.assign(3, null_descriptor);
ShaderRecompiler::IR::ResourceSpecialization null_specialization{};
null_specialization.images.resize(3);
for (auto &image : null_specialization.images) {
image.numeric_class = Prospero::TextureNumericClass::Float;
image.dimension = ShaderRecompiler::Decoder::ImageDimension::Dim2D;
}
ShaderRecompiler::IR::ApplyResourceSpecialization(
null_program, null_specialization);
null_program.info.images.back().mip_mode = ShaderRecompiler::IR::ImageMipMode::Dynamic;
null_program.info.images.back().mip_count = 3;
set_image_bindings(null_program);
ShaderRecompiler::IR::CompiledShaderInfo null_info{};
null_info.stage = null_program.stage;
null_info.info = std::move(null_program.info);
null_info.bindings = std::move(null_program.bindings);
ShaderStageRuntime null_runtime{&null_info, &null_snapshot};
PreparedBindings null_bindings;
executor.PrepareBindings(null_runtime, null_bindings);
executor.RebindImages(null_bindings);
Require(name, "null descriptor count",
null_bindings.images.size() == 3,
"null descriptor preparation lost an image binding");
descriptor_pipelines.push_back(RenderExecutorTestAccess::CommitBindings(
executor, scheduler.Current(), null_bindings));
Require(name, "null descriptors have no cache image or native view",
std::ranges::all_of(null_bindings.images,
[](const auto &binding) {
const auto info = MakeImageInfo(binding);
return !binding.image_id &&
binding.image_view == nullptr &&
binding.mip_views.empty() &&
binding.layout ==
vk::ImageLayout::eGeneral &&
info.imageView == nullptr &&
info.imageLayout == binding.layout;
}),
"sampled/storage null descriptors acquired a backing image or lost their general layout");
for (u32 mip = 0; mip < 3; ++mip) {
const auto info = MakeImageInfo(null_bindings.images.back(), mip);
Require(name, "null storage mip descriptors",
info.imageView == nullptr && info.imageLayout == vk::ImageLayout::eGeneral,
"a null storage mip acquired a backing image");
}
constexpr uint64_t target_mip_size = 0x10000;
const auto make_target_desc = [&](uint64_t address, uint64_t size,
vk::Extent3D extent) {
ImageDesc desc{};
desc.type = BindingType::RenderTarget;
desc.info.data = {address, size};
desc.info.pixel_format = vk::Format::eR32Uint;
desc.info.guest_format = Prospero::BufferFormat::k32UInt;
desc.info.type = Prospero::ImageType::kColor2D;
desc.info.extent = extent;
desc.info.resources = {1, 1};
desc.info.pitch = extent.width;
desc.info.bytes_per_block = 4;
desc.info.samples = 1;
desc.info.tile_mode = linear;
desc.info.mip_layout[0] = {
0, size, extent.width, extent.height, 0, size, extent.width};
desc.info.tiling.block.bytes_per_element = 4;
desc.info.linear_slice_stride = size;
desc.view_info.format = desc.info.pixel_format;
desc.view_info.type = vk::ImageViewType::e2D;
desc.view_info.aspect = vk::ImageAspectFlagBits::eColor;
desc.view_info.usage = vk::ImageUsageFlagBits::eColorAttachment;
return desc;
};
TextureCacheTestAccess::SetLinearReadback(texture_cache, true);
ShaderTextureResource storage{};
const uint64_t storage_address = base + 0x60000;
constexpr uint32_t storage_native_value = 0x13579bdfu;
constexpr uint32_t storage_stale_value = 0x2468ace0u;
std::memcpy(reinterpret_cast<uint8_t *>(mapped) + 0x60000,
&storage_native_value, sizeof(storage_native_value));
const uint64_t encoded_storage_address = storage_address >> 8u;
storage.fields[0] = static_cast<uint32_t>(encoded_storage_address);
storage.fields[1] =
static_cast<uint32_t>(encoded_storage_address >> 32u) |
(static_cast<uint32_t>(stencil_format) << 20u);
storage.fields[3] =
DstSel(4, 5, 6, 7) | (static_cast<uint32_t>(linear) << 20u) |
(static_cast<uint32_t>(Prospero::ImageType::kColor2D) << 28u);
storage.fields[5] = 0x00700000u;
ShaderRecompiler::IR::Program storage_program{};
storage_program.stage = ShaderType::Vertex;
storage_program.resource_tracking_complete = true;
ShaderRecompiler::IR::ImageResource storage_resource{};
storage_resource.resource_class =
ShaderRecompiler::IR::ImageResourceClass::Storage;
storage_resource.numeric_class = Prospero::TextureNumericClass::Uint;
storage_resource.dimension =
ShaderRecompiler::Decoder::ImageDimension::Dim2D;
storage_resource.written = true;
storage_program.info.images.push_back(storage_resource);
set_image_bindings(storage_program);
ShaderRecompiler::IR::DescriptorValue storage_descriptor{};
std::copy(std::begin(storage.fields), std::end(storage.fields),
storage_descriptor.dwords.begin());
storage_descriptor.dword_count = 8;
auto mipped_storage = storage;
const auto encoded_mipped_storage_address = mipped_storage_address >> 8u;
mipped_storage.fields[0] =
static_cast<uint32_t>(encoded_mipped_storage_address);
mipped_storage.fields[1] =
static_cast<uint32_t>(encoded_mipped_storage_address >> 32u) |
(static_cast<uint32_t>(Prospero::BufferFormat::k32UInt) << 20u) |
(3u << 30u);
mipped_storage.fields[2] = 15u | (63u << 14u);
mipped_storage.fields[3] =
DstSel(4, 5, 6, 7) | (1u << 12u) | (3u << 16u) |
(static_cast<uint32_t>(Prospero::TileMode::kStandard64KB) << 20u) |
(static_cast<uint32_t>(Prospero::ImageType::kColor2D) << 28u);
mipped_storage.fields[5] = 0x00700000u | (3u << 4u);
ShaderRecompiler::IR::DescriptorValue mipped_storage_descriptor{};
std::copy(std::begin(mipped_storage.fields),
std::end(mipped_storage.fields),
mipped_storage_descriptor.dwords.begin());
mipped_storage_descriptor.dword_count = 8;
auto overwide_mipped_storage = mipped_storage;
overwide_mipped_storage.fields[3] =
(overwide_mipped_storage.fields[3] & ~(0xfu << 16u)) | (4u << 16u);
ShaderRecompiler::IR::DescriptorValue
overwide_mipped_storage_descriptor{};
std::copy(std::begin(overwide_mipped_storage.fields),
std::end(overwide_mipped_storage.fields),
overwide_mipped_storage_descriptor.dwords.begin());
overwide_mipped_storage_descriptor.dword_count = 8;
// The captured IMAGE_STORE addresses only BASE_LEVEL=1. Including
// LAST_LEVEL=4 would shift this four-mip render-target allocation.
auto fixed_storage_resource = storage_resource;
fixed_storage_resource.numeric_class = Prospero::TextureNumericClass::Float;
ShaderRecompiler::IR::DescriptorValue fixed_storage_descriptor{};
fixed_storage_descriptor.dwords = {
static_cast<uint32_t>((base + 0x300000) >> 8u),
0xc4700000u, 0x001fc7ffu, 0x91b41facu,
0, 0x00700030u, 0, 0};
fixed_storage_descriptor.dword_count = 8;
const auto fixed_storage_binding = RenderExecutorTestAccess::ResolveTexture(
executor, fixed_storage_resource, fixed_storage_descriptor);
Require(name, "fixed storage preserves physical mip layout",
fixed_storage_binding.desc.info.resources.levels == 4 &&
fixed_storage_binding.desc.info.data.size == 0xb80000 &&
fixed_storage_binding.desc.info.mip_layout[1].offset == 0x180000 &&
fixed_storage_binding.desc.view_info.base_level == 1 &&
fixed_storage_binding.desc.view_info.level_count == 1 &&
texture_cache.FindTexture(fixed_storage_binding.image_id,
fixed_storage_binding.desc) != nullptr,
"IMAGE_STORE expanded an inaccessible mip and shifted physical storage");
auto mipped_storage_resource = storage_resource;
mipped_storage_resource.mip_mode =
ShaderRecompiler::IR::ImageMipMode::Dynamic;
mipped_storage_resource.mip_count = 3;
#if KYTY_PLATFORM != KYTY_PLATFORM_WINDOWS
ExpectFatal("MipViewPhysicalLayoutChange", [&] {
auto linear_view = overwide_mipped_storage_descriptor;
linear_view.dwords[3] &= ~(0x1fu << 20u);
(void)RenderExecutorTestAccess::ResolveTexture(executor, mipped_storage_resource,
linear_view);
});
#endif
auto sampled_overwide_resource = storage_resource;
sampled_overwide_resource.resource_class =
ShaderRecompiler::IR::ImageResourceClass::Sampled;
sampled_overwide_resource.numeric_class =
Prospero::TextureNumericClass::Uint;
sampled_overwide_resource.read = true;
sampled_overwide_resource.written = false;
{
// MLB The Show 21 compresses a standalone final block through mip 9.
constexpr uint64_t address = base + 0x180000;
ShaderRecompiler::IR::DescriptorValue descriptor{};
descriptor.dwords = {static_cast<uint32_t>(address >> 8u),
0x03e00000u, 0, 0x9019902cu, 0, 0x00700000u, 0, 0};
descriptor.dword_count = 8;
TestCase test;
test.name = "RebasedLastMipQueryStore";
test.has_user_data = true;
std::copy_n(descriptor.dwords.begin(), 8, test.user_data.begin());
test.opcodes = {ShaderOpcode::V_MOV_B32, ShaderOpcode::IMAGE_GET_RESINFO,
ShaderOpcode::IMAGE_STORE, ShaderOpcode::S_ENDPGM};
AppendVMovU32(&test.code, 24, 0);
AppendVMovU32(&test.code, 25, 0);
test.code.push_back(EncodeMimg0(0x0e, 0x3));
test.code.push_back(EncodeMimg1(0, 24));
test.code.push_back(EncodeMimg0(0x08, 0x3));
test.code.push_back(EncodeMimg1(0, 24));
AppendEnd(&test.code);
const auto compiled = CompileCase(test, SubgroupSize());
auto sampled_descriptor = descriptor;
sampled_descriptor.dwords[1] |= 9u << 16u;
const auto query = RenderExecutorTestAccess::ResolveTexture(
executor, compiled.program.info.images.at(0), sampled_descriptor);
const auto store = RenderExecutorTestAccess::ResolveTexture(
executor, compiled.program.info.images.at(1), descriptor);
Require(name, "rebased last mip",
query.image_id == store.image_id &&
query.desc.info.data.size == 256 &&
query.desc.info.resources.levels == 1 &&
query.desc.view_info.base_level == 0 &&
query.desc.view_info.level_count == 1 &&
query.desc.view_info.min_lod == 0 &&
store.desc.view_info.base_level == 0,
"equivalent selected mip changed backing or relative LOD");
Image sampled;
sampled.view = texture_cache.FindTexture(query.image_id, query.desc);
sampled.layout = vk::ImageLayout::eGeneral;
Image storage_view;
storage_view.view = texture_cache.FindTexture(store.image_id, store.desc);
storage_view.layout = vk::ImageLayout::eGeneral;
texture_cache.GetImage(store.image_id).Transit(
vk::ImageLayout::eGeneral,
vk::AccessFlagBits2::eShaderRead | vk::AccessFlagBits2::eShaderWrite,
{}, scheduler.Current().Handle());
scheduler.Finish();
Dispatch(test, compiled, {}, nullptr, &sampled, nullptr, &storage_view);
texture_cache.MarkGpuWritten(store.image_id);
Require(name, "rebased mip query and store",
ReadCachedTexel(name, context, store.image_id) ==
std::vector<uint32_t>{1, 1},
"query/store did not access the standalone 1x1 mip");
}
auto plain_mipped_storage_binding =
RenderExecutorTestAccess::ResolveTexture(executor, storage_resource,
mipped_storage_descriptor);
vk::ClearValue tail_clear{};
tail_clear.color.uint32[0] = tail_gpu_value;
TextureCacheTestAccess::ClearImage(
texture_cache, scheduler.Current(), plain_mipped_storage_binding.image_id,
{vk::ImageAspectFlagBits::eColor, 0, 4, 0, 1}, tail_clear);
auto mipped_storage_binding = RenderExecutorTestAccess::ResolveTexture(
executor, mipped_storage_resource, mipped_storage_descriptor);
auto overwide_mipped_storage_binding =
RenderExecutorTestAccess::ResolveTexture(
executor, storage_resource, overwide_mipped_storage_descriptor);
auto sampled_overwide_resolved = RenderExecutorTestAccess::ResolveTexture(
executor, sampled_overwide_resource,
overwide_mipped_storage_descriptor);
const auto tail0 = ReadCachedTexel(name, context, sampled_overwide_resolved.image_id);
const auto tail3 = ReadCachedTexel(name, context, sampled_overwide_resolved.image_id,
{}, {1, 1, 1}, 0, 3);
const auto tail4 = ReadCachedTexel(name, context, sampled_overwide_resolved.image_id,
{}, {1, 1, 1}, 0, 4);
Require(name, "expanded tail preserves GPU and backing data",
tail0 == std::vector<u32>{tail_gpu_value} &&
tail3 == std::vector<u32>{tail_gpu_value} &&
tail4 == std::vector<u32>{tail_backing_value},
"expanding a shared mip tail lost GPU writes or the added mip's backing bytes");
PreparedBindings mipped_prepared{};
ShaderRecompiler::IR::CompiledShaderInfo mipped_program{};
mipped_program.info.images.push_back(storage_resource);
mipped_program.info.images.push_back(mipped_storage_resource);
mipped_program.info.images.push_back(storage_resource);
mipped_program.info.images.push_back(sampled_overwide_resource);
ShaderRecompiler::IR::ResourceSnapshot mipped_snapshot{};
mipped_snapshot.images = {mipped_storage_descriptor,
mipped_storage_descriptor,
overwide_mipped_storage_descriptor,
overwide_mipped_storage_descriptor};
ShaderStageRuntime mipped_runtime{&mipped_program, &mipped_snapshot};
mipped_prepared.runtime = &mipped_runtime;
mipped_prepared.images.push_back(
std::move(plain_mipped_storage_binding));
mipped_prepared.images.push_back(
std::move(mipped_storage_binding));
mipped_prepared.images.push_back(
std::move(overwide_mipped_storage_binding));
mipped_prepared.images.push_back(
std::move(sampled_overwide_resolved));
executor.RebindImages(mipped_prepared);
auto &plain_mipped_binding = mipped_prepared.images[0];
auto &mipped_binding = mipped_prepared.images[1];
auto &overwide_mipped_binding = mipped_prepared.images[2];
auto &sampled_overwide_binding = mipped_prepared.images[3];
plain_mipped_binding.layout = vk::ImageLayout::eGeneral;
mipped_binding.layout = vk::ImageLayout::eGeneral;
overwide_mipped_binding.layout = vk::ImageLayout::eGeneral;
sampled_overwide_binding.layout = vk::ImageLayout::eGeneral;
Require(name, "dynamic storage mip views",
plain_mipped_binding.desc.view_info.base_level == 1 &&
plain_mipped_binding.desc.view_info.level_count == 1 &&
plain_mipped_binding.mip_views.empty() &&
plain_mipped_binding.image_view ==
texture_cache.FindTexture(plain_mipped_binding.image_id,
plain_mipped_binding.desc) &&
mipped_binding.desc.info.resources.levels == 4 &&
mipped_binding.desc.view_info.base_level == 1 &&
mipped_binding.desc.view_info.level_count == 3 &&
mipped_binding.mip_views.size() == 3 &&
mipped_binding.mip_views[0] != nullptr &&
mipped_binding.mip_views[1] != nullptr &&
mipped_binding.mip_views[2] != nullptr &&
mipped_binding.mip_views[0] != mipped_binding.mip_views[1] &&
mipped_binding.mip_views[1] != mipped_binding.mip_views[2] &&
MakeImageInfo(mipped_binding, 0).imageView ==
mipped_binding.mip_views[0] &&
MakeImageInfo(mipped_binding, 1).imageView ==
mipped_binding.mip_views[1] &&
MakeImageInfo(mipped_binding, 2).imageView ==
mipped_binding.mip_views[2],
"base-1 through last-3 storage view did not create three "
"ordered one-level descriptors");
Require(name, "over-wide fixed storage mip view",
overwide_mipped_storage.LastLevel() == 4 &&
overwide_mipped_storage.MaxMip() == 3 &&
overwide_mipped_binding.image_id ==
plain_mipped_binding.image_id &&
overwide_mipped_binding.desc.info.resources.levels == 4 &&
overwide_mipped_binding.desc.view_info.base_level == 1 &&
overwide_mipped_binding.desc.view_info.level_count == 1 &&
overwide_mipped_binding.mip_views.empty() &&
overwide_mipped_binding.image_view ==
plain_mipped_binding.image_view,
"fixed storage view included mips that IMAGE_STORE cannot address");
Require(name, "over-wide sampled mip view",
sampled_overwide_binding.image_id ==
plain_mipped_binding.image_id &&
sampled_overwide_binding.desc.info.resources.levels == 5 &&
sampled_overwide_binding.desc.view_info.base_level == 1 &&
sampled_overwide_binding.desc.view_info.level_count == 4 &&
sampled_overwide_binding.mip_views.empty() &&
sampled_overwide_binding.image_view != nullptr,
"sampled view lost addressable mips in the allocated tail");
const auto *image_storage = mipped_prepared.images.data();
const auto *mip_storage = mipped_prepared.images[1].mip_views.data();
executor.PrepareBindings(mipped_runtime, mipped_prepared);
Require(name, "prepared mip scratch",
mipped_prepared.images.data() == image_storage &&
mipped_prepared.images[1].mip_views.empty() &&
mipped_prepared.images[1].mip_views.capacity() >= 3,
"repeated image preparation retained stale mip views or reallocated scratch");
executor.RebindImages(mipped_prepared);
Require(name, "prepared mip views rebound",
mipped_prepared.images[1].mip_views.size() == 3 &&
mipped_prepared.images[1].mip_views.data() == mip_storage,
"reused image scratch did not rebuild dynamic storage mip views");
// Streaming T# clamps apply after S# max LOD, without changing the view
// base.
{
constexpr uint64_t lod_address = base + 0x170000;
auto lod_descriptor = mipped_storage_descriptor;
lod_descriptor.dwords[0] = static_cast<uint32_t>(lod_address >> 8u);
lod_descriptor.dwords[1] =
static_cast<uint32_t>(lod_address >> 40u) |
(static_cast<uint32_t>(Prospero::BufferFormat::k32Float) << 20u) |
(3u << 30u);
lod_descriptor.dwords[3] &= ~(0xfu << 12u);
ShaderSamplerResource lod_sampler{
{0, 0,
static_cast<uint32_t>(Prospero::SamplerMipFilter::kLinear) << 26u,
0}};
TestCase lod_test;
lod_test.name = "TextureMinLodAfterSamplerClamp";
lod_test.has_user_data = true;
lod_test.image_descriptor_swizzle = mipped_storage.DstSelXYZW();
std::copy_n(lod_descriptor.dwords.begin(), 8,
lod_test.user_data.begin());
std::copy_n(lod_sampler.fields, 4, lod_test.user_data.begin() + 8);
lod_test.user_data[50] = sizeof(uint32_t);
lod_test.opcodes = {ShaderOpcode::V_MOV_B32, ShaderOpcode::IMAGE_SAMPLE,
ShaderOpcode::BUFFER_STORE_DWORD,
ShaderOpcode::S_ENDPGM};
lod_test.required_spirv = {"OpImageSampleExplicitLod"};
AppendVMovLiteral(&lod_test.code, 20, std::bit_cast<uint32_t>(0.5f));
AppendVMovLiteral(&lod_test.code, 21, std::bit_cast<uint32_t>(0.5f));
lod_test.code.push_back(EncodeMimg0(0x27, 1));
lod_test.code.push_back(EncodeMimg1(0, 20, 0, 2));
AppendStoreVgpr(&lod_test.code, 0, 0);
AppendEnd(&lod_test.code);
const auto lod_program = CompileCase(lod_test, SubgroupSize());
#if KYTY_PLATFORM != KYTY_PLATFORM_WINDOWS
ExpectFatal("TextureMinLodBeyondView", [&] {
auto invalid_lod = lod_descriptor;
invalid_lod.dwords[1] |= 1024u << 8u;
(void)RenderExecutorTestAccess::ResolveTexture(
executor, lod_program.program.info.images[0], invalid_lod);
});
#endif
const auto lod_binding = RenderExecutorTestAccess::ResolveTexture(
executor, lod_program.program.info.images[0], lod_descriptor);
for (uint32_t mip = 0; mip < 4; ++mip) {
vk::ClearValue clear{};
clear.color.float32[0] = static_cast<float>(1u << mip);
TextureCacheTestAccess::ClearImage(
texture_cache, scheduler.Current(), lod_binding.image_id,
{vk::ImageAspectFlagBits::eColor, mip, 1, 0, 1}, clear);
}
const auto sampler = context.GetSamplerCache().GetSampler(
lod_sampler, lod_program.program.info.samplers[0].integer_border);
auto output = CreateStorageBuffer(lod_test.name, {}, 1);
struct LodCase {
uint32_t base_level;
uint32_t min_lod;
float expected;
float integer_min_lod_expected;
};
constexpr std::array lod_cases{
LodCase{0, 0, 1.0f, 1.0f}, LodCase{0, 256, 2.0f, 2.0f},
LodCase{0, 384, 3.0f, 2.0f}, LodCase{1, 384, 3.0f, 2.0f},
LodCase{1, 512, 4.0f, 4.0f}, LodCase{1, 0, 2.0f, 2.0f},
LodCase{0, 256, 2.0f, 2.0f}};
std::array<vk::ImageView, lod_cases.size()> views{};
for (size_t index = 0; index < lod_cases.size(); ++index) {
const auto &test = lod_cases[index];
lod_descriptor.dwords[1] =
(lod_descriptor.dwords[1] & ~0xfff00u) | (test.min_lod << 8u);
lod_descriptor.dwords[3] =
(lod_descriptor.dwords[3] & ~(0xfu << 12u)) |
(test.base_level << 12u);
std::copy_n(lod_descriptor.dwords.begin(), 8,
lod_test.user_data.begin());
const auto binding = RenderExecutorTestAccess::ResolveTexture(
executor, lod_program.program.info.images[0], lod_descriptor);
views[index] =
texture_cache.FindTexture(binding.image_id, binding.desc);
auto &image = texture_cache.GetImage(binding.image_id);
image.Transit(vk::ImageLayout::eShaderReadOnlyOptimal,
vk::AccessFlagBits2::eShaderRead, {},
scheduler.Current().Handle());
Image sampled;
sampled.view = views[index];
sampled.layout = image.backing.state.layout;
scheduler.Finish();
Dispatch(lod_test, lod_program, output, nullptr, &sampled, nullptr,
nullptr, sampler);
const auto result = ReadBuffer(lod_test.name, output, 1)[0];
// Vulkan permits flooring imageViewMinLod instead of retaining its
// fraction.
Require(lod_test.name, "sampled mip",
result == std::bit_cast<uint32_t>(test.expected) ||
result == std::bit_cast<uint32_t>(
test.integer_min_lod_expected),
"texture clamp was lost, applied before sampler max LOD, or "
"rebased incorrectly");
}
Require(lod_test.name, "view identity",
views[0] != views[1] && views[1] != views[2] &&
views[2] != views[3] && views[1] == views.back(),
"different texture clamps aliased or identical clamped views "
"were not reused");
ShaderRecompiler::IR::CompiledShaderInfo gather_info{};
gather_info.stage = ShaderType::Compute;
gather_info.info = lod_program.program.info;
gather_info.info.images[0].mip_mode =
ShaderRecompiler::IR::ImageMipMode::Dynamic;
gather_info.info.images[0].mip_count = 4;
auto gather_snapshot = lod_program.resources;
gather_snapshot.images[0] = lod_descriptor;
ShaderStageRuntime gather_runtime{&gather_info, &gather_snapshot};
PreparedBindings gather_bindings;
executor.PrepareBindings(gather_runtime, gather_bindings);
executor.RebindImages(gather_bindings);
const auto &gather_binding = gather_bindings.images[0];
Require(lod_test.name, "dynamic sampled mip views",
gather_binding.desc.view_info.min_lod == 256u &&
gather_binding.mip_views.size() == 4,
"dynamic gather lost the guest minimum or accessible mip range");
for (uint32_t mip = 0; mip < 4; ++mip) {
auto single_mip = gather_binding.desc;
single_mip.view_info.base_level = mip;
single_mip.view_info.level_count = 1;
single_mip.view_info.min_lod = 0;
Require(lod_test.name, "gather minimum applied once",
gather_binding.mip_views[mip] == texture_cache.FindTexture(
gather_binding.image_id, single_mip),
"the gather mip view reapplied the shader's streaming clamp");
}
DestroyBuffer(&output);
RenderExecutorTestAccess::ResetBindings(executor);
}
auto srgb_storage = storage;
constexpr auto srgb_format =
static_cast<uint32_t>(Prospero::BufferFormat::k8_8_8_8Srgb);
constexpr uint64_t srgb_storage_address = base + 0xc0000;
const auto encoded_srgb_storage_address = srgb_storage_address >> 8u;
srgb_storage.fields[0] =
static_cast<uint32_t>(encoded_srgb_storage_address);
srgb_storage.fields[1] =
static_cast<uint32_t>(encoded_srgb_storage_address >> 32u) |
(srgb_format << 20u);
ShaderRecompiler::IR::DescriptorValue srgb_storage_descriptor{};
std::copy(std::begin(srgb_storage.fields), std::end(srgb_storage.fields),
srgb_storage_descriptor.dwords.begin());
srgb_storage_descriptor.dword_count = 8;
auto srgb_storage_resource = storage_resource;
srgb_storage_resource.resource_class =
ShaderRecompiler::IR::ImageResourceClass::Storage;
srgb_storage_resource.numeric_class =
Prospero::TextureNumericClass::Float;
const auto srgb_storage_binding =
RenderExecutorTestAccess::ResolveTexture(
executor, srgb_storage_resource, srgb_storage_descriptor);
const auto srgb_storage_view = texture_cache.FindTexture(
srgb_storage_binding.image_id, srgb_storage_binding.desc);
Require(name, "sRGB storage view",
srgb_storage_view != nullptr &&
srgb_storage_binding.desc.info.pixel_format ==
vk::Format::eR8G8B8A8Srgb &&
srgb_storage_binding.desc.view_info.format ==
vk::Format::eR8G8B8A8Unorm &&
texture_cache.GetImage(srgb_storage_binding.image_id)
.backing.format == vk::Format::eR8G8B8A8Srgb,
"storage descriptor did not preserve its sRGB backing and "
"select an UNORM Vulkan view");
ShaderTextureResource sint_storage{{0x01514b00u, 0xc1500000u, 0x000bc00bu,
0x91b00204u, 0x00000000u, 0x00700000u,
0x102b0000u, 0x0001514au}};
Require(name, "PPSA06888 R32 SINT descriptor",
sint_storage.Base40() == 0x1514b0000ull &&
sint_storage.Width5() + 1u == 48 &&
sint_storage.Height5() + 1u == 48 &&
sint_storage.Depth() + 1u == 1 &&
sint_storage.Format() == Prospero::BufferFormat::k32SInt &&
sint_storage.Type() == Prospero::ImageType::kColor2D &&
sint_storage.TileMode() ==
Prospero::TileMode::kRenderTarget &&
sint_storage.DstSelXYZW() == DstSel(4, 0, 0, 1),
"captured write-only signed storage descriptor was decoded "
"incorrectly");
const uint64_t mapped_sint_address = base + 0xe0000;
const auto encoded_sint_address = mapped_sint_address >> 8u;
sint_storage.fields[0] = static_cast<uint32_t>(encoded_sint_address);
sint_storage.fields[1] =
(sint_storage.fields[1] & ~0xffu) |
static_cast<uint32_t>(encoded_sint_address >> 32u);
constexpr uint64_t sint_metadata_address = base + 0x160000;
const auto encoded_sint_metadata = sint_metadata_address >> 8u;
sint_storage.fields[6] = (sint_storage.fields[6] & 0x00ffffffu) |
(static_cast<uint32_t>(encoded_sint_metadata & 0xffu) << 24u);
sint_storage.fields[7] = static_cast<uint32_t>(encoded_sint_metadata >> 8u);
WriteMetadata(context, sint_metadata_address, 0x1000, UINT32_MAX);
ShaderRecompiler::IR::DescriptorValue sint_storage_descriptor{};
std::copy(std::begin(sint_storage.fields), std::end(sint_storage.fields),
sint_storage_descriptor.dwords.begin());
sint_storage_descriptor.dword_count = 8;
auto sint_storage_resource = srgb_storage_resource;
sint_storage_resource.resource_class =
ShaderRecompiler::IR::ImageResourceClass::Storage;
sint_storage_resource.numeric_class = Prospero::TextureNumericClass::Uint;
const auto sint_storage_binding =
RenderExecutorTestAccess::ResolveTexture(
executor, sint_storage_resource, sint_storage_descriptor);
const auto sint_storage_view = texture_cache.FindTexture(
sint_storage_binding.image_id, sint_storage_binding.desc);
Require(name, "PPSA06888 raw R32 SINT storage view",
sint_storage_view != nullptr &&
sint_storage_binding.desc.info.metadata.range ==
GuestRange{sint_metadata_address, 0x1000} &&
sint_storage_binding.desc.info.data.size == 0x10000 &&
sint_storage_binding.desc.info.pixel_format ==
vk::Format::eR32Sint &&
sint_storage_binding.desc.view_info.format ==
vk::Format::eR32Uint &&
texture_cache.GetImage(sint_storage_binding.image_id)
.backing.format == vk::Format::eR32Sint,
"write-only R32 SINT storage did not select a bit-compatible "
"uint view");
auto narrowed_storage = storage;
constexpr uint64_t narrowed_storage_address = base + 0xd0000;
const auto encoded_narrowed_address = narrowed_storage_address >> 8u;
narrowed_storage.fields[0] =
static_cast<uint32_t>(encoded_narrowed_address);
narrowed_storage.fields[1] =
static_cast<uint32_t>(encoded_narrowed_address >> 32u) |
(static_cast<uint32_t>(stencil_format) << 20u);
narrowed_storage.fields[3] =
DstSel(4, 5, 6, 7) | (static_cast<uint32_t>(linear) << 20u) |
(static_cast<uint32_t>(Prospero::ImageType::kColor1DArray) << 28u);
narrowed_storage.fields[4] = 1u | (1u << 16u);
ShaderRecompiler::IR::DescriptorValue narrowed_storage_descriptor{};
std::copy(std::begin(narrowed_storage.fields),
std::end(narrowed_storage.fields),
narrowed_storage_descriptor.dwords.begin());
narrowed_storage_descriptor.dword_count = 8;
auto narrowed_storage_resource = storage_resource;
narrowed_storage_resource.dimension =
ShaderRecompiler::Decoder::ImageDimension::Dim1D;
const auto narrowed_storage_binding =
RenderExecutorTestAccess::ResolveTexture(
executor, narrowed_storage_resource, narrowed_storage_descriptor);
const auto narrowed_storage_view = texture_cache.FindTexture(
narrowed_storage_binding.image_id, narrowed_storage_binding.desc);
const auto &narrowed_storage_image =
texture_cache.GetImage(narrowed_storage_binding.image_id);
Require(name, "narrowed 1D array storage view",
narrowed_storage_view != nullptr &&
narrowed_storage_binding.desc.info.type ==
Prospero::ImageType::kColor1D &&
narrowed_storage_binding.desc.info.resources.layers == 2 &&
narrowed_storage_binding.desc.view_info.type ==
vk::ImageViewType::e1D &&
narrowed_storage_binding.desc.view_info.base_layer == 1 &&
narrowed_storage_binding.desc.view_info.layer_count == 1 &&
narrowed_storage_image.backing.image_type ==
vk::ImageType::e1D,
"non-array 1D specialization did not select the descriptor "
"base layer from its 1D-array backing");
ShaderRecompiler::IR::ResourceSnapshot storage_snapshot{};
storage_snapshot.images.push_back(storage_descriptor);
ShaderRecompiler::IR::CompiledShaderInfo storage_info{};
storage_info.stage = storage_program.stage;
storage_info.info = std::move(storage_program.info);
storage_info.bindings = std::move(storage_program.bindings);
ShaderStageRuntime storage_runtime{&storage_info, &storage_snapshot};
PreparedBindings storage_discovery;
executor.PrepareBindings(storage_runtime, storage_discovery);
const auto storage_id = storage_discovery.images[0].image_id;
Require(name, "storage prefetch purity",
storage_discovery.images[0].image_view == nullptr &&
texture_cache.GetImage(storage_id).binding.is_bound &&
!texture_cache.GetImage(storage_id).usage.storage &&
!TextureCacheTestAccess::PendingDownload(texture_cache,
storage_id),
"the first descriptor pass performed final storage acquisition");
RenderExecutorTestAccess::ResetBindings(executor);
ShaderRecompiler::IR::Program sampled_program{};
sampled_program.stage = ShaderType::Pixel;
sampled_program.resource_tracking_complete = true;
ShaderRecompiler::IR::ImageResource sampled_resource{};
sampled_resource.resource_class =
ShaderRecompiler::IR::ImageResourceClass::Sampled;
sampled_resource.numeric_class = Prospero::TextureNumericClass::Uint;
sampled_resource.dimension =
ShaderRecompiler::Decoder::ImageDimension::Dim2D;
sampled_resource.read = true;
sampled_program.info.images.push_back(sampled_resource);
set_image_bindings(sampled_program);
ShaderRecompiler::IR::ResourceSnapshot sampled_snapshot{};
sampled_snapshot.images.push_back(storage_descriptor);
ShaderRecompiler::IR::CompiledShaderInfo sampled_info{};
sampled_info.stage = sampled_program.stage;
sampled_info.info = std::move(sampled_program.info);
sampled_info.bindings = std::move(sampled_program.bindings);
ShaderStageRuntime sampled_runtime{&sampled_info, &sampled_snapshot};
constexpr uint64_t ordered_sampled_address = base + 0x10000;
const uint32_t ordered_sampled_value = 0x89abcdefu;
std::memcpy(reinterpret_cast<uint8_t *>(mapped) + 0x10000,
&ordered_sampled_value, sizeof(ordered_sampled_value));
auto ordered_sampled = storage;
const auto encoded_ordered_address = ordered_sampled_address >> 8u;
ordered_sampled.fields[0] =
static_cast<uint32_t>(encoded_ordered_address);
ordered_sampled.fields[1] =
static_cast<uint32_t>(encoded_ordered_address >> 32u) |
(static_cast<uint32_t>(stencil_format) << 20u);
ShaderRecompiler::IR::DescriptorValue ordered_descriptor{};
std::copy(std::begin(ordered_sampled.fields),
std::end(ordered_sampled.fields),
ordered_descriptor.dwords.begin());
ordered_descriptor.dword_count = 8;
ShaderRecompiler::IR::ResourceSnapshot ordered_snapshot{};
ordered_snapshot.images.push_back(ordered_descriptor);
ShaderStageRuntime ordered_sampled_runtime{sampled_runtime.program,
&ordered_snapshot};
auto ordered_bindings = RenderExecutorTestAccess::PrepareGraphicsBindings(
executor, storage_runtime, ordered_sampled_runtime, true);
const auto ordered_sampled_id =
ordered_bindings.pixel->images[0].image_id;
Require(
name, "VS-before-PS retained-owner order",
ordered_bindings.vertex[0].images[0].image_id == storage_id &&
ordered_sampled_id != storage_id &&
RenderExecutorTestAccess::BoundImagesInOrder(executor, storage_id,
ordered_sampled_id),
"production graphics binding did not retain vertex resources "
"before pixel resources");
RenderExecutorTestAccess::ResetBindings(executor);
auto graphics_bindings =
RenderExecutorTestAccess::PrepareGraphicsBindings(
executor, storage_runtime, sampled_runtime, true);
const auto &storage_binding =
graphics_bindings.vertex[0].images[0];
const auto &sampled_binding =
graphics_bindings.pixel->images[0];
Require(name, "storage final acquisition",
storage_binding.image_view != nullptr &&
texture_cache.GetImage(storage_id).usage.storage &&
texture_cache.GetImage(storage_id).IsGpuModified() &&
TextureCacheTestAccess::PendingDownload(texture_cache,
storage_id),
"the production graphics binding path did not establish GPU "
"ownership before storage readback");
Require(name, "VS-to-PS image acquisition order",
storage_binding.image_id == storage_id &&
sampled_binding.image_id == storage_id &&
texture_cache.GetImage(storage_id).binding.force_general &&
texture_cache.GetImage(storage_id).binding.shader_write &&
texture_cache.GetImage(storage_id).IsGpuModified() &&
texture_cache.GetImage(storage_id).usage.storage,
"the production graphics binding path did not complete vertex "
"storage acquisition before pixel sampling");
descriptor_pipelines.push_back(RenderExecutorTestAccess::CommitBindings(
executor, scheduler.Current(), graphics_bindings.vertex[0],
*graphics_bindings.pixel));
Require(name, "early writable alias retention",
texture_cache.GetImage(storage_id).backing.state.access_mask ==
(vk::AccessFlagBits2::eShaderRead |
vk::AccessFlagBits2::eShaderWrite),
"a later sampled alias dropped an earlier storage write access");
RenderExecutorTestAccess::ResetBindings(executor);
ShaderRecompiler::IR::Program vertex_sampled_program{};
vertex_sampled_program.stage = ShaderType::Vertex;
vertex_sampled_program.resource_tracking_complete = true;
vertex_sampled_program.info.images.push_back(sampled_resource);
set_image_bindings(vertex_sampled_program);
ShaderRecompiler::IR::CompiledShaderInfo vertex_sampled_info{};
vertex_sampled_info.stage = vertex_sampled_program.stage;
vertex_sampled_info.info = std::move(vertex_sampled_program.info);
vertex_sampled_info.bindings =
std::move(vertex_sampled_program.bindings);
ShaderStageRuntime vertex_sampled_runtime{
&vertex_sampled_info, sampled_runtime.resources};
ShaderRecompiler::IR::Program pixel_storage_program{};
pixel_storage_program.stage = ShaderType::Pixel;
pixel_storage_program.resource_tracking_complete = true;
pixel_storage_program.info.images.push_back(storage_resource);
set_image_bindings(pixel_storage_program);
ShaderRecompiler::IR::CompiledShaderInfo pixel_storage_info{};
pixel_storage_info.stage = pixel_storage_program.stage;
pixel_storage_info.info = std::move(pixel_storage_program.info);
pixel_storage_info.bindings = std::move(pixel_storage_program.bindings);
ShaderStageRuntime pixel_storage_runtime{&pixel_storage_info,
storage_runtime.resources};
auto writable_alias_bindings =
RenderExecutorTestAccess::PrepareGraphicsBindings(
executor, vertex_sampled_runtime, pixel_storage_runtime, true);
Require(name, "late writable alias promotion",
texture_cache.GetImage(storage_id).binding.force_general,
"an already sampled image was not promoted when a later "
"storage alias bound the same backing");
descriptor_pipelines.push_back(RenderExecutorTestAccess::CommitBindings(
executor, scheduler.Current(), writable_alias_bindings.vertex[0],
*writable_alias_bindings.pixel));
Require(
name, "forced-general descriptor capture",
writable_alias_bindings.vertex[0].images[0].layout ==
vk::ImageLayout::eGeneral &&
writable_alias_bindings.pixel->images[0].layout ==
vk::ImageLayout::eGeneral &&
MakeImageInfo(writable_alias_bindings.vertex[0].images[0])
.imageLayout == vk::ImageLayout::eGeneral &&
MakeImageInfo(writable_alias_bindings.pixel->images[0])
.imageLayout == vk::ImageLayout::eGeneral &&
texture_cache.GetImage(storage_id).backing.state.layout ==
vk::ImageLayout::eGeneral &&
texture_cache.GetImage(storage_id).backing.state.access_mask ==
(vk::AccessFlagBits2::eShaderRead |
vk::AccessFlagBits2::eShaderWrite),
"sampled/storage aliases did not retain the promoted general "
"layout and writable access in both generated descriptors");
RenderExecutorTestAccess::ResetBindings(executor);
const std::array<uint32_t, 3> split_mip_data{0x01020304u, 0x05060708u,
0x090a0b0cu};
std::memcpy(reinterpret_cast<uint8_t *>(mapped) + 0x50000,
split_mip_data.data(), sizeof(split_mip_data));
ImageDesc split_desc{};
split_desc.type = BindingType::Texture;
split_desc.info.data = {base + 0x50000, sizeof(split_mip_data)};
split_desc.info.pixel_format = vk::Format::eR32Uint;
split_desc.info.guest_format = Prospero::BufferFormat::k32UInt;
split_desc.info.type = Prospero::ImageType::kColor2D;
split_desc.info.extent = {2, 1, 1};
split_desc.info.resources = {2, 1};
split_desc.info.pitch = 2;
split_desc.info.bytes_per_block = 4;
split_desc.info.samples = 1;
split_desc.info.tile_mode = linear;
split_desc.info.mip_layout[0] = {0, 8, 2, 1, 0, 8, 2};
split_desc.info.mip_layout[1] = {8, 4, 1, 1, 8, 4, 1};
split_desc.info.tiling.block.bytes_per_element = 4;
split_desc.view_info.format = split_desc.info.pixel_format;
split_desc.view_info.type = vk::ImageViewType::e2D;
split_desc.view_info.aspect = vk::ImageAspectFlagBits::eColor;
split_desc.view_info.level_count = 2;
split_desc.view_info.layer_count = 1;
split_desc.view_info.usage = vk::ImageUsageFlagBits::eSampled;
const auto split_id = texture_cache.FindImage(split_desc);
auto split_storage_desc = split_desc;
split_storage_desc.type = BindingType::Storage;
split_storage_desc.view_info.base_level = 0;
split_storage_desc.view_info.level_count = 1;
split_storage_desc.view_info.usage = vk::ImageUsageFlagBits::eStorage;
auto split_sampled_desc = split_desc;
split_sampled_desc.view_info.base_level = 1;
split_sampled_desc.view_info.level_count = 1;
split_sampled_desc.view_info.usage = vk::ImageUsageFlagBits::eSampled;
ShaderRecompiler::IR::Program split_ir{};
split_ir.stage = ShaderType::Vertex;
split_ir.resource_tracking_complete = true;
split_ir.info.images = {storage_resource, sampled_resource};
set_image_bindings(split_ir);
ShaderRecompiler::IR::CompiledShaderInfo split_program{};
split_program.stage = split_ir.stage;
split_program.info = std::move(split_ir.info);
split_program.bindings = std::move(split_ir.bindings);
PreparedBindings split_bindings{};
ShaderRecompiler::IR::ResourceSnapshot split_snapshot;
ShaderStageRuntime split_runtime{&split_program, &split_snapshot};
split_bindings.runtime = &split_runtime;
split_bindings.images.push_back(
{split_id, texture_cache.FindTexture(split_id, split_storage_desc),
split_storage_desc});
split_bindings.images.push_back(
{split_id, texture_cache.FindTexture(split_id, split_sampled_desc),
split_sampled_desc});
descriptor_pipelines.push_back(RenderExecutorTestAccess::CommitBindings(
executor, scheduler.Current(), split_bindings));
const auto &split_image = texture_cache.GetImage(split_id);
Require(
name, "per-binding subresource layouts",
split_bindings.images[0].layout ==
vk::ImageLayout::eGeneral &&
split_bindings.images[1].layout ==
vk::ImageLayout::eShaderReadOnlyOptimal &&
MakeImageInfo(split_bindings.images[0]).imageLayout ==
vk::ImageLayout::eGeneral &&
MakeImageInfo(split_bindings.images[1]).imageLayout ==
vk::ImageLayout::eShaderReadOnlyOptimal &&
split_image.backing.subresource_states.size() == 2 &&
split_image.backing.subresource_states[0].layout ==
vk::ImageLayout::eGeneral &&
split_image.backing.subresource_states[1].layout ==
vk::ImageLayout::eShaderReadOnlyOptimal,
"descriptor layouts were queried after a later mip transition "
"instead of being captured at each binding");
auto disjoint_depth_desc = split_desc;
disjoint_depth_desc.type = BindingType::DepthTarget;
disjoint_depth_desc.info.data.address = base + 0xe00000;
disjoint_depth_desc.info.pixel_format = vk::Format::eD32Sfloat;
disjoint_depth_desc.info.guest_format = Prospero::BufferFormat::k32Float;
disjoint_depth_desc.view_info.format = vk::Format::eD32Sfloat;
disjoint_depth_desc.view_info.aspect = vk::ImageAspectFlagBits::eDepth;
disjoint_depth_desc.view_info.level_count = 1;
disjoint_depth_desc.view_info.usage =
vk::ImageUsageFlagBits::eDepthStencilAttachment;
const auto disjoint_depth_id = texture_cache.FindImage(disjoint_depth_desc);
auto disjoint_sampled_desc = disjoint_depth_desc;
disjoint_sampled_desc.type = BindingType::Texture;
disjoint_sampled_desc.view_info.base_level = 1;
disjoint_sampled_desc.view_info.usage = vk::ImageUsageFlagBits::eSampled;
const auto disjoint_sampled_view =
texture_cache.FindTexture(disjoint_depth_id, disjoint_sampled_desc);
RenderDepthInfo disjoint_depth{};
disjoint_depth.desc = disjoint_depth_desc;
disjoint_depth.image_id = disjoint_depth_id;
disjoint_depth.depth_test_enable = true;
disjoint_depth.depth_write_enable = true;
disjoint_depth.depth_compare_op = vk::CompareOp::eAlways;
PreparedBindings disjoint_binding{};
disjoint_binding.runtime = &sampled_runtime;
disjoint_binding.images.push_back(
{disjoint_depth_id, disjoint_sampled_view, disjoint_sampled_desc});
RenderExecutorTestAccess::BindRenderTarget(executor, disjoint_depth_id);
vk::ImageAspectFlags disjoint_feedback;
std::array<PreparedBindings *, 1> disjoint_stages{&disjoint_binding};
RenderColorInfo no_disjoint_color{};
const auto disjoint_rendering = RenderExecutorTestAccess::AcquireRenderTargets(
executor, scheduler.Current(), &no_disjoint_color, 0, disjoint_depth,
disjoint_stages, &disjoint_feedback);
descriptor_pipelines.push_back(RenderExecutorTestAccess::CommitBindings(
executor, scheduler.Current(), disjoint_binding));
const auto &disjoint_image = texture_cache.GetImage(disjoint_depth_id);
Require(name, "disjoint depth sampling during depth writes",
!disjoint_feedback &&
disjoint_rendering.depth_stencil_attachment.image_layout ==
vk::ImageLayout::eAttachmentFeedbackLoopOptimalEXT &&
disjoint_binding.images[0].layout ==
disjoint_rendering.depth_stencil_attachment.image_layout &&
disjoint_image.backing.state.layout ==
disjoint_rendering.depth_stencil_attachment.image_layout &&
disjoint_image.backing.subresource_states.empty(),
"disjoint sampling used inconsistent image layouts or enabled feedback");
RenderExecutorTestAccess::ResetBindings(executor);
auto overlapping_sampled_desc = disjoint_sampled_desc;
overlapping_sampled_desc.view_info.base_level = 0;
const auto overlapping_view = texture_cache.FindTexture(
disjoint_depth_id, overlapping_sampled_desc);
ShaderRecompiler::IR::Program depth_views_ir{};
depth_views_ir.stage = ShaderType::Pixel;
depth_views_ir.resource_tracking_complete = true;
depth_views_ir.info.images = {sampled_resource, sampled_resource};
set_image_bindings(depth_views_ir);
ShaderRecompiler::IR::CompiledShaderInfo depth_views_program{};
depth_views_program.stage = depth_views_ir.stage;
depth_views_program.info = std::move(depth_views_ir.info);
depth_views_program.bindings = std::move(depth_views_ir.bindings);
ShaderRecompiler::IR::ResourceSnapshot depth_views_snapshot;
ShaderStageRuntime depth_views_runtime{&depth_views_program, &depth_views_snapshot};
PreparedBindings depth_views_binding{};
depth_views_binding.runtime = &depth_views_runtime;
depth_views_binding.images.push_back(
{disjoint_depth_id, overlapping_view, overlapping_sampled_desc});
depth_views_binding.images.push_back(
{disjoint_depth_id, disjoint_sampled_view, disjoint_sampled_desc});
RenderExecutorTestAccess::BindRenderTarget(executor, disjoint_depth_id);
std::array<PreparedBindings *, 1> depth_views_stages{&depth_views_binding};
vk::ImageAspectFlags depth_views_feedback;
const auto depth_views_rendering = RenderExecutorTestAccess::AcquireRenderTargets(
executor, scheduler.Current(), &no_disjoint_color, 0, disjoint_depth,
depth_views_stages, &depth_views_feedback);
descriptor_pipelines.push_back(RenderExecutorTestAccess::CommitBindings(
executor, scheduler.Current(), depth_views_binding));
Require(name, "overlapping and disjoint depth views",
overlapping_view != nullptr &&
depth_views_feedback == vk::ImageAspectFlagBits::eDepth &&
depth_views_rendering.depth_stencil_attachment.image_layout ==
vk::ImageLayout::eAttachmentFeedbackLoopOptimalEXT &&
MakeImageInfo(depth_views_binding.images[0]).imageLayout ==
depth_views_rendering.depth_stencil_attachment.image_layout &&
MakeImageInfo(depth_views_binding.images[1]).imageLayout ==
depth_views_rendering.depth_stencil_attachment.image_layout,
"two depth views chose different descriptor layouts or feedback aspects");
Libs::LibKernel::Memory::WriteBacking(
storage_address, &storage_stale_value, sizeof(storage_stale_value));
texture_cache.ProcessDownloadImages();
Require(
name, "storage acquisition download consumption",
!TextureCacheTestAccess::PendingDownload(texture_cache, storage_id),
"submit-time processing retained an acquired storage request");
RenderExecutorTestAccess::ResetBindings(executor);
scheduler.Finish();
scheduler.DrainPriorityOperations();
uint32_t storage_downloaded_value = 0;
Libs::LibKernel::Memory::TryReadBacking(storage_address,
&storage_downloaded_value,
sizeof(storage_downloaded_value));
const uint32_t storage_expected_value =
(storage_stale_value & 0xffffff00u) | (storage_native_value & 0xffu);
Require(name, "storage acquisition readback window",
storage_downloaded_value == storage_expected_value &&
texture_cache.GetImage(storage_id).IsGpuModified(),
"readback enrollment was consumed before final storage "
"ownership became visible");
constexpr uint64_t phased_depth_address = base + 0x90000;
constexpr uint64_t phased_stencil_address = base + 0xa0000;
constexpr uint64_t phased_htile_address = base + 0xb0000;
HW::DepthRenderTarget phased_depth_target{};
phased_depth_target.z_info.format = Prospero::DepthFormat::kZ32F;
phased_depth_target.z_info.texture_compatibility =
Prospero::TextureCompatiblePlaneCompression::kEnable;
phased_depth_target.z_info.htile_acceleration = true;
phased_depth_target.z_info.z_compare_base = Prospero::ZCompareBase::kZMax;
phased_depth_target.stencil_info.format = Prospero::StencilFormat::k8UInt;
phased_depth_target.stencil_info.texture_compatibility =
Prospero::TextureCompatibleStencil::kEnable;
phased_depth_target.stencil_info.htile_stencil_disabled = false;
phased_depth_target.z_read_base_addr = phased_depth_address;
phased_depth_target.z_write_base_addr = phased_depth_address;
phased_depth_target.stencil_read_base_addr = phased_stencil_address;
phased_depth_target.stencil_write_base_addr = phased_stencil_address;
phased_depth_target.htile_data_base_addr = phased_htile_address;
phased_depth_target.size.valid = true;
registers.SetDepthRenderTarget(phased_depth_target);
HW::DepthControl phased_depth_control{};
phased_depth_control.z_write_enable = true;
registers.SetDepthControl(phased_depth_control);
registers.SetRenderControl({});
RenderDepthInfo disabled_depth{};
RenderExecutorTestAccess::ResolveRenderDepthTarget(
executor, scheduler.Current(), disabled_depth);
Require(name, "disabled tests with stale depth write enable",
!disabled_depth.image_id,
"a dormant depth write bit bound a depth attachment");
phased_depth_control.z_enable = true;
registers.SetDepthControl(phased_depth_control);
HW::RenderControl phased_render_control{};
phased_render_control.resummarize_enable = true;
registers.SetRenderControl(phased_render_control);
RenderDepthInfo phased_depth{};
RenderExecutorTestAccess::ResolveRenderDepthTarget(
executor, scheduler.Current(), phased_depth);
// PRT permits missing backing; a fully resident target keeps the same layout and image.
phased_depth_target.z_info.partially_resident = true;
phased_depth_target.stencil_info.partially_resident = true;
registers.SetDepthRenderTarget(phased_depth_target);
RenderDepthInfo resident_prt_depth{};
RenderExecutorTestAccess::ResolveRenderDepthTarget(
executor, scheduler.Current(), resident_prt_depth);
Require(name, "fully resident PRT depth identity",
resident_prt_depth.image_id == phased_depth.image_id &&
resident_prt_depth.desc.info.data == phased_depth.desc.info.data &&
resident_prt_depth.desc.info.stencil == phased_depth.desc.info.stencil,
"PRT permission changed a fully backed depth/stencil attachment");
auto non_texture_compatible_target = phased_depth_target;
non_texture_compatible_target.z_info.texture_compatibility =
Prospero::TextureCompatiblePlaneCompression::kDisable;
non_texture_compatible_target.stencil_info.texture_compatibility =
Prospero::TextureCompatibleStencil::kDisable;
registers.SetDepthRenderTarget(non_texture_compatible_target);
RenderDepthInfo non_texture_compatible_depth{};
RenderExecutorTestAccess::ResolveRenderDepthTarget(
executor, scheduler.Current(), non_texture_compatible_depth);
Require(
name, "depth texture compatibility identity",
phased_depth.image_id && (phased_depth.desc.info.metadata.kind == ImageMetadataKind::Htile) &&
non_texture_compatible_depth.image_id == phased_depth.image_id &&
non_texture_compatible_depth.desc.info.data ==
phased_depth.desc.info.data &&
non_texture_compatible_depth.desc.info.stencil ==
phased_depth.desc.info.stencil &&
non_texture_compatible_depth.desc.info.metadata.range ==
phased_depth.desc.info.metadata.range &&
phased_depth.desc.info.metadata.stencil_compressed &&
!phased_depth.depth_clear_enable &&
!texture_cache.IsMeta(phased_depth.desc.info.metadata.range.address) &&
!texture_cache.GetImage(phased_depth.image_id).IsGpuModified(),
"valid PS5 texture-compatibility policy changed logical depth image "
"identity or discovery side effects");
auto read_only_depth_target = phased_depth_target;
read_only_depth_target.z_write_base_addr = 0;
read_only_depth_target.stencil_write_base_addr = 0;
read_only_depth_target.depth_view.depth_write_disable = true;
read_only_depth_target.depth_view.stencil_write_disable = true;
registers.SetDepthRenderTarget(read_only_depth_target);
auto read_only_depth_control = phased_depth_control;
read_only_depth_control.z_write_enable = false;
read_only_depth_control.zfunc =
static_cast<uint8_t>(vk::CompareOp::eAlways);
read_only_depth_control.stencil_enable = true;
read_only_depth_control.stencilfunc =
static_cast<uint8_t>(vk::CompareOp::eAlways);
registers.SetDepthControl(read_only_depth_control);
HW::StencilControl read_only_stencil_control{};
read_only_stencil_control.stencil_zpass =
static_cast<uint8_t>(Prospero::StencilOp::kReplaceOp);
registers.SetStencilControl(read_only_stencil_control);
HW::StencilMask read_only_stencil_mask{};
read_only_stencil_mask.stencil_testval = 0x12;
read_only_stencil_mask.stencil_mask = 0xff;
read_only_stencil_mask.stencil_writemask = 0xff;
read_only_stencil_mask.stencil_opval = 0x34;
registers.SetStencilMask(read_only_stencil_mask);
RenderDepthInfo read_only_depth{};
RenderExecutorTestAccess::ResolveRenderDepthTarget(
executor, scheduler.Current(), read_only_depth);
Require(
name, "read-only depth/stencil target without write addresses",
read_only_depth.image_id == phased_depth.image_id &&
read_only_depth.desc.info.data.address == phased_depth_address &&
read_only_depth.desc.info.stencil.address == phased_stencil_address &&
!read_only_depth.depth_write_enable &&
!read_only_depth.stencil_clear_enable &&
read_only_depth.stencil_test_enable &&
read_only_depth.stencil_front.writeMask == 0 &&
read_only_depth.stencil_back.writeMask == 0 &&
read_only_depth.stencil_front.passOp ==
vk::StencilOp::eKeep &&
!read_only_depth.AttachmentWriteAspects() &&
depth_attachment_layout(read_only_depth) ==
vk::ImageLayout::eDepthStencilReadOnlyOptimal,
"a PS5 read-only depth/stencil target required write addresses or "
"retained host writes");
auto read_only_render_control = phased_render_control;
read_only_render_control.stencil_clear_enable = true;
registers.SetRenderControl(read_only_render_control);
RenderDepthInfo read_only_clear_depth{};
RenderExecutorTestAccess::ResolveRenderDepthTarget(
executor, scheduler.Current(), read_only_clear_depth);
Require(name, "read-only stencil clear suppression",
read_only_clear_depth.image_id == phased_depth.image_id &&
!read_only_clear_depth.stencil_clear_enable &&
!read_only_clear_depth.AttachmentWriteAspects() &&
depth_attachment_layout(read_only_clear_depth) ==
vk::ImageLayout::eDepthStencilReadOnlyOptimal,
"a PS5 target-level stencil write disable retained a host clear");
registers.SetDepthRenderTarget(phased_depth_target);
registers.SetDepthControl(phased_depth_control);
registers.SetStencilControl({});
registers.SetStencilMask({});
registers.SetRenderControl(phased_render_control);
RenderColorInfo no_color{};
auto phased_rendering = RenderExecutorTestAccess::AcquireRenderTargets(
executor, scheduler.Current(), &no_color, 0, phased_depth);
Require(
name, "HTile final acquisition",
phased_rendering.depth_stencil_attachment.image_view != nullptr &&
phased_rendering.num_color_attachments == 0 &&
phased_rendering.depth_stencil_attachment.has_depth &&
phased_rendering.depth_stencil_attachment.depth_clear &&
phased_depth.depth_load_clear_enable &&
texture_cache.IsMeta(phased_depth.desc.info.metadata.range.address) &&
!texture_cache.IsMetaCleared(
phased_depth.desc.info.metadata.range.address,
phased_depth.desc.view_info.base_layer) &&
texture_cache.GetImage(phased_depth.image_id).IsGpuModified(),
"final depth acquisition did not publish and consume HTile "
"state at the final acquisition boundary");
RenderExecutorTestAccess::ResetBindings(executor);
constexpr uint64_t depth_only_address = base + 0x140000;
constexpr uint64_t depth_only_htile_address = base + 0x150000;
constexpr uint32_t captured_depth_only_z_info = 0x22900983u;
constexpr uint32_t captured_depth_only_stencil_info = 0x00100980u;
HW::DepthRenderTarget depth_only_target{};
depth_only_target.z_info =
HW::DepthZInfo::Decode(captured_depth_only_z_info);
depth_only_target.stencil_info =
HW::DepthStencilInfo::Decode(captured_depth_only_stencil_info);
depth_only_target.z_read_base_addr = depth_only_address;
depth_only_target.z_write_base_addr = depth_only_address;
depth_only_target.htile_data_base_addr = depth_only_htile_address;
depth_only_target.size = {63, 63, true};
registers.SetDepthRenderTarget(depth_only_target);
HW::DepthControl depth_only_control{};
depth_only_control.stencil_enable = true;
depth_only_control.z_enable = true;
depth_only_control.z_write_enable = true;
depth_only_control.zfunc = static_cast<uint8_t>(vk::CompareOp::eAlways);
registers.SetDepthControl(depth_only_control);
HW::RenderControl depth_only_render_control{};
depth_only_render_control.depth_clear_enable = true;
depth_only_render_control.stencil_clear_enable = true;
registers.SetRenderControl(depth_only_render_control);
RenderDepthInfo depth_only{};
RenderExecutorTestAccess::ResolveRenderDepthTarget(
executor, scheduler.Current(), depth_only);
Require(
name, "depth-only target with stale stencil state",
depth_only.image_id && depth_only.desc.view_info.format == vk::Format::eD32Sfloat &&
(depth_only.desc.info.metadata.kind == ImageMetadataKind::Htile) &&
depth_only.depth_test_enable && !depth_only.depth_write_enable &&
depth_only.depth_clear_enable &&
!depth_only.stencil_test_enable &&
!depth_only.stencil_clear_enable &&
depth_only.desc.info.stencil.Empty() &&
depth_only.desc.info.metadata.range.address ==
depth_only_htile_address &&
depth_only.desc.info.data.size != 0 &&
depth_only_address + depth_only.desc.info.data.size <=
base + allocation_size &&
depth_only.AttachmentWriteAspects() ==
vk::ImageAspectFlagBits::eDepth,
"inactive stencil state changed a depth-only HTile attachment");
RenderExecutorTestAccess::ResetBindings(executor);
const auto sampled_width = depth_only.desc.info.extent.width - 1u;
const auto sampled_height = depth_only.desc.info.extent.height - 1u;
const auto sampled_address = depth_only_address >> 8u;
ShaderTextureResource sampled_depth_descriptor{};
sampled_depth_descriptor.fields[0] =
static_cast<uint32_t>(sampled_address);
sampled_depth_descriptor.fields[1] =
static_cast<uint32_t>(sampled_address >> 32u) |
(static_cast<uint32_t>(Prospero::BufferFormat::k32UInt) << 20u) |
((sampled_width & 3u) << 30u);
sampled_depth_descriptor.fields[2] =
(sampled_width >> 2u) | (sampled_height << 14u);
sampled_depth_descriptor.fields[3] =
DstSel(4, 4, 4, 4) |
(static_cast<uint32_t>(Prospero::TileMode::kDepth) << 20u) |
(static_cast<uint32_t>(Prospero::ImageType::kColor2D) << 28u);
sampled_depth_descriptor.fields[5] = 0x00700000u;
ShaderRecompiler::IR::DescriptorValue sampled_depth_value{};
std::copy(std::begin(sampled_depth_descriptor.fields),
std::end(sampled_depth_descriptor.fields),
sampled_depth_value.dwords.begin());
sampled_depth_value.dword_count = 8;
ShaderRecompiler::IR::ImageResource sampled_depth_resource{};
sampled_depth_resource.resource_class =
ShaderRecompiler::IR::ImageResourceClass::Sampled;
sampled_depth_resource.numeric_class =
Prospero::TextureNumericClass::Uint;
sampled_depth_resource.dimension =
ShaderRecompiler::Decoder::ImageDimension::Dim2D;
sampled_depth_resource.read = true;
const auto sampled_depth_backing =
texture_cache.GetImage(depth_only.image_id).backing.image;
auto sampled_depth_binding = RenderExecutorTestAccess::ResolveTexture(
executor, sampled_depth_resource, sampled_depth_value);
const auto sampled_depth_view = texture_cache.FindTexture(
sampled_depth_binding.image_id, sampled_depth_binding.desc);
const auto &sampled_depth_image =
texture_cache.GetImage(sampled_depth_binding.image_id);
const bool normalized_depth_view = std::ranges::any_of(
sampled_depth_image.views, [&](const auto &cached) {
return cached.view == sampled_depth_view &&
cached.info.format == vk::Format::eD32Sfloat &&
cached.info.aspect == vk::ImageAspectFlagBits::eDepth &&
cached.info.usage == vk::ImageUsageFlags{};
});
Require(
name, "R32 uint depth-backed view",
sampled_depth_binding.image_id == depth_only.image_id &&
sampled_depth_binding.desc.info.guest_format ==
Prospero::BufferFormat::k32UInt &&
sampled_depth_binding.desc.info.pixel_format ==
vk::Format::eR32Uint &&
sampled_depth_binding.desc.view_info.format ==
vk::Format::eR32Uint &&
sampled_depth_binding.desc.view_info.aspect ==
vk::ImageAspectFlagBits::eColor &&
sampled_depth_binding.desc.view_info.usage ==
vk::ImageUsageFlagBits::eSampled &&
sampled_depth_image.info.pixel_format == vk::Format::eD32Sfloat &&
sampled_depth_image.backing.format == vk::Format::eD32Sfloat &&
sampled_depth_image.backing.image == sampled_depth_backing &&
sampled_depth_view != nullptr && normalized_depth_view,
"uint T# did not retain its requested format while the concrete "
"view followed the D32 backing");
RenderExecutorTestAccess::ResetBindings(executor);
// Attachment acquisition consumes pending HTile clears using DB_DEPTH_CLEAR.
(void)texture_cache.FindDepthTarget(depth_only.image_id, depth_only.desc);
auto bounds_target = depth_only_target;
bounds_target.z_write_base_addr = 0;
bounds_target.depth_view.depth_write_disable = true;
registers.SetDepthRenderTarget(bounds_target);
HW::DepthControl bounds_control{};
bounds_control.depth_bounds_enable = true;
registers.SetDepthControl(bounds_control);
registers.SetRenderControl({});
registers.SetDepthClearValue(0.375f);
ShaderRecompiler::IR::ResourceSnapshot bounds_snapshot;
bounds_snapshot.images.push_back(sampled_depth_value);
ShaderStageRuntime bounds_vertex{&vertex_sampled_info, &bounds_snapshot};
ShaderStageRuntime bounds_pixel{&sampled_info, &bounds_snapshot};
const auto depth_texels = depth_only.desc.info.extent.width *
depth_only.desc.info.extent.height;
const auto depth_bytes = depth_texels * sizeof(uint32_t);
auto bounds_readback = CreateHostBuffer(name, depth_bytes * 2,
vk::BufferUsageFlagBits::eTransferDst, {});
for (uint32_t pass = 0; pass < 2; ++pass) {
Require(name, "bounds prior depth contents",
texture_cache.ClearImageFromBuffer(scheduler.Current(),
depth_only.desc.info.data.address, depth_only.desc.info.data.size,
std::bit_cast<uint32_t>(0.625f)),
"failed to seed depth before the deferred HTile clear");
if (pass == 0) {
Require(name, "bounds deferred HTile clear",
texture_cache.ClearMeta(depth_only_htile_address),
"depth-only HTile was not registered");
} else {
bounds_target.z_write_base_addr = depth_only_address;
bounds_target.depth_view.depth_write_disable = false;
bounds_control.z_write_enable = true;
registers.SetDepthRenderTarget(bounds_target);
registers.SetDepthControl(bounds_control);
}
RenderDepthInfo bounds_depth{};
RenderExecutorTestAccess::ResolveRenderDepthTarget(
executor, scheduler.Current(), bounds_depth);
auto bounds_bindings = RenderExecutorTestAccess::PrepareGraphicsBindings(
executor, bounds_vertex, bounds_pixel, true);
std::array<PreparedBindings *, 2> bounds_stages{
&bounds_bindings.vertex[0], &*bounds_bindings.pixel};
vk::ImageAspectFlags bounds_feedback;
const auto bounds_rendering = RenderExecutorTestAccess::AcquireRenderTargets(
executor, scheduler.Current(), &no_color, 0, bounds_depth,
bounds_stages, &bounds_feedback);
descriptor_pipelines.push_back(RenderExecutorTestAccess::CommitBindings(
executor, scheduler.Current(), bounds_bindings.vertex[0], *bounds_bindings.pixel));
const auto &vertex_depth = bounds_bindings.vertex[0].images[0];
const auto &pixel_depth = bounds_bindings.pixel->images[0];
const auto expected_layout = pass == 0
? vk::ImageLayout::eAttachmentFeedbackLoopOptimalEXT
: vk::ImageLayout::eDepthReadOnlyOptimal;
Require(name, "deferred clear with sampled depth bounds",
bounds_depth.image_id == depth_only.image_id &&
bounds_depth.depth_bounds_test_enable &&
!bounds_depth.depth_write_enable &&
bounds_depth.depth_load_clear_enable == (pass == 0) &&
!texture_cache.IsMetaCleared(depth_only_htile_address, 0) &&
bounds_feedback == (pass == 0
? vk::ImageAspectFlagBits::eDepth
: vk::ImageAspectFlags{}) &&
bounds_rendering.depth_stencil_attachment.image_layout == expected_layout &&
bounds_rendering.depth_stencil_attachment.depth_clear == (pass == 0) &&
vertex_depth.image_id == depth_only.image_id &&
pixel_depth.image_id == depth_only.image_id &&
MakeImageInfo(vertex_depth).imageLayout == expected_layout &&
MakeImageInfo(pixel_depth).imageLayout == expected_layout,
"a deferred clear sampled by vertex and pixel stages missed depth feedback");
scheduler.BeginRendering(bounds_rendering);
scheduler.EndRendering();
RenderExecutorTestAccess::ResetBindings(executor);
const vk::BufferImageCopy copy{pass * depth_bytes, 0, 0,
{vk::ImageAspectFlagBits::eDepth, 0, 0, 1}, {}, depth_only.desc.info.extent};
texture_cache.GetImage(depth_only.image_id).Download(
std::span{&copy, 1}, bounds_readback.buffer, 0, bounds_readback.size);
}
vk::MemoryBarrier2 bounds_host_barrier{};
bounds_host_barrier.sType = vk::StructureType::eMemoryBarrier2;
bounds_host_barrier.srcStageMask = vk::PipelineStageFlagBits2::eTransfer;
bounds_host_barrier.srcAccessMask = vk::AccessFlagBits2::eTransferWrite;
bounds_host_barrier.dstStageMask = vk::PipelineStageFlagBits2::eHost;
bounds_host_barrier.dstAccessMask = vk::AccessFlagBits2::eHostRead;
vk::DependencyInfo bounds_dependency{};
bounds_dependency.sType = vk::StructureType::eDependencyInfo;
bounds_dependency.memoryBarrierCount = 1;
bounds_dependency.pMemoryBarriers = &bounds_host_barrier;
scheduler.Current().Handle().pipelineBarrier2(bounds_dependency);
scheduler.Finish();
const auto bounds_result = ReadBuffer(name, bounds_readback, depth_texels * 2);
Require(name, "deferred HTile clear occurs once",
std::ranges::all_of(std::span{bounds_result}.first(depth_texels),
[](uint32_t value) { return value == std::bit_cast<uint32_t>(0.375f); }) &&
std::ranges::all_of(std::span{bounds_result}.subspan(depth_texels),
[](uint32_t value) { return value == std::bit_cast<uint32_t>(0.625f); }),
"pending clear did not use DB_DEPTH_CLEAR or cleared the next acquisition again");
DestroyBuffer(&bounds_readback);
// Captured DB_DEPTH_VIEW selects one layer of an 87-layer HTile target.
auto high_slice_target = depth_only_target;
high_slice_target.z_read_base_addr = base + 0x800000;
high_slice_target.z_write_base_addr = high_slice_target.z_read_base_addr;
high_slice_target.htile_data_base_addr = base + 0x500000;
high_slice_target.depth_view.slice_start = 86;
high_slice_target.depth_view.slice_max = 86;
high_slice_target.size = {127, 127, true};
registers.SetDepthRenderTarget(high_slice_target);
registers.SetDepthControl(HW::DepthControl{.depth_bounds_enable = true});
registers.SetRenderControl({});
RenderDepthInfo high_slice{};
RenderExecutorTestAccess::ResolveRenderDepthTarget(
executor, scheduler.Current(), high_slice);
(void)texture_cache.FindDepthTarget(high_slice.image_id, high_slice.desc);
Require(name, "captured high HTile slice layout",
high_slice.desc.info.resources.layers == 87 &&
high_slice.desc.view_info.base_layer == 86 &&
high_slice.desc.view_info.layer_count == 1 &&
high_slice.desc.info.data.End() <= base + 0xe00000 &&
high_slice.desc.info.metadata.range.End() <= base + 0x800000 &&
texture_cache.ClearMeta(high_slice_target.htile_data_base_addr),
"single high slice was rejected or changed the backing layer layout");
auto high_slice_readback = CreateHostBuffer(
name, 4 * sizeof(uint32_t), vk::BufferUsageFlagBits::eTransferDst, {});
for (uint32_t pass = 0; pass < 2; ++pass) {
vk::ClearValue prior_depth{};
prior_depth.depthStencil.depth = 0.625f;
TextureCacheTestAccess::ClearImage(texture_cache, scheduler.Current(),
high_slice.image_id, {vk::ImageAspectFlagBits::eDepth, 0, 1, 0, 87},
prior_depth);
RenderDepthInfo acquired_high_slice{};
RenderExecutorTestAccess::ResolveRenderDepthTarget(
executor, scheduler.Current(), acquired_high_slice);
const auto rendering = RenderExecutorTestAccess::AcquireRenderTargets(
executor, scheduler.Current(), &no_color, 0, acquired_high_slice);
Require(name, "high HTile slice clear consumption",
acquired_high_slice.image_id == high_slice.image_id &&
acquired_high_slice.depth_load_clear_enable == (pass == 0) &&
rendering.depth_stencil_attachment.depth_clear == (pass == 0) &&
!texture_cache.IsMetaCleared(high_slice_target.htile_data_base_addr, 86) &&
texture_cache.IsMetaCleared(high_slice_target.htile_data_base_addr, 85),
"high slice was cleared repeatedly or consumed its neighbor's state");
scheduler.BeginRendering(rendering);
scheduler.EndRendering();
RenderExecutorTestAccess::ResetBindings(executor);
const vk::BufferImageCopy copy{pass * 2 * sizeof(uint32_t), 0, 0,
{vk::ImageAspectFlagBits::eDepth, 0, 85, 2}, {}, {1, 1, 1}};
texture_cache.GetImage(high_slice.image_id).Download(
std::span{&copy, 1}, high_slice_readback.buffer, 0, high_slice_readback.size);
}
scheduler.Current().Handle().pipelineBarrier2(bounds_dependency);
scheduler.Finish();
Require(name, "high HTile slice contents and neighbor",
ReadBuffer(name, high_slice_readback, 4) == std::vector<uint32_t>{
std::bit_cast<uint32_t>(0.625f), std::bit_cast<uint32_t>(0.375f),
std::bit_cast<uint32_t>(0.625f), std::bit_cast<uint32_t>(0.625f)},
"layer86 clear missed, repeated, or changed adjacent layer85");
DestroyBuffer(&high_slice_readback);
registers.SetDepthClearValue(0.0f);
// DB override copies run with depth and stencil tests disabled.
constexpr uint64_t copy_source_depth = base + 0x180000;
constexpr uint64_t copy_source_stencil = base + 0x1a0000;
constexpr uint64_t copy_destination_depth = base + 0x1c0000;
constexpr uint64_t copy_destination_stencil = base + 0x1e0000;
constexpr uint32_t copy_side = 8;
constexpr uint32_t copy_layer_texels = copy_side * copy_side;
constexpr uint32_t copy_depth_bytes =
copy_layer_texels * 2 * sizeof(uint32_t);
constexpr uint32_t copy_readback_stride =
copy_depth_bytes + copy_layer_texels * 2;
HW::DepthRenderTarget copy_target{};
copy_target.z_info.format = Prospero::DepthFormat::kZ32F;
copy_target.z_info.texture_compatibility =
Prospero::TextureCompatiblePlaneCompression::kEnable;
copy_target.stencil_info.format = Prospero::StencilFormat::k8UInt;
copy_target.stencil_info.texture_compatibility =
Prospero::TextureCompatibleStencil::kEnable;
copy_target.stencil_info.htile_stencil_disabled = true;
copy_target.depth_view.slice_max = 1;
copy_target.size = {copy_side - 1, copy_side - 1, true};
HW::DepthControl copy_seed_control{};
copy_seed_control.z_enable = true;
copy_seed_control.z_write_enable = true;
copy_seed_control.stencil_enable = true;
registers.SetDepthControl(copy_seed_control);
registers.SetRenderControl({});
const auto make_copy_target = [&](uint64_t depth, uint64_t stencil) {
auto target = copy_target;
target.z_read_base_addr = depth;
target.z_write_base_addr = depth;
target.stencil_read_base_addr = stencil;
target.stencil_write_base_addr = stencil;
registers.SetDepthRenderTarget(target);
RenderDepthInfo resolved{};
RenderExecutorTestAccess::ResolveRenderDepthTarget(
executor, scheduler.Current(), resolved);
(void)texture_cache.FindDepthTarget(resolved.image_id, resolved.desc);
RenderExecutorTestAccess::ResetBindings(executor);
return resolved;
};
const auto copy_source =
make_copy_target(copy_source_depth, copy_source_stencil);
const auto copy_destination =
make_copy_target(copy_destination_depth, copy_destination_stencil);
const auto clear_copy_target = [&](ImageId id, bool source) {
auto &image = texture_cache.GetImage(id);
image.Transit(vk::ImageLayout::eTransferDstOptimal,
vk::AccessFlagBits2::eTransferWrite, {},
scheduler.Current().Handle());
for (uint32_t layer = 0; layer < 2; ++layer) {
const vk::ClearDepthStencilValue clear{
source ? (layer == 0 ? 0.75f : 0.25f)
: (layer == 0 ? 0.875f : 0.625f),
source ? (layer == 0 ? 0xa3u : 0x5au)
: (layer == 0 ? 0x31u : 0xc4u)};
const vk::ImageSubresourceRange range{
vk::ImageAspectFlagBits::eDepth |
vk::ImageAspectFlagBits::eStencil,
0, 1, layer, 1};
scheduler.Current().Handle().clearDepthStencilImage(
image.backing.image, vk::ImageLayout::eTransferDstOptimal, &clear,
1, &range);
}
texture_cache.MarkGpuWritten(id);
};
struct CopyCase {
const char *label;
HW::DepthRenderOverride override;
uint32_t color_mode;
bool depth;
bool stencil;
};
constexpr std::array copy_cases{
CopyCase{"no override", {}, 0, false, false},
CopyCase{
"depth valid only", {true, false, false, false}, 0, false, false},
CopyCase{
"depth dirty only", {false, true, false, false}, 0, false, false},
CopyCase{"stencil valid only",
{false, false, true, false},
0,
false,
false},
CopyCase{"stencil dirty only",
{false, false, false, true},
0,
false,
false},
CopyCase{
"unpaired aspects", {true, false, false, true}, 0, false, false},
CopyCase{"color enabled", {true, true, true, true}, 1, false, false},
CopyCase{"depth only", {true, true, false, false}, 0, true, false},
CopyCase{"stencil only", {false, false, true, true}, 0, false, true},
CopyCase{
"depth and stencil", {true, true, true, true}, 0, true, true}};
auto copy_readback =
CreateHostBuffer(name, copy_cases.size() * copy_readback_stride,
vk::BufferUsageFlagBits::eTransferDst, {});
copy_target.z_read_base_addr = copy_source_depth;
copy_target.z_write_base_addr = copy_destination_depth;
copy_target.stencil_read_base_addr = copy_source_stencil;
copy_target.stencil_write_base_addr = copy_destination_stencil;
copy_target.depth_view.slice_start = 1;
registers.SetDepthRenderTarget(copy_target);
registers.SetDepthControl({});
RenderDepthInfo copy_disabled_depth{};
RenderExecutorTestAccess::ResolveRenderDepthTarget(
executor, scheduler.Current(), copy_disabled_depth);
Require(name, "copy without attachment tests",
!copy_disabled_depth.image_id,
"disabled depth/stencil tests acquired a regular attachment");
for (size_t index = 0; index < copy_cases.size(); ++index) {
const auto &test = copy_cases[index];
clear_copy_target(copy_source.image_id, true);
clear_copy_target(copy_destination.image_id, false);
registers.SetDepthRenderOverride(test.override);
auto color_control = registers.GetColorControl();
color_control.mode = test.color_mode;
registers.SetColorControl(color_control);
Require(name, test.label,
RenderExecutorTestAccess::DepthStencilCopy(
executor, scheduler.Current()) ==
(test.depth || test.stencil),
"DB override copy did not honor paired flags and disabled "
"color output");
RenderExecutorTestAccess::ResetBindings(executor);
const auto offset = index * copy_readback_stride;
const std::array<vk::BufferImageCopy, 2> copies{
{{offset,
0,
0,
{vk::ImageAspectFlagBits::eDepth, 0, 0, 2},
{},
copy_destination.desc.info.extent},
{offset + copy_depth_bytes,
0,
0,
{vk::ImageAspectFlagBits::eStencil, 0, 0, 2},
{},
copy_destination.desc.info.extent}}};
texture_cache.GetImage(copy_destination.image_id)
.Download(copies, copy_readback.buffer, 0, copy_readback.size);
}
scheduler.Current().Handle().pipelineBarrier2(bounds_dependency);
scheduler.Finish();
const auto copy_result = ReadBuffer(
name, copy_readback,
copy_cases.size() * copy_readback_stride / sizeof(uint32_t));
for (size_t index = 0; index < copy_cases.size(); ++index) {
const auto &test = copy_cases[index];
const auto *depth = copy_result.data() +
index * copy_readback_stride / sizeof(uint32_t);
const auto *stencil =
reinterpret_cast<const uint8_t *>(depth) + copy_depth_bytes;
for (uint32_t layer = 0; layer < 2; ++layer) {
const auto expected_depth =
std::bit_cast<uint32_t>(layer == 0 ? 0.875f
: test.depth ? 0.25f
: 0.625f);
const auto expected_stencil = layer == 0 ? 0x31u
: test.stencil ? 0x5au
: 0xc4u;
for (uint32_t texel = 0; texel < copy_layer_texels; ++texel) {
const auto position = layer * copy_layer_texels + texel;
Require(name, test.label,
depth[position] == expected_depth &&
stencil[position] == expected_stencil,
"DB override copy changed an unselected layer/aspect or "
"missed copied texels");
}
}
}
DestroyBuffer(&copy_readback);
ShaderTextureResource copy_sample_descriptor{};
copy_sample_descriptor.fields[0] =
static_cast<uint32_t>(copy_destination_depth >> 8u);
copy_sample_descriptor.fields[1] =
static_cast<uint32_t>(copy_destination_depth >> 40u) |
(static_cast<uint32_t>(Prospero::BufferFormat::k32Float) << 20u) |
(((copy_side - 1u) & 3u) << 30u);
copy_sample_descriptor.fields[2] =
((copy_side - 1u) >> 2u) | ((copy_side - 1u) << 14u);
copy_sample_descriptor.fields[3] =
DstSel(4, 4, 4, 4) |
(static_cast<uint32_t>(Prospero::TileMode::kDepth) << 20u) |
(static_cast<uint32_t>(Prospero::ImageType::kColor2DArray) << 28u);
copy_sample_descriptor.fields[4] = 1;
copy_sample_descriptor.fields[5] = 0x00700000u;
TestCase copy_sample_test;
copy_sample_test.name = "DepthOverrideR32Sampling";
copy_sample_test.has_user_data = true;
copy_sample_test.image_descriptor_swizzle =
copy_sample_descriptor.DstSelXYZW();
std::copy_n(copy_sample_descriptor.fields, 8,
copy_sample_test.user_data.begin());
copy_sample_test.user_data[50] = 2 * sizeof(uint32_t);
copy_sample_test.opcodes = {
ShaderOpcode::V_MOV_B32, ShaderOpcode::IMAGE_SAMPLE,
ShaderOpcode::BUFFER_STORE_DWORD, ShaderOpcode::S_ENDPGM};
copy_sample_test.required_spirv = {"OpImageSampleExplicitLod"};
for (uint32_t layer = 0; layer < 2; ++layer) {
AppendVMovLiteral(&copy_sample_test.code, 20,
std::bit_cast<uint32_t>(0.5f));
AppendVMovLiteral(&copy_sample_test.code, 21,
std::bit_cast<uint32_t>(0.5f));
AppendVMovLiteral(&copy_sample_test.code, 22,
std::bit_cast<uint32_t>(static_cast<float>(layer)));
copy_sample_test.code.push_back(EncodeMimg0(0x20, 1, 0, false, 5));
copy_sample_test.code.push_back(EncodeMimg1(0, 20, 0, 2));
AppendStoreVgpr(&copy_sample_test.code, 0, layer);
copy_sample_test.expected.push_back(
std::bit_cast<uint32_t>(layer == 0 ? 0.875f : 0.25f));
}
AppendEnd(&copy_sample_test.code);
const auto copy_sample_program =
CompileCase(copy_sample_test, SubgroupSize());
ShaderRecompiler::IR::DescriptorValue copy_sample_value{};
copy_sample_value.dword_count = 8;
std::copy_n(copy_sample_descriptor.fields, 8,
copy_sample_value.dwords.begin());
const auto copy_sample_binding = RenderExecutorTestAccess::ResolveTexture(
executor, copy_sample_program.program.info.images[0],
copy_sample_value);
const auto copy_sample_view = texture_cache.FindTexture(
copy_sample_binding.image_id, copy_sample_binding.desc);
Require(
name, "copied depth sampled owner",
copy_sample_binding.image_id == copy_destination.image_id &&
copy_sample_binding.desc.info.pixel_format ==
vk::Format::eR32Sfloat,
"R32 sampling did not reuse the GPU-written depth-copy destination");
auto &copy_sample_image = texture_cache.GetImage(copy_destination.image_id);
copy_sample_image.Transit(vk::ImageLayout::eShaderReadOnlyOptimal,
vk::AccessFlagBits2::eShaderRead, {},
scheduler.Current().Handle());
Image copied_depth_sample;
copied_depth_sample.view = copy_sample_view;
copied_depth_sample.layout = copy_sample_image.backing.state.layout;
scheduler.Finish();
auto copy_sample_output = CreateStorageBuffer(name, {}, 2);
const auto copy_sampler = CreateSampler(name);
Dispatch(copy_sample_test, copy_sample_program, copy_sample_output,
nullptr, &copied_depth_sample, nullptr, nullptr, copy_sampler);
Require(
name, "copied depth R32 samples",
ReadBuffer(name, copy_sample_output, 2) == copy_sample_test.expected,
"R32 post-effect sampling saw stale depth after the override copy");
m_device.destroySampler(copy_sampler);
DestroyBuffer(&copy_sample_output);
RenderExecutorTestAccess::ResetBindings(executor);
registers.SetDepthRenderOverride({});
registers.SetColorControl({});
auto shared_depth_descriptor = sampled_depth_descriptor;
shared_depth_descriptor.fields[0] =
static_cast<uint32_t>(phased_depth_address >> 8u);
shared_depth_descriptor.fields[1] =
static_cast<uint32_t>(phased_depth_address >> 40u) |
(static_cast<uint32_t>(Prospero::BufferFormat::k32UInt) << 20u);
shared_depth_descriptor.fields[2] = 0;
ShaderRecompiler::IR::DescriptorValue shared_depth_value{};
std::copy(std::begin(shared_depth_descriptor.fields),
std::end(shared_depth_descriptor.fields),
shared_depth_value.dwords.begin());
shared_depth_value.dword_count = 8;
ShaderRecompiler::IR::ResourceSnapshot shared_depth_snapshot;
shared_depth_snapshot.images.push_back(shared_depth_value);
ShaderStageRuntime shared_depth_vertex{&vertex_sampled_info,
&shared_depth_snapshot};
ShaderStageRuntime shared_depth_pixel{&sampled_info,
&shared_depth_snapshot};
for (const bool stencil_write : {true, false}) {
registers.SetDepthRenderTarget(phased_depth_target);
registers.SetRenderControl({});
registers.SetDepthControl(read_only_depth_control);
HW::StencilControl shared_stencil_control{};
shared_stencil_control.stencil_zpass =
static_cast<uint8_t>(Prospero::StencilOp::kReplaceTest);
registers.SetStencilControl(shared_stencil_control);
HW::StencilMask shared_stencil_mask{};
shared_stencil_mask.stencil_testval = 0x20;
shared_stencil_mask.stencil_mask = 0xff;
shared_stencil_mask.stencil_writemask = stencil_write ? 0xf4 : 0;
registers.SetStencilMask(shared_stencil_mask);
RenderDepthInfo shared_depth{};
RenderExecutorTestAccess::ResolveRenderDepthTarget(
executor, scheduler.Current(), shared_depth);
auto shared_bindings =
RenderExecutorTestAccess::PrepareGraphicsBindings(
executor, shared_depth_vertex, shared_depth_pixel, true);
auto& shared_image = texture_cache.GetImage(shared_depth.image_id);
auto stencil_view_info = shared_depth.desc.view_info;
stencil_view_info.aspect = vk::ImageAspectFlagBits::eStencil;
stencil_view_info.usage = vk::ImageUsageFlagBits::eSampled;
const auto stencil_view = shared_image.FindView(stencil_view_info);
auto& pixel_image = shared_bindings.pixel->images[0];
pixel_image.image_view = stencil_view;
pixel_image.desc.view_info = stencil_view_info;
std::array<PreparedBindings *, 2> shared_stages{
&shared_bindings.vertex[0], &*shared_bindings.pixel};
vk::ImageAspectFlags shared_feedback;
const auto shared_rendering =
RenderExecutorTestAccess::AcquireRenderTargets(
executor, scheduler.Current(), &no_color, 0, shared_depth,
shared_stages, &shared_feedback);
descriptor_pipelines.push_back(RenderExecutorTestAccess::CommitBindings(
executor, scheduler.Current(), shared_bindings.vertex[0],
*shared_bindings.pixel));
const auto expected_layout =
stencil_write
? vk::ImageLayout::eAttachmentFeedbackLoopOptimalEXT
: vk::ImageLayout::eDepthStencilReadOnlyOptimal;
const auto expected_access =
vk::AccessFlagBits2::eShaderRead |
vk::AccessFlagBits2::eDepthStencilAttachmentRead |
vk::AccessFlagBits2::eDepthStencilAttachmentWrite;
const auto &vertex_image = shared_bindings.vertex[0].images[0];
Require(name, "sampled depth and stencil attachment layout",
shared_depth.image_id == phased_depth.image_id &&
stencil_view != nullptr &&
shared_feedback == (stencil_write
? vk::ImageAspectFlagBits::eStencil
: vk::ImageAspectFlags{}) &&
vertex_image.image_id == shared_depth.image_id &&
pixel_image.image_id == shared_depth.image_id &&
pixel_image.image_view == stencil_view &&
MakeImageInfo(vertex_image).imageLayout == expected_layout &&
MakeImageInfo(pixel_image).imageLayout == expected_layout &&
shared_rendering.depth_stencil_attachment.image_layout ==
expected_layout &&
shared_image.backing.state.layout == expected_layout &&
shared_image.backing.state.access_mask == expected_access,
"stencil sampling and writes lost the feedback aspect, shared "
"layout, or attachment access");
scheduler.BeginRendering(shared_rendering);
scheduler.EndRendering();
RenderExecutorTestAccess::ResetBindings(executor);
Require(name, "attachment binding reset",
!shared_image.binding.is_target &&
shared_image.binding.attachment_layout ==
vk::ImageLayout::eUndefined &&
!shared_image.binding.attachment_access,
"attachment usage leaked into the next draw");
}
// PPSA03541 reuses a tile24 depth allocation through a tile27 RGB10A2 T#.
auto pooled_resource = sampled_depth_resource;
pooled_resource.numeric_class = Prospero::TextureNumericClass::Float;
ImageId pooled_color_id{};
for (const bool color : {false, true, false, true}) {
auto value = shared_depth_value;
value.dwords[1] = static_cast<uint32_t>(phased_depth_address >> 40u) |
(static_cast<uint32_t>(color ? Prospero::BufferFormat::k10_10_10_2UNorm
: Prospero::BufferFormat::k32Float) << 20u);
value.dwords[3] = color ? 0x91b00facu : 0x91800204u;
const auto binding = RenderExecutorTestAccess::ResolveTexture(
executor, pooled_resource, value);
const auto view = texture_cache.FindTexture(binding.image_id, binding.desc);
if (color && !pooled_color_id) {
pooled_color_id = binding.image_id;
}
Require(name, "pooled depth/color tile identity",
view != nullptr &&
binding.image_id == (color ? pooled_color_id : phased_depth.image_id) &&
(!color || binding.image_id != phased_depth.image_id) &&
texture_cache.GetImage(binding.image_id).backing.format ==
(color ? vk::Format::eA2B10G10R10UnormPack32
: vk::Format::eD32SfloatS8Uint) &&
binding.desc.info.tile_mode ==
(color ? Prospero::TileMode::kRenderTarget
: Prospero::TileMode::kDepth),
"a changed tile family reused depth or repeated sampling lost its backing");
RenderExecutorTestAccess::ResetBindings(executor);
}
auto video_subresource =
make_target_desc(base + 0x20000, target_mip_size, {1, 1, 1});
video_subresource.type = BindingType::VideoOut;
video_subresource.info.pixel_format = vk::Format::eR8G8B8A8Srgb;
video_subresource.info.guest_format =
Prospero::BufferFormat::k8_8_8_8Srgb;
video_subresource.view_info.format = video_subresource.info.pixel_format;
video_subresource.view_info.usage = vk::ImageUsageFlagBits::eSampled;
const auto video_subresource_id =
texture_cache.FindImage(video_subresource);
const auto video_subresource_owner =
TextureCacheTestAccess::Owner(texture_cache, video_subresource_id);
auto video_parent =
make_target_desc(base + 0x20000, target_mip_size * 2, {2, 2, 1});
video_parent.type = BindingType::VideoOut;
video_parent.info.pixel_format = vk::Format::eR8G8B8A8Srgb;
video_parent.info.guest_format = Prospero::BufferFormat::k8_8_8_8Srgb;
video_parent.info.resources.levels = 2;
video_parent.info.mip_layout[0] = {0, target_mip_size, 2, 2,
0, target_mip_size, 2};
video_parent.info.mip_layout[1] = {target_mip_size, target_mip_size, 1, 1,
target_mip_size, target_mip_size, 1};
video_parent.view_info.format = video_parent.info.pixel_format;
video_parent.view_info.usage = vk::ImageUsageFlagBits::eSampled;
const auto video_parent_id = texture_cache.FindImage(video_parent);
const auto video_parent_owner =
TextureCacheTestAccess::Owner(texture_cache, video_parent_id);
Require(name, "video-out replacement remains cache-only",
video_subresource_owner != nullptr &&
video_parent_owner != nullptr &&
video_parent_id != video_subresource_id &&
!video_subresource_owner->binding.is_bound &&
!video_subresource_owner->binding.is_target &&
!video_subresource_owner->binding.needs_rebind &&
!video_subresource_owner->usage.video_out &&
!video_parent_owner->binding.is_bound &&
!video_parent_owner->binding.is_target &&
!video_parent_owner->binding.needs_rebind &&
!video_parent_owner->usage.video_out,
"FindImage leaked caller-owned state across a video-out "
"replacement");
constexpr uint64_t array_target_address = base + 0xd0000;
auto array_target = make_target_desc(array_target_address,
target_mip_size, {128, 128, 1});
const auto array_target_id = texture_cache.FindImage(array_target);
const auto array_target_owner =
TextureCacheTestAccess::Owner(texture_cache, array_target_id);
RenderExecutorTestAccess::BindRenderTarget(executor, array_target_id);
const auto array_target_view =
texture_cache.FindRenderTarget(array_target_id, array_target);
ShaderTextureResource array_texture{};
const auto encoded_array_address = array_target_address >> 8u;
constexpr auto array_format =
static_cast<uint32_t>(Prospero::BufferFormat::k32UInt);
array_texture.fields[0] = static_cast<uint32_t>(encoded_array_address);
array_texture.fields[1] =
static_cast<uint32_t>(encoded_array_address >> 32u) |
(array_format << 20u) | (3u << 30u);
array_texture.fields[2] = (127u >> 2u) | (127u << 14u);
array_texture.fields[3] =
DstSel(4, 5, 6, 7) | (static_cast<uint32_t>(linear) << 20u) |
(static_cast<uint32_t>(Prospero::ImageType::kColor2DArray) << 28u);
array_texture.fields[4] = 1;
ShaderRecompiler::IR::CompiledShaderInfo array_program{};
array_program.stage = ShaderType::Compute;
ShaderRecompiler::IR::ImageResource array_resource{};
array_resource.resource_class =
ShaderRecompiler::IR::ImageResourceClass::Sampled;
array_resource.numeric_class = Prospero::TextureNumericClass::Uint;
array_resource.dimension =
ShaderRecompiler::Decoder::ImageDimension::Dim2DArray;
array_resource.read = true;
array_program.info.images.push_back(array_resource);
ShaderRecompiler::IR::DescriptorValue array_descriptor{};
std::copy(std::begin(array_texture.fields),
std::end(array_texture.fields),
array_descriptor.dwords.begin());
array_descriptor.dword_count = 8;
ShaderRecompiler::IR::ResourceSnapshot array_snapshot{};
array_snapshot.images.push_back(array_descriptor);
ShaderStageRuntime array_runtime{&array_program, &array_snapshot};
PreparedBindings array_binding;
executor.PrepareBindings(array_runtime, array_binding);
executor.RebindImages(array_binding);
const auto expanded_array_id = array_binding.images[0].image_id;
const auto &expanded_array = texture_cache.GetImage(expanded_array_id);
Require(
name, "2D target to array backing expansion",
array_target_view != nullptr && array_target_owner != nullptr &&
array_target_owner->binding.needs_rebind &&
expanded_array_id != array_target_id &&
!TextureCacheTestAccess::Contains(texture_cache,
array_target_id) &&
array_binding.images[0].image_view != nullptr &&
array_binding.images[0].desc.info.type ==
Prospero::ImageType::kColor2D &&
array_binding.images[0].desc.info.resources ==
ImageSubresources{1, 2} &&
array_binding.images[0].desc.view_info.type ==
vk::ImageViewType::e2DArray &&
array_binding.images[0].desc.view_info.layer_count ==
2 &&
expanded_array.backing.layers == 2 &&
expanded_array.IsGpuModified(),
"raw array view did not expand and reuse the Color2D backing");
RenderColorInfo rebound_array_target{};
rebound_array_target.desc = array_target;
rebound_array_target.image_id = array_target_id;
RenderDepthInfo no_array_depth{};
const auto acquired_array_view = array_binding.images[0].image_view;
RenderExecutorTestAccess::PrepareGraphicsBindings(
executor, std::span<PreparedBindings *const>{},
std::span{&rebound_array_target, 1u});
auto array_rendering = RenderExecutorTestAccess::AcquireRenderTargets(
executor, scheduler.Current(), &rebound_array_target, 1,
no_array_depth);
Require(name, "expanded array target rebind",
rebound_array_target.image_id == expanded_array_id &&
array_binding.images[0].image_id == expanded_array_id &&
array_binding.images[0].image_view == acquired_array_view &&
array_rendering.color_attachments[0].image_view != nullptr &&
array_rendering.num_color_attachments == 1 &&
array_rendering.width == 128 &&
array_rendering.height == 128 &&
array_rendering.num_layers == 1 &&
texture_cache.GetImage(expanded_array_id).binding.is_bound &&
texture_cache.GetImage(expanded_array_id).binding.is_target,
"the retained single-layer target did not rebind to the expanded "
"backing");
RenderExecutorTestAccess::ResetBindings(executor);
Require(name, "raw image synchronization default",
!Config::SyncRawImageBuffersEnabled(),
"raw image synchronization should be disabled by default");
for (const auto [sync_raw, formatted] :
{std::pair{false, false}, {true, false}, {false, true}, {true, true}}) {
Config::Load({.sync_raw_image_buffers = sync_raw});
const bool sync_image = sync_raw || formatted;
const uint32_t before = 0x13579bd0u | (uint32_t(sync_raw) << 1u) | uint32_t(formatted);
constexpr uint32_t guard = 0x2468ace0u;
constexpr uint32_t after = 0xa1b2c3d4u;
vk::ClearValue clear{};
clear.color.uint32[0] = before;
TextureCacheTestAccess::ClearImage(
texture_cache, scheduler.Current(), expanded_array_id,
{vk::ImageAspectFlagBits::eColor, 0, 1, 0, 2}, clear);
clear.color.uint32[0] = guard;
TextureCacheTestAccess::ClearImage(
texture_cache, scheduler.Current(), expanded_array_id,
{vk::ImageAspectFlagBits::eColor, 0, 1, 1, 1}, clear);
const auto buffer_program = make_buffer_program(
ShaderType::Vertex, {.read = true, .formatted = formatted});
constexpr uint64_t buffer_size = 2 * target_mip_size;
static_assert(buffer_size > BufferCache::CACHING_PAGESIZE);
ShaderBufferResource descriptor{};
descriptor.UpdateAddress48(array_target_address);
descriptor.fields[1] |= 4u << 16u;
descriptor.fields[2] = buffer_size / sizeof(uint32_t);
descriptor.fields[3] = 0x00014004u;
ShaderRecompiler::IR::ResourceSnapshot buffer_snapshot;
ShaderStageRuntime buffer_runtime{&buffer_program, &buffer_snapshot};
auto &value = buffer_snapshot.buffers.emplace_back();
std::memcpy(value.dwords.data(), descriptor.fields, sizeof(descriptor.fields));
value.dword_count = 4;
PreparedBindings buffer_bindings;
executor.PrepareBindings(buffer_runtime, buffer_bindings);
std::array stages{&buffer_bindings};
RenderColorInfo target{};
target.desc = array_target;
target.image_id = array_target_id;
RenderExecutorTestAccess::PrepareGraphicsBindings(
executor, stages, std::span{&target, 1u});
const auto &buffer = buffer_bindings.buffers[0];
Require(name, "rediscovered target buffer read",
target.image_id == expanded_array_id &&
buffer.range == buffer_size &&
buffer_bindings.images.empty() &&
texture_cache.GetImage(target.image_id).backing.state.layout ==
(sync_image ? vk::ImageLayout::eTransferSrcOptimal
: vk::ImageLayout::eTransferDstOptimal),
"image-to-buffer synchronization ignored its setting or formatted reads");
auto readback = CreateHostBuffer(name, 2 * sizeof(uint32_t),
vk::BufferUsageFlagBits::eTransferDst, {0, 0});
const std::array copies{
vk::BufferCopy{buffer.offset, 0, sizeof(uint32_t)},
vk::BufferCopy{buffer.offset + target_mip_size, sizeof(uint32_t),
sizeof(uint32_t)}};
scheduler.Current().Handle().copyBuffer(buffer.buffer, readback.buffer,
copies.size(), copies.data());
vk::BufferMemoryBarrier readback_barrier{};
readback_barrier.srcAccessMask = vk::AccessFlagBits::eTransferWrite;
readback_barrier.dstAccessMask = vk::AccessFlagBits::eHostRead;
readback_barrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
readback_barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
readback_barrier.buffer = readback.buffer;
readback_barrier.size = readback.size;
scheduler.Current().Handle().pipelineBarrier(
vk::PipelineStageFlagBits::eTransfer, vk::PipelineStageFlagBits::eHost,
{}, 0, nullptr, 1, &readback_barrier, 0, nullptr);
const auto rendering = RenderExecutorTestAccess::AcquireRenderTargets(
executor, scheduler.Current(), &target, 1, no_array_depth);
Require(name, "attachment layout after buffer read",
rendering.color_attachments[0].image_layout ==
vk::ImageLayout::eColorAttachmentOptimal &&
texture_cache.GetImage(target.image_id).backing.state.layout ==
vk::ImageLayout::eColorAttachmentOptimal,
"the buffer copy left the render attachment in its transfer layout");
vk::ClearAttachment clear_attachment{};
clear_attachment.aspectMask = vk::ImageAspectFlagBits::eColor;
clear_attachment.colorAttachment = 0;
clear_attachment.clearValue.color.uint32[0] = after;
const vk::ClearRect clear_rect{{{0, 0}, {128, 128}}, 0, 1};
scheduler.BeginRendering(rendering);
scheduler.Current().Handle().clearAttachments(1, &clear_attachment, 1, &clear_rect);
scheduler.EndRendering();
scheduler.Finish();
Require(name, "buffer alias observes prior GPU contents",
ReadBuffer(name, readback, 2) ==
(sync_image ? std::vector<u32>{before, guard} : std::vector<u32>{0, 0}),
"buffer contents did not match the selected image synchronization mode");
DestroyBuffer(&readback);
Require(name, "rendering after buffer acquisition",
ReadCachedTexel(name, context, target.image_id) == std::vector<u32>{after} &&
ReadCachedTexel(name, context, target.image_id, {}, {1, 1, 1}, 1) ==
std::vector<u32>{guard},
"attachment rendering lost its new color or changed the untouched array layer");
RenderExecutorTestAccess::ResetBindings(executor);
}
Config::Load({});
auto colliding_msaa_texture = array_texture;
colliding_msaa_texture.fields[3] =
DstSel(4, 5, 6, 7) | (1u << 16u) |
(static_cast<uint32_t>(Prospero::TileMode::kRenderTarget) << 20u) |
(static_cast<uint32_t>(Prospero::ImageType::kColor2DMsaa) << 28u);
colliding_msaa_texture.fields[4] = 0;
colliding_msaa_texture.fields[5] = (7u << 20u) | (1u << 4u);
ShaderRecompiler::IR::DescriptorValue colliding_msaa_descriptor{};
std::copy(std::begin(colliding_msaa_texture.fields),
std::end(colliding_msaa_texture.fields),
colliding_msaa_descriptor.dwords.begin());
colliding_msaa_descriptor.dword_count = 8;
ShaderRecompiler::IR::CompiledShaderInfo colliding_msaa_program{};
colliding_msaa_program.stage = ShaderType::Compute;
auto colliding_msaa_resource = array_resource;
colliding_msaa_resource.dimension =
ShaderRecompiler::Decoder::ImageDimension::Dim2DMsaa;
colliding_msaa_program.info.images.push_back(colliding_msaa_resource);
ShaderRecompiler::IR::ResourceSnapshot colliding_msaa_snapshot{};
colliding_msaa_snapshot.images.push_back(colliding_msaa_descriptor);
ShaderStageRuntime colliding_msaa_runtime{
&colliding_msaa_program, &colliding_msaa_snapshot};
PreparedBindings colliding_msaa_binding;
executor.PrepareBindings(colliding_msaa_runtime, colliding_msaa_binding);
executor.RebindImages(colliding_msaa_binding);
const auto &resolved_colliding_msaa =
colliding_msaa_binding.images[0];
Require(
name, "equal-footprint sample-count identity",
resolved_colliding_msaa.image_id != expanded_array_id &&
resolved_colliding_msaa.image_view != nullptr &&
resolved_colliding_msaa.desc.info.type ==
Prospero::ImageType::kColor2D &&
resolved_colliding_msaa.desc.info.resources ==
ImageSubresources{1, 1} &&
resolved_colliding_msaa.desc.info.samples == 2 &&
resolved_colliding_msaa.desc.info.data.size == 0x20000 &&
texture_cache.GetImage(resolved_colliding_msaa.image_id)
.backing.samples == 2 &&
texture_cache.GetImage(expanded_array_id).backing.samples == 1 &&
texture_cache.GetImage(expanded_array_id).backing.layers == 2,
"equal-size 1x array and 2x image shared an incompatible backing");
RenderExecutorTestAccess::ResetBindings(executor);
constexpr uint64_t msaa_target_address = base + 0x100000;
constexpr uint32_t msaa_samples = 4;
auto msaa_target =
make_target_desc(msaa_target_address, 0x40000, {128, 128, 1});
msaa_target.info.samples = msaa_samples;
msaa_target.info.tile_mode = Prospero::TileMode::kRenderTarget;
msaa_target.info.mip_layout[0].size = msaa_target.info.data.size;
const auto msaa_target_id = texture_cache.FindImage(msaa_target);
RenderExecutorTestAccess::BindRenderTarget(executor, msaa_target_id);
const auto msaa_target_view =
texture_cache.FindRenderTarget(msaa_target_id, msaa_target);
auto msaa_texture = array_texture;
const auto encoded_msaa_address = msaa_target_address >> 8u;
msaa_texture.fields[0] = static_cast<uint32_t>(encoded_msaa_address);
msaa_texture.fields[1] =
static_cast<uint32_t>(encoded_msaa_address >> 32u) |
(array_format << 20u) | (3u << 30u);
msaa_texture.fields[3] =
DstSel(4, 5, 6, 7) | (2u << 16u) |
(static_cast<uint32_t>(Prospero::TileMode::kRenderTarget) << 20u) |
(static_cast<uint32_t>(Prospero::ImageType::kColor2DMsaa) << 28u);
msaa_texture.fields[4] = 0;
msaa_texture.fields[5] = (7u << 20u) | (2u << 4u);
ShaderRecompiler::IR::DescriptorValue msaa_descriptor{};
std::copy(std::begin(msaa_texture.fields), std::end(msaa_texture.fields),
msaa_descriptor.dwords.begin());
msaa_descriptor.dword_count = 8;
ShaderRecompiler::IR::CompiledShaderInfo msaa_program{};
msaa_program.stage = ShaderType::Compute;
auto msaa_resource = array_resource;
msaa_resource.dimension =
ShaderRecompiler::Decoder::ImageDimension::Dim2DMsaa;
msaa_program.info.images.push_back(msaa_resource);
ShaderRecompiler::IR::ResourceSnapshot msaa_snapshot{};
msaa_snapshot.images.push_back(msaa_descriptor);
ShaderStageRuntime msaa_runtime{&msaa_program, &msaa_snapshot};
PreparedBindings msaa_binding;
executor.PrepareBindings(msaa_runtime, msaa_binding);
executor.RebindImages(msaa_binding);
const auto &resolved_msaa = msaa_binding.images[0];
Require(
name, "MSAA descriptor backing reuse",
msaa_target_view != nullptr &&
resolved_msaa.image_id == msaa_target_id &&
resolved_msaa.image_view != nullptr &&
resolved_msaa.desc.info.type == Prospero::ImageType::kColor2D &&
resolved_msaa.desc.info.resources == ImageSubresources{1, 1} &&
resolved_msaa.desc.info.samples == msaa_samples &&
resolved_msaa.desc.info.data.size == 0x40000 &&
resolved_msaa.desc.view_info.level_count == 1 &&
texture_cache.GetImage(msaa_target_id).backing.samples ==
msaa_samples,
"MSAA view fields did not resolve to the matching multisample "
"backing");
RenderExecutorTestAccess::ResetBindings(executor);
auto msaa_array_texture = msaa_texture;
msaa_array_texture.fields[3] =
(msaa_array_texture.fields[3] & ~(0xfu << 28u)) |
(static_cast<uint32_t>(Prospero::ImageType::kColor2DMsaaArray)
<< 28u);
msaa_array_texture.fields[4] = 1;
ShaderRecompiler::IR::DescriptorValue msaa_array_descriptor{};
std::copy(std::begin(msaa_array_texture.fields),
std::end(msaa_array_texture.fields),
msaa_array_descriptor.dwords.begin());
msaa_array_descriptor.dword_count = 8;
ShaderRecompiler::IR::CompiledShaderInfo msaa_array_program{};
msaa_array_program.stage = ShaderType::Compute;
auto msaa_array_resource = array_resource;
msaa_array_resource.dimension =
ShaderRecompiler::Decoder::ImageDimension::Dim2DMsaaArray;
msaa_array_program.info.images.push_back(msaa_array_resource);
ShaderRecompiler::IR::ResourceSnapshot msaa_array_snapshot{};
msaa_array_snapshot.images.push_back(msaa_array_descriptor);
ShaderStageRuntime msaa_array_runtime{&msaa_array_program, &msaa_array_snapshot};
PreparedBindings msaa_array_binding;
executor.PrepareBindings(msaa_array_runtime, msaa_array_binding);
executor.RebindImages(msaa_array_binding);
const auto &resolved_msaa_array = msaa_array_binding.images[0];
Require(name, "MSAA array backing expansion",
resolved_msaa_array.image_id != msaa_target_id &&
resolved_msaa_array.image_view != nullptr &&
resolved_msaa_array.desc.info.type ==
Prospero::ImageType::kColor2D &&
resolved_msaa_array.desc.info.resources ==
ImageSubresources{1, 2} &&
resolved_msaa_array.desc.info.samples == msaa_samples &&
resolved_msaa_array.desc.info.data.size == 0x80000 &&
resolved_msaa_array.desc.view_info.type ==
vk::ImageViewType::e2DArray &&
resolved_msaa_array.desc.view_info.layer_count == 2 &&
texture_cache.GetImage(resolved_msaa_array.image_id)
.backing.layers == 2 &&
texture_cache.GetImage(resolved_msaa_array.image_id)
.backing.samples == msaa_samples,
"MSAA array view did not expand the matching Color2D backing");
RenderExecutorTestAccess::ResetBindings(executor);
ImageDesc depth{};
depth.type = BindingType::DepthTarget;
depth.info.data = {depth_address, 0x10000};
depth.info.stencil = {stencil_address, stencil_layout.size};
depth.info.pixel_format = vk::Format::eD32SfloatS8Uint;
depth.info.guest_format = Prospero::BufferFormat::k32Float;
depth.info.type = Prospero::ImageType::kColor2D;
depth.info.extent = {1, 1, 1};
depth.info.resources = {1, 1};
depth.info.pitch = 1;
depth.info.bytes_per_block = 4;
depth.info.samples = 1;
depth.info.tile_mode = linear;
depth.info.UpdateSize();
depth.view_info.format = depth.info.pixel_format;
depth.view_info.type = vk::ImageViewType::e2D;
depth.view_info.aspect = vk::ImageAspectFlagBits::eDepth;
depth.view_info.usage = vk::ImageUsageFlagBits::eDepthStencilAttachment;
const auto depth_id = texture_cache.FindImage(depth);
Require(name, "cache-only depth discovery",
!texture_cache.FindImageFromRange(stencil_address,
stencil_layout.size, false) &&
!texture_cache.GetImage(depth_id).binding.is_bound &&
!texture_cache.GetImage(depth_id).binding.is_target,
"FindImage created depth acquisition state or a stencil proxy");
RenderExecutorTestAccess::BindRenderTarget(executor, depth_id);
Require(name, "depth prefetch purity",
texture_cache.GetImage(depth_id).binding.is_target &&
!texture_cache.FindImageFromRange(stencil_address,
stencil_layout.size, false),
"depth prefetch performed final target acquisition");
auto target_parent = make_target_desc(base, 1536, {4, 4, 1});
target_parent.type = BindingType::RenderTarget;
target_parent.info.resources.levels = 2;
// PS5 linear rows align to 256 bytes; smaller mips precede larger ones.
target_parent.info.pitch = 64;
target_parent.info.UpdateSize();
target_parent.view_info.usage = vk::ImageUsageFlagBits::eColorAttachment;
auto target_base_subresource =
make_target_desc(base + 512, 1024, {4, 4, 1});
target_base_subresource.info.pitch = 64;
target_base_subresource.info.UpdateSize();
const auto target_base_subresource_id =
texture_cache.FindImage(target_base_subresource);
auto target_subresource = make_target_desc(base, 512, {2, 2, 1});
target_subresource.info.pitch = 64;
target_subresource.info.UpdateSize();
const auto target_subresource_id =
texture_cache.FindImage(target_subresource);
const auto *target_subresource_owner =
TextureCacheTestAccess::Owner(texture_cache, target_subresource_id);
const auto &discovered_target =
texture_cache.GetImage(target_subresource_id);
Require(name, "cache-only target discovery",
!discovered_target.binding.is_bound &&
!discovered_target.binding.is_target &&
!discovered_target.binding.needs_rebind &&
!discovered_target.usage.render_target &&
!TextureCacheTestAccess::PendingDownload(
texture_cache, target_subresource_id),
"FindImage claimed RenderExecutor-owned render-target state");
vk::ClearValue target_clear{};
target_clear.color.uint32[0] = 0x12345678;
TextureCacheTestAccess::ClearImage(
texture_cache, scheduler.Current(), target_base_subresource_id,
{vk::ImageAspectFlagBits::eColor, 0, 1, 0, 1}, target_clear);
target_clear.color.uint32[0] = 0x89abcdef;
TextureCacheTestAccess::ClearImage(
texture_cache, scheduler.Current(), target_subresource_id,
{vk::ImageAspectFlagBits::eColor, 0, 1, 0, 1}, target_clear);
RenderExecutorTestAccess::BindRenderTarget(executor,
target_subresource_id);
Require(name, "target prefetch purity",
texture_cache.GetImage(target_subresource_id).binding.is_target &&
!texture_cache.GetImage(target_subresource_id)
.usage.render_target &&
!TextureCacheTestAccess::PendingDownload(
texture_cache, target_subresource_id),
"target prefetch performed final target acquisition");
const auto target_parent_id = texture_cache.FindImage(target_parent);
Require(name, "active target overlap",
target_parent_id &&
target_parent_id != target_base_subresource_id &&
target_parent_id != target_subresource_id &&
target_subresource_owner != nullptr &&
target_subresource_owner->binding.needs_rebind &&
texture_cache.GetImage(target_parent_id).binding.is_target &&
!TextureCacheTestAccess::PendingDownload(texture_cache,
target_parent_id),
"target overlap did not transfer target state to the merged "
"owner");
auto target_readback = CreateHostBuffer(
name, 2 * sizeof(u32), vk::BufferUsageFlagBits::eTransferDst,
{0xaaaaaaaa, 0xaaaaaaaa});
const std::array target_probes{
vk::BufferImageCopy{0,
0,
0,
{vk::ImageAspectFlagBits::eColor, 0, 0, 1},
{},
{1, 1, 1}},
vk::BufferImageCopy{sizeof(u32),
0,
0,
{vk::ImageAspectFlagBits::eColor, 1, 0, 1},
{},
{1, 1, 1}}};
texture_cache.GetImage(target_parent_id)
.Download(target_probes, target_readback.buffer, 0,
target_readback.size);
const vk::MemoryBarrier2 target_host_barrier{
.srcStageMask = vk::PipelineStageFlagBits2::eCopy,
.srcAccessMask = vk::AccessFlagBits2::eTransferWrite,
.dstStageMask = vk::PipelineStageFlagBits2::eHost,
.dstAccessMask = vk::AccessFlagBits2::eHostRead};
vk::DependencyInfo target_dependency{};
target_dependency.memoryBarrierCount = 1;
target_dependency.pMemoryBarriers = &target_host_barrier;
scheduler.Current().Handle().pipelineBarrier2(target_dependency);
RenderColorInfo rebound_color{};
rebound_color.desc = target_subresource;
rebound_color.image_id = target_subresource_id;
RenderDepthInfo rebound_depth{};
rebound_depth.desc = depth;
rebound_depth.image_id = depth_id;
scheduler.Finish();
Require(name, "promoted target mip contents",
ReadBuffer(name, target_readback, 2) ==
std::vector<u32>{0x12345678, 0x89abcdef},
"promotion discarded GPU contents from a child mip");
DestroyBuffer(&target_readback);
Require(name, "deferred target slot erasure",
TextureCacheTestAccess::Owner(texture_cache,
target_subresource_id) == nullptr,
"the displaced target slot was not erased after GPU completion");
RenderExecutorTestAccess::PrepareGraphicsBindings(
executor, std::span<PreparedBindings *const>{}, std::span{&rebound_color, 1u});
auto rebound_rendering = RenderExecutorTestAccess::AcquireRenderTargets(
executor, scheduler.Current(), &rebound_color, 1, rebound_depth);
Require(name, "guest target extent after host mip remap",
rebound_color.desc.view_info.base_level == 1 &&
rebound_color.guest_mip_level == 0 &&
rebound_color.Extent() == vk::Extent2D{2, 2},
"remapping a guest target into its parent mip shrank the guest "
"render area twice");
const auto stencil_proxy_id = texture_cache.FindImageFromRange(
stencil_address, stencil_layout.size, false);
Require(
name, "production target rebind",
rebound_color.image_id == target_parent_id &&
rebound_rendering.color_attachments[0].image_view != nullptr &&
rebound_rendering.depth_stencil_attachment.image_view != nullptr &&
rebound_rendering.num_color_attachments == 1 &&
rebound_rendering.width == 1 && rebound_rendering.height == 1 &&
texture_cache.GetImage(target_parent_id).binding.is_target &&
texture_cache.GetImage(target_parent_id).usage.render_target &&
texture_cache.GetImage(target_parent_id).IsGpuModified() &&
texture_cache.GetImage(depth_id).IsGpuModified() &&
TextureCacheTestAccess::PendingDownload(texture_cache,
target_parent_id) &&
stencil_proxy_id &&
texture_cache.GetImage(stencil_proxy_id).depth_id == depth_id,
"the final target pass did not acquire color before depth and "
"build the dynamic render state");
scheduler.BeginRendering(rebound_rendering);
scheduler.EndRendering();
RenderExecutorTestAccess::ResetBindings(executor);
Require(name, "draw-scoped target reset",
TextureCacheTestAccess::Owner(texture_cache,
target_subresource_id) == nullptr &&
!texture_cache.GetImage(target_parent_id).binding.is_target &&
!texture_cache.GetImage(depth_id).binding.is_target,
"RenderExecutor retained target binding state after reset");
constexpr uint64_t ordered_color_address = base + 0x80000;
auto ordered_color_desc =
make_target_desc(ordered_color_address, target_mip_size, {1, 1, 1});
const auto stale_ordered_color =
texture_cache.FindImage(ordered_color_desc);
RenderExecutorTestAccess::BindRenderTarget(executor, stale_ordered_color);
resources.UnmapMemory(ordered_color_address, target_mip_size);
resources.MapMemory(ordered_color_address, target_mip_size);
auto ordered_depth_desc = depth;
ordered_depth_desc.info.stencil = {ordered_color_address,
target_mip_size};
const auto ordered_depth_id = texture_cache.FindImage(ordered_depth_desc);
RenderExecutorTestAccess::BindRenderTarget(executor, ordered_depth_id);
RenderColorInfo ordered_color{};
ordered_color.desc = ordered_color_desc;
ordered_color.image_id = stale_ordered_color;
RenderDepthInfo ordered_depth{};
ordered_depth.desc = ordered_depth_desc;
ordered_depth.image_id = ordered_depth_id;
RenderExecutorTestAccess::PrepareGraphicsBindings(
executor, std::span<PreparedBindings *const>{}, std::span{&ordered_color, 1u});
auto ordered_rendering = RenderExecutorTestAccess::AcquireRenderTargets(
executor, scheduler.Current(), &ordered_color, 1, ordered_depth);
Require(name, "color-before-depth acquisition order",
ordered_color.image_id != stale_ordered_color &&
ordered_rendering.color_attachments[0].image_view != nullptr &&
ordered_rendering.depth_stencil_attachment.image_view != nullptr &&
texture_cache.GetImage(ordered_color.image_id).depth_id ==
ordered_depth.image_id,
"depth acquisition ran before the stale color target was "
"recreated and associated");
RenderExecutorTestAccess::ResetBindings(executor);
TextureCacheTestAccess::SetLinearReadback(texture_cache, false);
ShaderTextureResource stencil{};
const uint64_t encoded_address = stencil_address >> 8u;
stencil.fields[0] = static_cast<uint32_t>(encoded_address);
stencil.fields[1] = static_cast<uint32_t>(encoded_address >> 32u) |
(static_cast<uint32_t>(stencil_format) << 20u);
stencil.fields[3] =
DstSel(4, 5, 6, 7) | (static_cast<uint32_t>(linear) << 20u) |
(static_cast<uint32_t>(Prospero::ImageType::kColor2D) << 28u);
ShaderRecompiler::IR::CompiledShaderInfo program{};
program.stage = ShaderType::Compute;
ShaderRecompiler::IR::ImageResource stencil_resource{};
stencil_resource.resource_class =
ShaderRecompiler::IR::ImageResourceClass::Sampled;
stencil_resource.numeric_class = Prospero::TextureNumericClass::Uint;
stencil_resource.dimension =
ShaderRecompiler::Decoder::ImageDimension::Dim2D;
stencil_resource.read = true;
program.info.images.push_back(stencil_resource);
ShaderRecompiler::IR::ResourceSnapshot snapshot{};
ShaderRecompiler::IR::DescriptorValue descriptor{};
std::copy(std::begin(stencil.fields), std::end(stencil.fields),
descriptor.dwords.begin());
descriptor.dword_count = 8;
snapshot.images.push_back(descriptor);
ShaderStageRuntime runtime{&program, &snapshot};
PreparedBindings prepared;
context.GetRenderExecutor().PrepareBindings(runtime, prepared);
const auto sampled_stencil_id = prepared.images[0].image_id;
Require(name, "first stencil discovery",
prepared.images.size() == 1 &&
sampled_stencil_id == depth_id &&
prepared.images[0].image_view == nullptr &&
texture_cache.GetImage(sampled_stencil_id).binding.is_bound &&
!texture_cache.GetImage(stencil_proxy_id).binding.is_bound &&
!texture_cache.GetImage(stencil_proxy_id).binding.is_target,
"the first sampled-stencil lookup did not use the canonical "
"depth owner without acquiring the stencil proxy");
context.GetRenderExecutor().RebindImages(prepared);
Require(name, "first stencil acquisition",
prepared.images[0].image_id == sampled_stencil_id &&
prepared.images[0].image_view != nullptr,
"the first sampled-stencil image was not finally acquired");
RenderExecutorTestAccess::ResetBindings(executor);
RenderDepthInfo reassociated_depth = rebound_depth;
RenderColorInfo no_reassociated_color{};
(void)RenderExecutorTestAccess::AcquireRenderTargets(
executor, scheduler.Current(), &no_reassociated_color, 0,
reassociated_depth);
Require(name, "existing stencil association selection",
texture_cache.GetImage(stencil_proxy_id).depth_id ==
reassociated_depth.image_id,
"final depth acquisition did not associate the existing image "
"at the stencil guest address");
RenderExecutorTestAccess::ResetBindings(executor);
PreparedBindings redirected;
context.GetRenderExecutor().PrepareBindings(runtime, redirected);
Require(name, "redirected owner discovery",
redirected.images.size() == 1 &&
redirected.images[0].image_id == depth_id &&
redirected.images[0].image_view == nullptr &&
texture_cache.GetImage(depth_id).binding.is_bound &&
!texture_cache.GetImage(stencil_proxy_id).binding.is_bound,
"the established stencil association did not redirect the next "
"discovery to the depth owner");
context.GetRenderExecutor().RebindImages(redirected);
Require(name, "second-pass stencil acquisition",
redirected.images[0].image_id == depth_id &&
redirected.images[0].image_view != nullptr,
"RebindImages did not acquire the associated depth owner");
RenderExecutorTestAccess::ResetBindings(executor);
// Execute the captured IMAGE_STORE clear through production fill recognition.
const std::array<uint32_t, 12> stencil_shader{
0xd7460000u, 0x0401060cu, 0xf4201a84u, 0xfa000000u,
0xbf8cc07fu, 0x7e06026au, 0xd7460001u, 0x0405060du,
0x7e04020eu, 0xf0200128u, 0x00000300u, 0xbf810000u};
uint32_t stencil_byte = 0x35;
const auto value_address = reinterpret_cast<uint64_t>(&stencil_byte);
auto storage_stencil = stencil;
storage_stencil.fields[3] = (storage_stencil.fields[3] & 0x0fffffffu) |
(static_cast<uint32_t>(Prospero::ImageType::kColor2DArray) << 28u);
storage_stencil.fields[5] = 0x00700000u;
std::array<uint32_t, 12> stencil_userdata{};
std::copy_n(storage_stencil.fields, 8, stencil_userdata.begin());
stencil_userdata[8] = static_cast<uint32_t>(value_address);
stencil_userdata[9] = static_cast<uint32_t>(value_address >> 32) | (4u << 16);
stencil_userdata[10] = 1;
stencil_userdata[11] = 0x14204u;
ShaderComputeInputInfo stencil_compute{};
stencil_compute.threads_num[0] = stencil_compute.threads_num[1] = 8;
stencil_compute.threads_num[2] = 1;
stencil_compute.workgroup_register = 12;
stencil_compute.thread_ids_num = 2;
std::fill_n(stencil_compute.group_id, 3, true);
ShaderRecompiler::CompileOptions stencil_options;
stencil_options.stage = ShaderType::Compute;
stencil_options.user_data = stencil_userdata;
stencil_options.input_info.compute = &stencil_compute;
auto stencil_translated = ShaderRecompiler::TranslateProgram(stencil_shader, stencil_options);
auto stencil_plan = ShaderRecompiler::IR::ExtractResourcePlan(stencil_translated.program);
ShaderRecompiler::IR::ResourceSpecialization stencil_specialization;
ShaderRecompiler::IR::ResourceSnapshot stencil_snapshot;
stencil_compute.stage.resources = &stencil_snapshot;
Require(name, "stencil fill materialization",
ShaderRecompiler::IR::MaterializeResources(stencil_plan,
{.user_data = stencil_userdata, .userdata = &stencil_byte,
.read_specialization_memory = +[](void *data, uint64_t address, std::span<uint32_t> words) {
if (address != reinterpret_cast<uint64_t>(data) || words.size() != 1u) return false;
words[0] = *static_cast<uint32_t *>(data);
return true;
}}, stencil_snapshot, stencil_specialization),
"captured stencil shader could not resolve its clear byte");
ShaderRecompiler::IR::ApplyResourceSpecialization(stencil_translated.program,
stencil_specialization);
ShaderRecompiler::IR::CompiledShaderInfo stencil_program{};
stencil_program.stage = ShaderType::Compute;
stencil_program.info = std::move(stencil_translated.program.info);
stencil_compute.stage.program = &stencil_program;
vk::ClearValue stencil_initial{};
stencil_initial.depthStencil = vk::ClearDepthStencilValue{0.625f, 0x91};
TextureCacheTestAccess::ClearImage(texture_cache, scheduler.Current(), depth_id,
{vk::ImageAspectFlagBits::eDepth | vk::ImageAspectFlagBits::eStencil, 0, 1, 0, 1},
stencil_initial);
Require(name, "stencil fill coverage",
!RenderExecutorTestAccess::TryConsumeComputeImageClear(executor, stencil_compute,
scheduler.Current(), 0, 1, 1, 0x41u) &&
!RenderExecutorTestAccess::TryConsumeComputeImageClear(executor, stencil_compute,
scheduler.Current(), 2, 1, 1, 0x41u),
"incomplete or excessive stencil dispatch was consumed");
auto &stencil_source = stencil_snapshot.buffers[0];
const auto clean_source = stencil_source;
stencil_source.dwords[0] = static_cast<uint32_t>(stencil_address);
stencil_source.dwords[1] = static_cast<uint32_t>(stencil_address >> 32) | (4u << 16);
Require(name, "stencil fill source alias",
!RenderExecutorTestAccess::TryConsumeComputeImageClear(executor, stencil_compute,
scheduler.Current(), 1, 1, 1, 0x41u),
"an aliased scalar load was consumed as a uniform image clear");
stencil_source = clean_source;
Require(name, "stencil IMAGE_STORE clear",
RenderExecutorTestAccess::TryConsumeComputeImageClear(executor, stencil_compute,
scheduler.Current(), 1, 1, 1, 0x41u),
"captured clear attempted an unsupported depth/stencil storage binding");
auto stencil_readback = CreateHostBuffer(name, 8,
vk::BufferUsageFlagBits::eTransferDst, {});
const std::array<vk::BufferImageCopy, 2> stencil_copies{{
{0, 0, 0, {vk::ImageAspectFlagBits::eDepth, 0, 0, 1}, {}, {1, 1, 1}},
{4, 0, 0, {vk::ImageAspectFlagBits::eStencil, 0, 0, 1}, {}, {1, 1, 1}}}};
texture_cache.GetImage(depth_id).Download(stencil_copies, stencil_readback.buffer, 0, 8);
vk::MemoryBarrier2 stencil_host_barrier{};
stencil_host_barrier.sType = vk::StructureType::eMemoryBarrier2;
stencil_host_barrier.srcStageMask = vk::PipelineStageFlagBits2::eTransfer;
stencil_host_barrier.srcAccessMask = vk::AccessFlagBits2::eTransferWrite;
stencil_host_barrier.dstStageMask = vk::PipelineStageFlagBits2::eHost;
stencil_host_barrier.dstAccessMask = vk::AccessFlagBits2::eHostRead;
vk::DependencyInfo stencil_dependency{};
stencil_dependency.sType = vk::StructureType::eDependencyInfo;
stencil_dependency.memoryBarrierCount = 1;
stencil_dependency.pMemoryBarriers = &stencil_host_barrier;
scheduler.Current().Handle().pipelineBarrier2(stencil_dependency);
scheduler.Finish();
const auto stencil_result = ReadBuffer(name, stencil_readback, 2);
Require(name, "stencil clear retains depth",
stencil_result[0] == std::bit_cast<uint32_t>(0.625f) &&
(stencil_result[1] & 0xffu) == 0x35 &&
!texture_cache.GetImage(depth_id).usage.storage,
"stencil clear altered depth, lost its scalar value, or used a storage view");
DestroyBuffer(&stencil_readback);
// Buffer writes to the independent tiled stencil plane must reach the
// combined native attachment, including values that are not a clear.
{
LibKernel::Memory::InstallGpuResources(&resources);
constexpr uint64_t plane_size = 0x10000;
constexpr uint64_t plane_address = base + 0x910000;
constexpr uint32_t texels = 6;
auto plane_depth = depth;
plane_depth.info.data = {base + 0x900000, plane_size};
plane_depth.info.stencil = {plane_address, plane_size};
plane_depth.info.extent = {3, 2, 1};
plane_depth.info.pitch = TileGetDepthPitch(3, sizeof(uint32_t));
plane_depth.info.tile_mode = Prospero::TileMode::kDepth;
plane_depth.info.UpdateSize();
const auto plane_depth_id = texture_cache.FindImage(plane_depth);
auto sampled_plane = plane_depth;
sampled_plane.type = BindingType::Texture;
sampled_plane.info.data = plane_depth.info.stencil;
sampled_plane.info.stencil = {};
sampled_plane.info.metadata = {};
sampled_plane.info.pixel_format = vk::Format::eR8Uint;
sampled_plane.info.guest_format = Prospero::BufferFormat::k8UInt;
sampled_plane.info.bytes_per_block = 1;
sampled_plane.info.pitch = TileGetDepthPitch(3, 1);
sampled_plane.info.UpdateSize();
sampled_plane.view_info.format = vk::Format::eR8Uint;
sampled_plane.view_info.aspect = vk::ImageAspectFlagBits::eColor;
sampled_plane.view_info.usage = vk::ImageUsageFlagBits::eSampled;
const auto sampled_plane_id = texture_cache.FindImage(sampled_plane);
(void)texture_cache.FindTexture(sampled_plane_id, sampled_plane);
(void)texture_cache.FindDepthTarget(plane_depth_id, plane_depth);
Require(name, "sampled stencil association",
texture_cache.GetImage(sampled_plane_id).depth_id == plane_depth_id,
"depth acquisition did not associate the sampled stencil plane");
TextureCacheTestAccess::ClearImage(texture_cache, scheduler.Current(), plane_depth_id,
{vk::ImageAspectFlagBits::eDepth | vk::ImageAspectFlagBits::eStencil, 0, 1, 0, 1},
stencil_initial);
TileBlockLayout stencil_block{};
Require(name, "independent stencil tile pitch",
TileGetBlockLayout(TileBlockFamily::Depth64KB, 1, stencil_block) &&
plane_depth.info.pitch == 128 && TileGetDepthPitch(3, 1) == 256,
"stencil must use its own one-byte tile layout");
std::vector<uint32_t> plane_words(plane_size / sizeof(uint32_t), 0);
auto *plane_bytes = reinterpret_cast<uint8_t *>(plane_words.data());
std::array<uint32_t, texels> plane_offsets{};
for (uint32_t i = 0; i < texels; ++i) {
Require(name, "stencil tile offset",
TileGetBlockOffset(stencil_block, i % 3, i / 3, 0, plane_offsets[i]),
"stencil texel offset is unavailable");
plane_bytes[plane_offsets[i]] = static_cast<uint8_t>(0x21 + i * 7);
}
auto plane_upload = CreateHostBuffer(name, plane_size,
vk::BufferUsageFlagBits::eTransferSrc, plane_words);
const auto plane_program = make_buffer_program(
ShaderType::Compute, {.written = true, .formatted = true});
ShaderBufferResource plane_descriptor{};
plane_descriptor.UpdateAddress48(plane_address);
plane_descriptor.fields[1] |= 4u << 16u;
plane_descriptor.fields[2] = plane_size / sizeof(uint32_t);
plane_descriptor.fields[3] = 0x00014204u;
ShaderRecompiler::IR::ResourceSnapshot plane_snapshot;
auto &plane_value = plane_snapshot.buffers.emplace_back();
std::memcpy(plane_value.dwords.data(), plane_descriptor.fields,
sizeof(plane_descriptor.fields));
plane_value.dword_count = 4;
const ShaderStageRuntime plane_runtime{&plane_program, &plane_snapshot};
PreparedBindings plane_bindings;
executor.PrepareBindings(plane_runtime, plane_bindings);
vk::MemoryBarrier2 plane_barrier{};
plane_barrier.srcStageMask = plane_barrier.dstStageMask =
vk::PipelineStageFlagBits2::eAllCommands;
plane_barrier.srcAccessMask = plane_barrier.dstAccessMask =
vk::AccessFlagBits2::eMemoryRead | vk::AccessFlagBits2::eMemoryWrite;
vk::DependencyInfo plane_dependency{};
plane_dependency.memoryBarrierCount = 1;
plane_dependency.pMemoryBarriers = &plane_barrier;
const auto write_plane = [&] {
std::array<PreparedBindings *, 1> stages{&plane_bindings};
executor.FindBuffers(stages);
executor.RebindBuffers(plane_bindings);
const auto &plane_buffer = plane_bindings.buffers[0];
const vk::BufferCopy plane_copy{0, plane_buffer.offset, plane_size};
scheduler.Current().Handle().pipelineBarrier2(plane_dependency);
scheduler.Current().Handle().copyBuffer(plane_upload.buffer, plane_buffer.buffer,
1, &plane_copy);
scheduler.Current().Handle().pipelineBarrier2(plane_dependency);
RenderExecutorTestAccess::ResetBindings(executor);
};
write_plane();
auto plane_readback = CreateHostBuffer(name, texels * 4 + 8,
vk::BufferUsageFlagBits::eTransferDst, {});
const std::array<vk::BufferImageCopy, 2> plane_copies{{
{0, 0, 0, {vk::ImageAspectFlagBits::eDepth, 0, 0, 1}, {}, {3, 2, 1}},
{texels * 4, 0, 0, {vk::ImageAspectFlagBits::eStencil, 0, 0, 1}, {}, {3, 2, 1}}}};
const auto check_plane = [&](const char *phase, bool native_clear) {
(void)texture_cache.FindDepthTarget(plane_depth_id, plane_depth);
texture_cache.GetImage(plane_depth_id).Download(
plane_copies, plane_readback.buffer, 0, plane_readback.size);
scheduler.Current().Handle().pipelineBarrier2(stencil_dependency);
scheduler.Finish();
const auto values = ReadBuffer(name, plane_readback, texels + 2);
const auto *bytes = reinterpret_cast<const uint8_t *>(values.data() + texels);
for (uint32_t i = 0; i < texels; ++i) {
Require(name, phase,
values[i] == std::bit_cast<uint32_t>(0.625f) &&
bytes[i] == (native_clear ? 0x67 : plane_bytes[plane_offsets[i]]),
"stencil acquisition lost bytes or overwrote the independent depth plane");
}
};
check_plane("GPU stencil buffer write", false);
write_plane();
vk::ClearValue stencil_again{};
stencil_again.depthStencil.stencil = 0x67;
TextureCacheTestAccess::ClearImage(texture_cache, scheduler.Current(), plane_depth_id,
{vk::ImageAspectFlagBits::eStencil, 0, 1, 0, 1}, stencil_again);
check_plane("consumed stencil dirtiness", true);
check_plane("repeated clean stencil acquisition", true);
for (uint32_t i = 0; i < texels; ++i) {
plane_bytes[plane_offsets[i]] += 0x10;
}
std::memcpy(reinterpret_cast<void *>(plane_address), plane_words.data(), plane_size);
check_plane("CPU stencil plane write", false);
std::memcpy(reinterpret_cast<void *>(plane_address), plane_words.data(), plane_size);
TextureCacheTestAccess::ClearImage(texture_cache, scheduler.Current(), plane_depth_id,
{vk::ImageAspectFlagBits::eStencil, 0, 1, 0, 1}, stencil_again);
for (uint32_t i = 0; i < texels; ++i) {
plane_bytes[plane_offsets[i]] += 0x10;
}
std::memcpy(reinterpret_cast<void *>(plane_address), plane_words.data(), plane_size);
check_plane("CPU stencil write after native clear", false);
for (uint32_t i = 0; i < texels; ++i) {
plane_bytes[plane_offsets[i]] += 0x10;
}
std::memcpy(reinterpret_cast<void *>(plane_address), plane_words.data(), plane_size);
Require(name, "sampled stencil view",
texture_cache.FindTexture(plane_depth_id, sampled_plane) != nullptr,
"an associated stencil plane could not be sampled");
texture_cache.GetImage(plane_depth_id).Download(
plane_copies, plane_readback.buffer, 0, plane_readback.size);
scheduler.Current().Handle().pipelineBarrier2(stencil_dependency);
scheduler.Finish();
const auto sampled_values = ReadBuffer(name, plane_readback, texels + 2);
const auto *sampled_bytes =
reinterpret_cast<const uint8_t *>(sampled_values.data() + texels);
for (uint32_t i = 0; i < texels; ++i) {
Require(name, "sampled stencil plane refresh",
sampled_bytes[i] == plane_bytes[plane_offsets[i]],
"sampling without another depth acquisition used stale stencil bytes");
}
DestroyBuffer(&plane_readback);
DestroyBuffer(&plane_upload);
LibKernel::Memory::InstallGpuResources(nullptr);
}
resources.UnmapMemory(base, allocation_size);
scheduler.Finish();
RenderExecutorTestAccess::DestroyDescriptorPipelines(
executor, descriptor_pipelines);
context.ShutdownGpu();
}
Require(name, "unmap direct backing",
Libs::LibKernel::Memory::KernelMunmap(base, allocation_size) == 0,
"descriptor discovery direct mapping release failed");
Require(name, "release direct backing",
Libs::LibKernel::Memory::KernelReleaseDirectMemory(
direct_offset, allocation_size) == 0,
"descriptor discovery direct-memory allocation release failed");
std::printf("[host] %-32s ok\n", name);
}
Buffer CreateStorageBuffer(const char *shader_name,
const std::vector<u32> &initial,
size_t dword_count, bool device_address = false) {
Buffer ret;
ret.size = static_cast<vk::DeviceSize>(std::max<size_t>(dword_count, 1u) *
sizeof(u32));
vk::BufferCreateInfo buffer_info{};
buffer_info.sType = vk::StructureType::eBufferCreateInfo;
buffer_info.size = ret.size;
buffer_info.usage = vk::BufferUsageFlagBits::eStorageBuffer;
if (device_address) {
buffer_info.usage |= vk::BufferUsageFlagBits::eShaderDeviceAddress;
}
buffer_info.sharingMode = vk::SharingMode::eExclusive;
RequireVk(shader_name, "dispatch",
m_device.createBuffer(&buffer_info, nullptr, &ret.buffer),
"vkCreateBuffer");
vk::MemoryRequirements req{};
m_device.getBufferMemoryRequirements(ret.buffer, &req);
u32 memory_type = 0;
ret.coherent = FindMemoryType(req.memoryTypeBits,
vk::MemoryPropertyFlagBits::eHostVisible |
vk::MemoryPropertyFlagBits::eHostCoherent,
&memory_type);
if (!ret.coherent) {
Require(shader_name, "dispatch",
FindMemoryType(req.memoryTypeBits,
vk::MemoryPropertyFlagBits::eHostVisible,
&memory_type),
"no host-visible memory type for storage buffer");
}
vk::MemoryAllocateInfo alloc{};
alloc.sType = vk::StructureType::eMemoryAllocateInfo;
alloc.allocationSize = req.size;
alloc.memoryTypeIndex = memory_type;
vk::MemoryAllocateFlagsInfo flags{};
if (device_address) {
flags.sType = vk::StructureType::eMemoryAllocateFlagsInfo;
flags.flags = vk::MemoryAllocateFlagBits::eDeviceAddress;
alloc.pNext = &flags;
}
RequireVk(shader_name, "dispatch",
m_device.allocateMemory(&alloc, nullptr, &ret.memory),
"vkAllocateMemory");
RequireVk(shader_name, "dispatch",
m_device.bindBufferMemory(ret.buffer, ret.memory, 0),
"vkBindBufferMemory");
if (device_address) {
vk::BufferDeviceAddressInfo address_info{};
address_info.sType = vk::StructureType::eBufferDeviceAddressInfo;
address_info.buffer = ret.buffer;
ret.device_address = m_device.getBufferAddress(address_info);
Require(shader_name, "dispatch", ret.device_address != 0,
"storage buffer has no device address");
}
std::vector<u32> contents(dword_count, 0);
for (size_t i = 0; i < initial.size() && i < contents.size(); i++) {
contents[i] = initial[i];
}
WriteBuffer(shader_name, ret, contents);
return ret;
}
void DestroyBuffer(Buffer *buffer) {
if (buffer == nullptr) {
return;
}
if (buffer->buffer != nullptr) {
m_device.destroyBuffer(buffer->buffer, nullptr);
buffer->buffer = nullptr;
}
if (buffer->memory != nullptr) {
m_device.freeMemory(buffer->memory, nullptr);
buffer->memory = nullptr;
}
}
Image CreateImage2D(const char *shader_name, u32 width, u32 height,
vk::Format format, vk::ImageUsageFlags usage,
const std::vector<u32> &initial, u32 dwords_per_pixel,
vk::ImageLayout final_layout,
vk::Format view_format = vk::Format::eUndefined) {
std::vector<std::vector<u32>> mips;
if (!initial.empty()) {
mips.push_back(initial);
}
return CreateImageMips(shader_name, width, height, format, usage, mips,
dwords_per_pixel, final_layout, vk::ImageType::e2D,
vk::ImageViewType::e2D, 1, 0, 0,
vk::SampleCountFlagBits::e1, view_format);
}
static u32 MipExtent(u32 value, u32 level) {
for (u32 i = 0; i < level && value > 1u; i++) {
value >>= 1u;
}
return std::max(value, 1u);
}
static size_t ImageMipDwordCount(u32 width, u32 height, vk::Format format,
u32 level, u32 layers = 1) {
const auto block = vk::blockExtent(format);
const auto blocks_x = (MipExtent(width, level) + block[0] - 1u) / block[0];
const auto blocks_y = (MipExtent(height, level) + block[1] - 1u) / block[1];
return (size_t{blocks_x} * blocks_y * layers * vk::blockSize(format) + 3u) / 4u;
}
Image CreateImageMips(const char *shader_name, u32 width, u32 height,
vk::Format format, vk::ImageUsageFlags usage,
const std::vector<std::vector<u32>> &initial_mips,
u32 dwords_per_pixel, vk::ImageLayout final_layout,
vk::ImageType image_type, vk::ImageViewType view_type,
u32 layers, u32 view_base_layer = 0,
u32 view_layers = 0,
vk::SampleCountFlagBits samples = vk::SampleCountFlagBits::e1,
vk::Format view_format = vk::Format::eUndefined,
u32 view_base_mip = 0) {
Image ret;
ret.format = format;
ret.width = width;
ret.height = height;
ret.layers = layers;
ret.mip_levels = std::max<u32>(static_cast<u32>(initial_mips.size()), 1u);
Require(shader_name, "image format size", dwords_per_pixel == (vk::blockSize(format) + 3u) / 4u,
"image fixture word size disagrees with its native format");
vk::ImageCreateInfo image_info{};
image_info.sType = vk::StructureType::eImageCreateInfo;
image_info.imageType = image_type;
image_info.format = format;
image_info.extent.width = width;
image_info.extent.height = height;
image_info.extent.depth = 1;
image_info.mipLevels = ret.mip_levels;
image_info.arrayLayers = layers;
image_info.samples = samples;
image_info.tiling = vk::ImageTiling::eOptimal;
image_info.usage = usage | vk::ImageUsageFlagBits::eTransferDst |
vk::ImageUsageFlagBits::eTransferSrc;
image_info.sharingMode = vk::SharingMode::eExclusive;
image_info.initialLayout = vk::ImageLayout::eUndefined;
if (view_format != vk::Format::eUndefined && view_format != format) {
image_info.flags |= vk::ImageCreateFlagBits::eMutableFormat;
}
RequireVk(shader_name, "dispatch",
m_device.createImage(&image_info, nullptr, &ret.image),
"vkCreateImage");
vk::MemoryRequirements req{};
m_device.getImageMemoryRequirements(ret.image, &req);
u32 memory_type = 0;
if (!FindMemoryType(req.memoryTypeBits,
vk::MemoryPropertyFlagBits::eDeviceLocal,
&memory_type)) {
Require(shader_name, "dispatch",
FindMemoryType(req.memoryTypeBits, {}, &memory_type),
"no memory type for image");
}
vk::MemoryAllocateInfo alloc{};
alloc.sType = vk::StructureType::eMemoryAllocateInfo;
alloc.allocationSize = req.size;
alloc.memoryTypeIndex = memory_type;
RequireVk(shader_name, "dispatch",
m_device.allocateMemory(&alloc, nullptr, &ret.memory),
"vk::Device::allocateMemory(image)");
RequireVk(shader_name, "dispatch",
m_device.bindImageMemory(ret.image, ret.memory, 0),
"vkBindImageMemory");
vk::ImageViewCreateInfo view_info{};
view_info.sType = vk::StructureType::eImageViewCreateInfo;
view_info.image = ret.image;
view_info.viewType = view_type;
view_info.format = view_format == vk::Format::eUndefined ? format : view_format;
view_info.subresourceRange.aspectMask = vk::ImageAspectFlagBits::eColor;
Require(shader_name, "dispatch", view_base_mip < ret.mip_levels,
"image view base mip is out of bounds");
view_info.subresourceRange.baseMipLevel = view_base_mip;
view_info.subresourceRange.levelCount = ret.mip_levels - view_base_mip;
Require(shader_name, "dispatch", view_base_layer < layers,
"image view base layer is out of bounds");
if (view_layers == 0) {
view_layers = layers - view_base_layer;
}
Require(shader_name, "dispatch", view_layers <= layers - view_base_layer,
"image view layer count is out of bounds");
view_info.subresourceRange.baseArrayLayer = view_base_layer;
view_info.subresourceRange.layerCount = view_layers;
RequireVk(shader_name, "dispatch",
m_device.createImageView(&view_info, nullptr, &ret.view),
"vkCreateImageView");
if (!initial_mips.empty()) {
size_t total_dwords = 0;
for (u32 level = 0; level < ret.mip_levels; level++) {
total_dwords +=
ImageMipDwordCount(width, height, format, level, layers);
}
std::vector<u32> contents(total_dwords, 0);
size_t offset = 0;
for (u32 level = 0; level < ret.mip_levels; level++) {
const auto level_dwords =
ImageMipDwordCount(width, height, format, level, layers);
const auto &src = initial_mips[level];
for (size_t i = 0; i < src.size() && i < level_dwords; i++) {
contents[offset + i] = src[i];
}
offset += level_dwords;
}
auto staging =
CreateHostBuffer(shader_name, contents.size() * sizeof(u32),
vk::BufferUsageFlagBits::eTransferSrc, contents);
UploadImageMips(shader_name, &ret, staging.buffer, final_layout);
DestroyBuffer(&staging);
} else {
TransitionImage(shader_name, &ret, final_layout,
vk::PipelineStageFlagBits::eTopOfPipe,
vk::PipelineStageFlagBits::eComputeShader, {},
AccessForLayout(final_layout));
}
return ret;
}
void DestroyImage(Image *image) {
if (image == nullptr) {
return;
}
if (image->view != nullptr) {
m_device.destroyImageView(image->view, nullptr);
image->view = nullptr;
}
if (image->image != nullptr) {
m_device.destroyImage(image->image, nullptr);
image->image = nullptr;
}
if (image->memory != nullptr) {
m_device.freeMemory(image->memory, nullptr);
image->memory = nullptr;
}
}
std::vector<u32> ReadImage(const char *shader_name, Image *image) {
const auto dword_count = ImageMipDwordCount(image->width, image->height,
image->format, 0, image->layers);
auto staging = CreateHostBuffer(shader_name, dword_count * sizeof(u32),
vk::BufferUsageFlagBits::eTransferDst, {});
vk::CommandBuffer cmd = BeginCommands(shader_name, "readback");
AddImageBarrier(
cmd, image->image, image->layout, vk::ImageLayout::eTransferSrcOptimal,
vk::PipelineStageFlagBits::eAllCommands,
vk::PipelineStageFlagBits::eTransfer,
vk::AccessFlagBits::eMemoryRead | vk::AccessFlagBits::eMemoryWrite,
vk::AccessFlagBits::eTransferRead, image->mip_levels, image->layers);
vk::BufferImageCopy copy{};
copy.bufferOffset = 0;
copy.imageSubresource.aspectMask = vk::ImageAspectFlagBits::eColor;
copy.imageSubresource.mipLevel = 0;
copy.imageSubresource.baseArrayLayer = 0;
copy.imageSubresource.layerCount = image->layers;
copy.imageExtent.width = image->width;
copy.imageExtent.height = image->height;
copy.imageExtent.depth = 1;
cmd.copyImageToBuffer(image->image, vk::ImageLayout::eTransferSrcOptimal,
staging.buffer, 1, &copy);
EndSubmitAndFree(shader_name, "readback", cmd);
image->layout = vk::ImageLayout::eTransferSrcOptimal;
auto ret = ReadBuffer(shader_name, staging, dword_count);
DestroyBuffer(&staging);
return ret;
}
vk::Sampler CreateSampler(const char *shader_name, u32 mip_levels = 1,
vk::Filter filter = vk::Filter::eNearest) {
vk::SamplerCreateInfo sampler_info{};
sampler_info.sType = vk::StructureType::eSamplerCreateInfo;
sampler_info.magFilter = filter;
sampler_info.minFilter = filter;
sampler_info.mipmapMode = vk::SamplerMipmapMode::eNearest;
sampler_info.addressModeU = vk::SamplerAddressMode::eClampToEdge;
sampler_info.addressModeV = vk::SamplerAddressMode::eClampToEdge;
sampler_info.addressModeW = vk::SamplerAddressMode::eClampToEdge;
sampler_info.minLod = 0.0f;
sampler_info.maxLod = static_cast<float>(mip_levels - 1u);
vk::Sampler sampler = nullptr;
RequireVk(shader_name, "dispatch",
m_device.createSampler(&sampler_info, nullptr, &sampler),
"vkCreateSampler");
return sampler;
}
std::vector<u32> ReadBuffer(const char *shader_name, const Buffer &buffer,
size_t dword_count) {
if (!buffer.coherent) {
vk::MappedMemoryRange range{};
range.sType = vk::StructureType::eMappedMemoryRange;
range.memory = buffer.memory;
range.offset = 0;
range.size = VK_WHOLE_SIZE;
RequireVk(shader_name, "readback",
m_device.invalidateMappedMemoryRanges(1, &range),
"vkInvalidateMappedMemoryRanges");
}
void *data = nullptr;
RequireVk(shader_name, "readback",
m_device.mapMemory(buffer.memory, 0, buffer.size, {}, &data),
"vkMapMemory");
std::vector<u32> ret(dword_count, 0);
std::memcpy(ret.data(), data, dword_count * sizeof(u32));
m_device.unmapMemory(buffer.memory);
return ret;
}
void Dispatch(const TestCase &test, const CompiledShader &compiled,
const Buffer &buffer, const Buffer *gds_buffer = nullptr,
const Image *sampled_image = nullptr,
const Image *storage_image = nullptr,
const Image *storage_image_uint = nullptr,
vk::Sampler sampler = nullptr,
std::span<const Image> image_resources = {},
std::span<const vk::Sampler> sampler_resources = {}) {
using Kind = ShaderRecompiler::IR::DescriptorBindingKind;
const auto &layout = compiled.program.bindings;
auto shader_data = compiled.packed_user_data;
if (layout.dispatch_thread_dword != ShaderRecompiler::IR::PushData::NoStart) {
Require(test.name, "dispatch dimensions", test.has_compute_info,
"runtime thread extents are required by the compiled shader");
std::copy_n(test.compute_info.dispatch_threads_num, 3,
shader_data.begin() + layout.dispatch_thread_dword);
}
auto Count = [&](Kind kind) {
return layout.descriptor_counts[static_cast<size_t>(kind)];
};
auto Native = [](Kind kind) {
return ShaderRecompiler::IR::NativeBinding(ShaderType::Compute, kind);
};
for (const auto &image : compiled.program.info.images) {
bool supported =
image.dimension == ShaderRecompiler::Decoder::ImageDimension::Dim1D ||
image.dimension == ShaderRecompiler::Decoder::ImageDimension::Dim2D ||
image.dimension == ShaderRecompiler::Decoder::ImageDimension::Dim2DArray;
if (image.resource_class == ShaderRecompiler::IR::ImageResourceClass::Sampled) {
supported = supported ||
image.dimension == ShaderRecompiler::Decoder::ImageDimension::Dim1DArray;
}
Require(test.name, "dispatch", supported,
"unsupported image dimension needs a matching Vulkan test view");
}
vk::ShaderModuleCreateInfo module_info{};
module_info.sType = vk::StructureType::eShaderModuleCreateInfo;
module_info.codeSize = compiled.spirv.size() * sizeof(u32);
module_info.pCode = compiled.spirv.data();
vk::ShaderModule module = nullptr;
RequireVk(test.name, "SPIR-V module",
m_device.createShaderModule(&module_info, nullptr, &module),
"vkCreateShaderModule");
std::vector<vk::DescriptorSetLayoutBinding> layout_bindings;
auto add_layout_binding =
[&layout_bindings](u32 binding, vk::DescriptorType type, u32 count) {
if (count == 0) {
return;
}
vk::DescriptorSetLayoutBinding item{};
item.binding = binding;
item.descriptorType = type;
item.descriptorCount = count;
item.stageFlags = vk::ShaderStageFlagBits::eCompute;
layout_bindings.push_back(item);
};
for (auto mask = layout.descriptor_mask; mask != 0; mask &= mask - 1) {
const auto index = std::countr_zero(mask);
const auto kind = static_cast<Kind>(index);
add_layout_binding(Native(kind), NativeDescriptorType(kind), layout.descriptor_counts[index]);
}
vk::DescriptorSetLayoutCreateInfo layout_info{};
layout_info.sType = vk::StructureType::eDescriptorSetLayoutCreateInfo;
layout_info.bindingCount = static_cast<u32>(layout_bindings.size());
layout_info.pBindings =
layout_bindings.empty() ? nullptr : layout_bindings.data();
vk::DescriptorSetLayout descriptor_layout = nullptr;
RequireVk(test.name, "dispatch",
m_device.createDescriptorSetLayout(&layout_info, nullptr,
&descriptor_layout),
"vkCreateDescriptorSetLayout");
vk::PipelineLayoutCreateInfo pipeline_layout_info{};
pipeline_layout_info.sType = vk::StructureType::ePipelineLayoutCreateInfo;
pipeline_layout_info.setLayoutCount = 1;
pipeline_layout_info.pSetLayouts = &descriptor_layout;
vk::PushConstantRange push_range{};
if (layout.UsesPushData()) {
push_range.stageFlags = vk::ShaderStageFlagBits::eCompute;
push_range.offset = 0;
push_range.size = ShaderRecompiler::IR::NativePushConstantSize;
pipeline_layout_info.pushConstantRangeCount = 1;
pipeline_layout_info.pPushConstantRanges = &push_range;
}
vk::PipelineLayout pipeline_layout = nullptr;
RequireVk(test.name, "dispatch",
m_device.createPipelineLayout(&pipeline_layout_info, nullptr,
&pipeline_layout),
"vkCreatePipelineLayout");
vk::PipelineShaderStageCreateInfo stage{};
stage.sType = vk::StructureType::ePipelineShaderStageCreateInfo;
stage.stage = vk::ShaderStageFlagBits::eCompute;
stage.module = module;
stage.pName = "main";
vk::ComputePipelineCreateInfo pipeline_info{};
pipeline_info.sType = vk::StructureType::eComputePipelineCreateInfo;
pipeline_info.stage = stage;
pipeline_info.layout = pipeline_layout;
vk::Pipeline pipeline = nullptr;
RequireVk(test.name, "dispatch",
m_device.createComputePipelines(nullptr, 1, &pipeline_info,
nullptr, &pipeline),
"vkCreateComputePipelines");
std::vector<vk::DescriptorPoolSize> pool_sizes;
auto add_pool_size = [&pool_sizes](vk::DescriptorType type, u32 count) {
if (count == 0) {
return;
}
for (auto &size : pool_sizes) {
if (size.type == type) {
size.descriptorCount += count;
return;
}
}
pool_sizes.push_back({type, count});
};
for (auto mask = layout.descriptor_mask; mask != 0; mask &= mask - 1) {
const auto index = std::countr_zero(mask);
add_pool_size(NativeDescriptorType(static_cast<Kind>(index)), layout.descriptor_counts[index]);
}
vk::DescriptorPoolCreateInfo pool_info{};
pool_info.sType = vk::StructureType::eDescriptorPoolCreateInfo;
pool_info.maxSets = 1;
pool_info.poolSizeCount = static_cast<u32>(pool_sizes.size());
pool_info.pPoolSizes = pool_sizes.empty() ? nullptr : pool_sizes.data();
vk::DescriptorPool descriptor_pool = nullptr;
RequireVk(
test.name, "dispatch",
m_device.createDescriptorPool(&pool_info, nullptr, &descriptor_pool),
"vkCreateDescriptorPool");
vk::DescriptorSetAllocateInfo set_info{};
set_info.sType = vk::StructureType::eDescriptorSetAllocateInfo;
set_info.descriptorPool = descriptor_pool;
set_info.descriptorSetCount = 1;
set_info.pSetLayouts = &descriptor_layout;
vk::DescriptorSet descriptor_set = nullptr;
RequireVk(test.name, "dispatch",
m_device.allocateDescriptorSets(&set_info, &descriptor_set),
"vkAllocateDescriptorSets");
std::vector<vk::WriteDescriptorSet> writes;
std::vector<vk::DescriptorBufferInfo> buffer_infos;
std::array<std::vector<vk::DescriptorImageInfo>, ShaderRecompiler::IR::ImageBindingCount> image_infos;
std::vector<vk::ImageView> sampled_mip_views;
std::vector<vk::DescriptorImageInfo> sampler_infos;
Buffer flattened_buffer;
Buffer user_data_buffer;
Buffer shared_buffer;
vk::DescriptorBufferInfo flattened_info{};
vk::DescriptorBufferInfo user_data_info{};
vk::DescriptorBufferInfo shared_info{};
vk::DescriptorBufferInfo gds_info{};
vk::DescriptorBufferInfo bda_pagetable_info{};
vk::DescriptorBufferInfo fault_buffer_info{};
if (Count(Kind::SharedMemory) != 0) {
const auto bytes = uint64_t{test.compute_info.lds_size_dwords} * sizeof(u32) *
test.dispatch_x * test.dispatch_y * test.dispatch_z;
shared_buffer = CreateHostBuffer(test.name, bytes, vk::BufferUsageFlagBits::eStorageBuffer, {});
shared_info = {shared_buffer.buffer, 0, shared_buffer.size};
vk::WriteDescriptorSet write{};
write.dstSet = descriptor_set;
write.dstBinding = Native(Kind::SharedMemory);
write.descriptorCount = 1;
write.descriptorType = vk::DescriptorType::eStorageBuffer;
write.pBufferInfo = &shared_info;
writes.push_back(write);
}
const bool uses_bda = Count(Kind::BdaPagetable) != 0;
Require(test.name, "dispatch",
uses_bda == (Count(Kind::FaultBuffer) != 0),
"BDA page table and fault buffer must be bound together");
if (uses_bda) {
EnsureBdaBuffers(test.name);
bda_pagetable_info = {m_bda_pagetable_buffer.buffer, 0,
m_bda_pagetable_buffer.size};
fault_buffer_info = {m_fault_buffer.buffer, 0, m_fault_buffer.size};
const std::array special_buffers{
std::pair{Kind::BdaPagetable, &bda_pagetable_info},
std::pair{Kind::FaultBuffer, &fault_buffer_info},
};
for (const auto &[kind, info] : special_buffers) {
vk::WriteDescriptorSet write{};
write.sType = vk::StructureType::eWriteDescriptorSet;
write.dstSet = descriptor_set;
write.dstBinding = Native(kind);
write.descriptorCount = 1;
write.descriptorType = vk::DescriptorType::eStorageBuffer;
write.pBufferInfo = info;
writes.push_back(write);
}
}
if (Count(Kind::Buffers) != 0) {
buffer_infos.resize(Count(Kind::Buffers));
for (u32 resource = 0; resource < compiled.program.info.buffers.size(); ++resource) {
const auto index = compiled.program.info.buffers[resource].descriptor_index;
if (index == UINT32_MAX) continue;
auto &info = buffer_infos[index];
info.buffer = buffer.buffer;
info.offset = 0;
info.range = buffer.size;
if (resource < test.storage_buffer_ranges.size()) {
const auto [offset, size] = test.storage_buffer_ranges[resource];
info.offset = offset;
info.range = size;
Require(test.name, "dispatch", uint64_t{offset} + size <= buffer.size,
"storage buffer view exceeds its backing buffer");
} else if (test.storage_buffer_range_bytes != 0) {
const auto offset = resource < test.storage_buffer_offsets.size()
? test.storage_buffer_offsets[resource]
: 0u;
info.range = static_cast<vk::DeviceSize>(
test.storage_buffer_range_bytes + offset);
Require(test.name, "dispatch", info.range <= buffer.size,
"storage buffer descriptor range exceeds backing buffer");
}
}
vk::WriteDescriptorSet write{};
write.sType = vk::StructureType::eWriteDescriptorSet;
write.dstSet = descriptor_set;
write.dstBinding = Native(Kind::Buffers);
write.descriptorCount = static_cast<u32>(buffer_infos.size());
write.descriptorType = vk::DescriptorType::eStorageBuffer;
write.pBufferInfo = buffer_infos.data();
writes.push_back(write);
}
if (Count(Kind::FlattenedSrt) != 0) {
flattened_buffer =
CreateStorageBuffer(test.name, compiled.resources.flattened_srt,
compiled.resources.flattened_srt.size());
flattened_info = {flattened_buffer.buffer, 0, flattened_buffer.size};
vk::WriteDescriptorSet write{};
write.sType = vk::StructureType::eWriteDescriptorSet;
write.dstSet = descriptor_set;
write.dstBinding = Native(Kind::FlattenedSrt);
write.descriptorCount = 1;
write.descriptorType = vk::DescriptorType::eStorageBuffer;
write.pBufferInfo = &flattened_info;
writes.push_back(write);
}
if (Count(Kind::ShaderData) != 0) {
user_data_buffer =
CreateStorageBuffer(test.name, shader_data, shader_data.size());
user_data_info = {user_data_buffer.buffer, 0, user_data_buffer.size};
vk::WriteDescriptorSet write{};
write.sType = vk::StructureType::eWriteDescriptorSet;
write.dstSet = descriptor_set;
write.dstBinding = Native(Kind::ShaderData);
write.descriptorCount = 1;
write.descriptorType = vk::DescriptorType::eStorageBuffer;
write.pBufferInfo = &user_data_info;
writes.push_back(write);
}
if (Count(Kind::Gds) != 0) {
Require(test.name, "dispatch", gds_buffer != nullptr,
"GDS descriptor requested but no GDS buffer was provided");
gds_info = {gds_buffer->buffer, 0, gds_buffer->size};
vk::WriteDescriptorSet write{};
write.sType = vk::StructureType::eWriteDescriptorSet;
write.dstSet = descriptor_set;
write.dstBinding = Native(Kind::Gds);
write.descriptorCount = 1;
write.descriptorType = vk::DescriptorType::eStorageBuffer;
write.pBufferInfo = &gds_info;
writes.push_back(write);
}
Require(test.name, "dispatch", image_resources.empty() ||
image_resources.size() == compiled.program.info.images.size(),
"image resources must match the logical images");
for (u32 resource = 0; resource < compiled.program.info.images.size(); ++resource) {
const auto &image = compiled.program.info.images[resource];
const auto kind = *ShaderRecompiler::IR::DescriptorBindingForImage(image);
auto &infos = image_infos[ShaderRecompiler::IR::ImageBindingIndex(kind)];
infos.resize(Count(kind));
const bool sampled = image.resource_class == ShaderRecompiler::IR::ImageResourceClass::Sampled;
const auto *source_image = !image_resources.empty() ? &image_resources[resource]
: sampled ? sampled_image
: image.numeric_class == Prospero::TextureNumericClass::Float
? storage_image : storage_image_uint;
Require(test.name, "dispatch", source_image != nullptr,
"image descriptor requested but no matching image was provided");
for (u32 mip_index = 0; mip_index < image.mip_count; ++mip_index) {
auto &info = infos[image.descriptor_index + mip_index];
info.imageView = source_image->view;
info.imageLayout = source_image->layout;
if (sampled && image.mip_mode == ShaderRecompiler::IR::ImageMipMode::Dynamic) {
const auto mip = test.sampled_image_view_base_mip + mip_index;
Require(test.name, "dispatch", mip < source_image->mip_levels,
"sampled mip descriptor exceeds the supplied image");
vk::ImageViewCreateInfo view{};
view.image = source_image->image;
view.viewType = test.sampled_image_view_type;
view.format = source_image->format;
view.subresourceRange = {
image.depth_compare ? vk::ImageAspectFlagBits::eDepth : vk::ImageAspectFlagBits::eColor, mip, 1,
test.sampled_image_view_base_layer,
test.sampled_image_view_layers != 0
? test.sampled_image_view_layers
: source_image->layers - test.sampled_image_view_base_layer};
RequireVk(test.name, "dispatch",
m_device.createImageView(&view, nullptr, &info.imageView),
"vkCreateImageView(sampled mip)");
sampled_mip_views.push_back(info.imageView);
}
}
}
for (u32 group = 0; group < image_infos.size(); ++group) {
const auto &infos = image_infos[group];
if (infos.empty()) continue;
const auto kind = static_cast<Kind>(ShaderRecompiler::IR::FirstImageBinding + group);
vk::WriteDescriptorSet write{};
write.dstSet = descriptor_set;
write.dstBinding = Native(kind);
write.descriptorCount = static_cast<u32>(infos.size());
write.descriptorType = NativeDescriptorType(kind);
write.pImageInfo = infos.data();
writes.push_back(write);
}
if (Count(Kind::Samplers) != 0) {
Require(test.name, "dispatch", sampler_resources.empty()
? sampler != nullptr : sampler_resources.size() == Count(Kind::Samplers),
"sampler resources must match the native sampler array");
sampler_infos.resize(Count(Kind::Samplers));
for (u32 index = 0; index < sampler_infos.size(); ++index) {
sampler_infos[index].sampler = sampler_resources.empty() ? sampler : sampler_resources[index];
}
vk::WriteDescriptorSet write{};
write.dstSet = descriptor_set;
write.dstBinding = Native(Kind::Samplers);
write.descriptorCount = static_cast<u32>(sampler_infos.size());
write.descriptorType = vk::DescriptorType::eSampler;
write.pImageInfo = sampler_infos.data();
writes.push_back(write);
}
if (!writes.empty()) {
m_device.updateDescriptorSets(static_cast<u32>(writes.size()),
writes.data(), 0, nullptr);
}
vk::CommandBuffer cmd = BeginCommands(test.name, "dispatch");
if (uses_bda) {
Require(test.name, "dispatch", buffer.device_address != 0,
"BDA test backing has no device address");
Require(test.name, "dispatch", !test.bda_mappings.empty(),
"BDA test has no explicit guest mapping");
cmd.fillBuffer(m_bda_pagetable_buffer.buffer, 0,
m_bda_pagetable_buffer.size, 0);
cmd.fillBuffer(m_fault_buffer.buffer, 0, m_fault_buffer.size, 0);
std::array<vk::BufferMemoryBarrier, 2> barriers{};
barriers[0].sType = vk::StructureType::eBufferMemoryBarrier;
barriers[0].srcAccessMask = vk::AccessFlagBits::eTransferWrite;
barriers[0].dstAccessMask = vk::AccessFlagBits::eTransferWrite;
barriers[0].srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
barriers[0].dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
barriers[0].buffer = m_bda_pagetable_buffer.buffer;
barriers[0].offset = 0;
barriers[0].size = m_bda_pagetable_buffer.size;
barriers[1] = barriers[0];
barriers[1].dstAccessMask = vk::AccessFlagBits::eShaderRead |
vk::AccessFlagBits::eShaderWrite;
barriers[1].buffer = m_fault_buffer.buffer;
barriers[1].size = m_fault_buffer.size;
cmd.pipelineBarrier(vk::PipelineStageFlagBits::eTransfer,
vk::PipelineStageFlagBits::eTransfer, {}, 0, nullptr,
1, barriers.data(), 0, nullptr);
for (const auto &[guest_base, backing] : test.bda_mappings) {
const auto page_offset = guest_base &
(BufferCache::CACHING_PAGESIZE - 1);
Require(test.name, "dispatch",
backing >= page_offset && backing < buffer.size,
"BDA test mapping begins outside its backing buffer");
const auto bytes = page_offset + buffer.size - backing;
const auto pages =
(bytes + BufferCache::CACHING_PAGESIZE - 1) /
BufferCache::CACHING_PAGESIZE;
auto address = buffer.device_address + backing - page_offset;
auto page = BufferCache::PageIndex(guest_base);
for (uint64_t mapped = 0; mapped < pages; mapped++) {
cmd.updateBuffer(m_bda_pagetable_buffer.buffer,
(page + mapped) * sizeof(vk::DeviceAddress),
sizeof(address), &address);
address += BufferCache::CACHING_PAGESIZE;
}
}
barriers[0].dstAccessMask = vk::AccessFlagBits::eShaderRead;
cmd.pipelineBarrier(vk::PipelineStageFlagBits::eTransfer,
vk::PipelineStageFlagBits::eComputeShader, {}, 0,
nullptr, static_cast<u32>(barriers.size()),
barriers.data(), 0, nullptr);
}
cmd.bindPipeline(vk::PipelineBindPoint::eCompute, pipeline);
cmd.bindDescriptorSets(vk::PipelineBindPoint::eCompute, pipeline_layout, 0,
1, &descriptor_set, 0, nullptr);
if (layout.UsesPushData()) {
ShaderRecompiler::IR::PushData push_data;
std::copy(shader_data.begin(), shader_data.end(),
push_data.dwords.begin() + layout.push_data_start_dword);
cmd.pushConstants(pipeline_layout, vk::ShaderStageFlagBits::eCompute, 0,
sizeof(push_data), push_data.dwords.data());
}
cmd.dispatch(test.dispatch_x, test.dispatch_y, test.dispatch_z);
if (Count(Kind::Buffers) != 0) {
vk::BufferMemoryBarrier barrier{};
barrier.sType = vk::StructureType::eBufferMemoryBarrier;
barrier.srcAccessMask =
vk::AccessFlagBits::eShaderRead | vk::AccessFlagBits::eShaderWrite;
barrier.dstAccessMask = vk::AccessFlagBits::eHostRead;
barrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
barrier.buffer = buffer.buffer;
barrier.offset = 0;
barrier.size = buffer.size;
cmd.pipelineBarrier(vk::PipelineStageFlagBits::eComputeShader,
vk::PipelineStageFlagBits::eHost, {}, 0, nullptr, 1,
&barrier, 0, nullptr);
}
if (gds_buffer != nullptr) {
vk::BufferMemoryBarrier barrier{};
barrier.sType = vk::StructureType::eBufferMemoryBarrier;
barrier.srcAccessMask =
vk::AccessFlagBits::eShaderRead | vk::AccessFlagBits::eShaderWrite;
barrier.dstAccessMask = vk::AccessFlagBits::eHostRead;
barrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
barrier.buffer = gds_buffer->buffer;
barrier.offset = 0;
barrier.size = gds_buffer->size;
cmd.pipelineBarrier(vk::PipelineStageFlagBits::eComputeShader,
vk::PipelineStageFlagBits::eHost, {}, 0, nullptr, 1,
&barrier, 0, nullptr);
}
Buffer fault_result;
if (test.fault_free_address) {
Require(test.name, "fault check", uses_bda, "fault check requires BDA");
fault_result = CreateHostBuffer(test.name, sizeof(u32),
vk::BufferUsageFlagBits::eTransferDst, {});
const vk::MemoryBarrier ready{
.srcAccessMask = vk::AccessFlagBits::eShaderWrite | vk::AccessFlagBits::eTransferWrite,
.dstAccessMask = vk::AccessFlagBits::eTransferRead};
cmd.pipelineBarrier(vk::PipelineStageFlagBits::eComputeShader | vk::PipelineStageFlagBits::eTransfer,
vk::PipelineStageFlagBits::eTransfer, {}, 1, &ready, 0, nullptr, 0, nullptr);
const vk::BufferCopy region{(BufferCache::PageIndex(*test.fault_free_address) >> 5u) * sizeof(u32),
0, sizeof(u32)};
cmd.copyBuffer(m_fault_buffer.buffer, fault_result.buffer, 1, &region);
const vk::MemoryBarrier readable{.srcAccessMask = vk::AccessFlagBits::eTransferWrite,
.dstAccessMask = vk::AccessFlagBits::eHostRead};
cmd.pipelineBarrier(vk::PipelineStageFlagBits::eTransfer, vk::PipelineStageFlagBits::eHost,
{}, 1, &readable, 0, nullptr, 0, nullptr);
}
EndSubmitAndFree(test.name, "dispatch", cmd);
if (test.fault_free_address) {
Require(test.name, "fault check", ReadBuffer(test.name, fault_result, 1)[0] == 0u,
"suppressed formatted access touched an unmapped page");
DestroyBuffer(&fault_result);
}
for (const auto view : sampled_mip_views) {
m_device.destroyImageView(view, nullptr);
}
if (flattened_buffer.buffer != nullptr) {
DestroyBuffer(&flattened_buffer);
}
if (user_data_buffer.buffer != nullptr) {
DestroyBuffer(&user_data_buffer);
}
DestroyBuffer(&shared_buffer);
m_device.destroyDescriptorPool(descriptor_pool, nullptr);
m_device.destroyPipeline(pipeline, nullptr);
m_device.destroyPipelineLayout(pipeline_layout, nullptr);
m_device.destroyDescriptorSetLayout(descriptor_layout, nullptr);
m_device.destroyShaderModule(module, nullptr);
}
void CheckRasterization(
bool depth_feedback,
Prospero::BufferFormat color_format = Prospero::BufferFormat::k32_32_32_32Float) {
const char *name = depth_feedback ? "DepthAttachmentFeedback" : "PolygonModeRasterization";
uint32_t packed_color = 0;
std::array<float, 4> packed_expected{};
switch (color_format) {
case Prospero::BufferFormat::k11_11_10Float:
name = "PackedFloatVertexColor";
packed_color = 0x801e0380u;
packed_expected = {0.5f, 1.0f, 2.0f, 1.0f};
break;
case Prospero::BufferFormat::k10_10_10_2UScaled:
name = "PackedUscaledVertexColor";
packed_color = 513u | (257u << 10u) | (1023u << 20u) | (3u << 30u);
packed_expected = {513.0f, 257.0f, 1023.0f, 3.0f};
break;
default: break;
}
const bool packed_vertex_color = color_format != Prospero::BufferFormat::k32_32_32_32Float;
const uint32_t extent = depth_feedback ? 8 : 32;
constexpr uintptr_t depth_address = 0x0000000204400000ull;
constexpr uint64_t allocation_size = 0x40000;
constexpr uint64_t rect_address = depth_address + 0x8000;
constexpr uint64_t index_address = rect_address + 0x1000;
EnsureRuntimeContext();
RenderContext context(m_runtime_context);
auto &scheduler = context.GetCommandScheduler();
HW::Context registers{};
HW::UserConfig user_config{};
HW::Shader shaders{};
registers.SetRenderTargetMask(0xf);
registers.SetShaderMask(0xf);
scheduler.Begin(registers, user_config, shaders);
auto &resources = context;
auto &cache = context.GetTextureCache();
auto &executor = context.GetRenderExecutor();
int64_t direct_offset = -1;
RenderDepthInfo depth{};
Require(name, "direct allocation",
Libs::LibKernel::Memory::KernelAllocateDirectMemory(
0, Libs::LibKernel::Memory::KernelGetDirectMemorySize(),
allocation_size, allocation_size, 0, &direct_offset) == 0,
"depth allocation failed");
void *mapped = reinterpret_cast<void *>(depth_address);
Require(name, "direct mapping",
Libs::LibKernel::Memory::KernelMapDirectMemory(
&mapped, allocation_size, 0x3, 0x10, direct_offset,
allocation_size) == 0 &&
mapped == reinterpret_cast<void *>(depth_address),
"depth mapping failed");
std::memset(mapped, 0, allocation_size);
const auto initial_depth = [](uint32_t x, uint32_t y) {
return 0.125f + static_cast<float>(x + 2 * y) / 128.0f;
};
for (uint32_t y = 0; y < extent; y++) {
for (uint32_t x = 0; x < extent; x++) {
static_cast<float *>(mapped)[y * 64 + x] = initial_depth(x, y);
}
}
if (!depth_feedback && !packed_vertex_color) {
// Rectangle input contains three position/UV records.
constexpr std::array<float, 24> rect_vertices{
-0.75f, -0.75f, 0, 0, 0, 0, 0, 0,
0.75f, -0.75f, 0, 0, 1, 0, 0, 0,
-0.75f, 0.75f, 0, 0, 0, 1, 0, 0};
std::memcpy(reinterpret_cast<void *>(rect_address), rect_vertices.data(),
sizeof(rect_vertices));
constexpr std::array<uint32_t, 3> indices{0, 1, 2};
std::memcpy(reinterpret_cast<void *>(index_address), indices.data(), sizeof(indices));
}
resources.MapMemory(depth_address, allocation_size);
if (depth_feedback) {
depth.desc.type = BindingType::DepthTarget;
depth.desc.info.data = {depth_address, 256 * extent};
depth.desc.info.pixel_format = vk::Format::eD32Sfloat;
depth.desc.info.guest_format = Prospero::BufferFormat::k32Float;
depth.desc.info.extent = {extent, extent, 1};
depth.desc.info.pitch = 64;
depth.desc.info.bytes_per_block = 4;
depth.desc.info.tile_mode = Prospero::TileMode::kLinear;
depth.desc.info.UpdateSize();
depth.desc.view_info.format = depth.desc.info.pixel_format;
depth.desc.view_info.aspect = vk::ImageAspectFlagBits::eDepth;
depth.desc.view_info.usage = vk::ImageUsageFlagBits::eDepthStencilAttachment;
depth.depth_test_enable = depth.depth_write_enable = true;
depth.depth_compare_op = vk::CompareOp::eAlways;
depth.image_id = cache.FindImage(depth.desc);
}
GraphicsCase test;
test.name = name;
test.pixel_depth_export = depth_feedback;
if (depth_feedback) {
const ShaderTextureResource descriptor{{
static_cast<uint32_t>(depth_address >> 8u),
(static_cast<uint32_t>(Prospero::BufferFormat::k32Float) << 20u) |
(((extent - 1u) & 3u) << 30u),
((extent - 1u) >> 2u) | ((extent - 1u) << 14u),
DstSel(4, 4, 4, 4) |
(static_cast<uint32_t>(Prospero::ImageType::kColor2D) << 28u),
0, 0, 0, 0}};
test.has_user_data = true;
std::copy_n(descriptor.fields, 8, test.user_data.begin());
// Each fragment samples its own texel through interpolated screen UVs.
for (uint32_t component = 0; component < 2; component++) {
test.fragment_code.push_back(EncodeVintrp(0, 20 + component, 0, component, 0));
test.fragment_code.push_back(EncodeVintrp(1, 20 + component, 0, component, 1));
}
test.fragment_code.push_back(EncodeMimg0(0x27, 1));
test.fragment_code.push_back(EncodeMimg1(0, 20, 0, 2));
AppendVMovLiteral(&test.fragment_code, 1, 0x3e000000u);
test.fragment_code.push_back(EncodeVop2(0x03, 1, Vgpr(0), 1));
test.fragment_code.push_back(EncodeExp0(0x08, 0x1, false));
test.fragment_code.push_back(EncodeExp1(1, 1, 1, 1));
} else {
test.pixel_interpolator_settings = {0x400u};
for (uint32_t component = 0; component < (packed_vertex_color ? 4u : 1u); component++) {
test.fragment_code.push_back(EncodeVintrp(0x02, component, 0, component, 2));
}
}
test.fragment_code.push_back(EncodeExp0(0x00, 0xf));
test.fragment_code.push_back(packed_vertex_color ? EncodeExp1(0, 1, 2, 3)
: EncodeExp1(0, 0, 0, 0));
AppendEnd(&test.fragment_code);
auto fragment = CompileFragmentCase(test);
const auto vertex_spirv = TestSpv::MakePassthroughVertexSpirv(false);
ShaderProgram vertex_shader{1, CreateShaderModule(name, vertex_spirv)};
ShaderProgram pixel_shader{2, CreateShaderModule(name, fragment.spirv)};
ShaderRecompiler::IR::CompiledShaderInfo vertex_program{};
vertex_program.stage = ShaderType::Vertex;
vertex_program.info.vertex_fetch_components[0] = 2;
vertex_program.info.vertex_fetch_components[1] = packed_vertex_color ? 3 : 4;
ShaderRecompiler::IR::CompiledShaderInfo pixel_program{};
pixel_program.stage = ShaderType::Pixel;
pixel_program.info = fragment.program.info;
pixel_program.bindings = fragment.program.bindings;
ShaderVertexInputInfo vertex{};
ShaderRecompiler::IR::ResourceSnapshot vertex_snapshot;
vertex.stage.program = &vertex_program;
vertex.stage.resources = &vertex_snapshot;
vertex.resources_num = 2;
vertex.buffers_num = 1;
vertex.buffers[0].stride = 6 * sizeof(float);
vertex.resources[1].UpdateAddress48(2 * sizeof(float));
for (uint32_t i = 0; i < 2; i++) {
const auto format = i == 0 ? Prospero::BufferFormat::k32_32Float : color_format;
vertex.resources[i].fields[3] = DstSel(4, 5, 6, 7) |
(static_cast<uint32_t>(format) << 12u);
vertex.resources_dst[i].registers_num =
i == 0 ? 2 : vertex_program.info.vertex_fetch_components[1];
}
ShaderPixelInputInfo pixel{};
pixel.stage.program = &pixel_program;
pixel.stage.resources = &fragment.resources;
RenderColorInfo color{};
color.desc.type = BindingType::RenderTarget;
color.desc.info.data = {depth_address + 0x10000, extent * extent * 16};
color.desc.info.mip_layout[0] = {
0, color.desc.info.data.size, extent, extent, 0, color.desc.info.data.size, extent};
color.desc.info.tiling.block.bytes_per_element = 16;
color.desc.info.linear_slice_stride = color.desc.info.data.size;
color.desc.info.pixel_format = vk::Format::eR32G32B32A32Sfloat;
color.desc.info.guest_format = Prospero::BufferFormat::k32_32_32_32Float;
color.desc.info.extent = {extent, extent, 1};
color.desc.info.pitch = extent;
color.desc.info.bytes_per_block = 16;
color.desc.view_info.format = color.desc.info.pixel_format;
color.desc.view_info.usage = vk::ImageUsageFlagBits::eColorAttachment;
color.image_id = cache.FindImage(color.desc);
std::array<float, 18> vertices{
-0.75f, -0.75f, 1, 1, 1, 1, 0.75f, -0.75f, 0.5f, 1, 1, 1,
0.0f, 0.75f, 0.25f, 1, 1, 1};
if (depth_feedback) {
vertices = {-1, -1, 0, 0, 0, 1, 3, -1, 2, 0, 0, 1, -1, 3, 0, 2, 0, 1};
}
std::vector<u32> vertex_words(vertices.size());
std::memcpy(vertex_words.data(), vertices.data(), sizeof(vertices));
if (packed_vertex_color) {
for (uint32_t i = 0; i < 3; i++) {
vertex_words[i * 6 + 2] = packed_color;
}
}
auto buffer = CreateHostBuffer(name, sizeof(vertices), vk::BufferUsageFlagBits::eVertexBuffer,
vertex_words);
const auto pipeline = [&](bool enabled, uint8_t front, uint8_t back,
bool provoking_last = false,
bool clockwise = false,
vk::PrimitiveTopology topology = vk::PrimitiveTopology::eTriangleList)
-> PipelineCache::Pipeline & {
HW::ModeControl mode{};
mode.face = clockwise;
mode.poly_mode = enabled;
mode.polymode_front_ptype = front;
mode.polymode_back_ptype = back;
mode.provoking_vtx_last = provoking_last;
registers.SetModeControl(mode);
return context.GetPipelineCache().GetGraphicsPipeline(
std::span{&color, 1u}, depth, std::span{&vertex, 1u}, scheduler.Current(), &pixel,
topology, false,
PipelineCache::GraphicsPrograms{{vertex_shader}, pixel_shader});
};
auto &filled = pipeline(true, 2, 2);
const auto draw = [&](const PipelineCache::Pipeline &selected, uint32_t vertex_count = 3,
std::array<u32, 4> clear_value = {}) {
RenderExecutorTestAccess::BindRenderTarget(executor, color.image_id);
if (depth.image_id) {
RenderExecutorTestAccess::BindRenderTarget(executor, depth.image_id);
}
auto bindings = RenderExecutorTestAccess::PrepareGraphicsBindings(
executor, vertex.stage, pixel.stage, true);
Require(name, "sampled attachment identity",
!depth_feedback || bindings.pixel->images[0].image_id == depth.image_id,
"the fragment must sample the depth attachment's native image");
auto &command = scheduler.Current();
vk::ImageAspectFlags feedback_aspects;
std::array<PreparedBindings *, 2> stages{
&bindings.vertex[0], &*bindings.pixel};
auto rendering = RenderExecutorTestAccess::AcquireRenderTargets(
executor, command, &color, 1, depth, stages, &feedback_aspects);
const bool feedback_enabled = rendering.depth_stencil_attachment.image_layout ==
vk::ImageLayout::eAttachmentFeedbackLoopOptimalEXT;
Require(name, "per-draw depth feedback",
feedback_enabled == (depth_feedback && depth.depth_write_enable) &&
feedback_aspects == (feedback_enabled
? vk::ImageAspectFlagBits::eDepth
: vk::ImageAspectFlags{}),
"feedback was not selected only for an overlapping fragment depth read/write");
RenderExecutorTestAccess::CommitBindings(
executor, command, selected, bindings.vertex[0], *bindings.pixel);
rendering.color_attachments[0].is_clear = true;
rendering.color_attachments[0].clear_value = clear_value;
command.BeginRendering(rendering);
auto cmd = command.Handle();
cmd.bindPipeline(vk::PipelineBindPoint::eGraphics, selected.pipeline);
const vk::Viewport viewport{0, 0, static_cast<float>(extent),
static_cast<float>(extent), 0, 1};
const vk::Rect2D scissor{{0, 0}, {extent, extent}};
cmd.setViewportWithCount(1, &viewport);
cmd.setScissorWithCount(1, &scissor);
cmd.setLineWidth(1);
cmd.setDepthTestEnable(depth.depth_test_enable);
cmd.setDepthWriteEnable(depth.depth_write_enable);
cmd.setDepthCompareOp(depth.depth_compare_op);
#if !defined(__APPLE__)
cmd.setDepthBoundsTestEnable(depth.depth_bounds_test_enable);
if (depth.depth_bounds_test_enable) {
cmd.setDepthBounds(registers.GetDepthBoundsMin(), registers.GetDepthBoundsMax());
}
#endif
cmd.setDepthBiasEnable(false);
cmd.setStencilTestEnable(depth.stencil_test_enable);
if (depth.stencil_test_enable) {
const auto set_stencil = [&](vk::StencilFaceFlagBits face,
const vk::StencilOpState &state) {
cmd.setStencilOp(face, state.failOp, state.passOp, state.depthFailOp,
state.compareOp);
cmd.setStencilCompareMask(face, state.compareMask);
cmd.setStencilWriteMask(face, state.writeMask);
cmd.setStencilReference(face, state.reference);
};
set_stencil(vk::StencilFaceFlagBits::eFront, depth.stencil_front);
set_stencil(vk::StencilFaceFlagBits::eBack, depth.stencil_back);
}
const vk::Bool32 write = true;
cmd.setColorWriteEnableEXT(1, &write);
cmd.setAttachmentFeedbackLoopEnableEXT(feedback_aspects);
const vk::DeviceSize offset = 0;
cmd.bindVertexBuffers(0, 1, &buffer.buffer, &offset);
cmd.draw(vertex_count, 1, 0, 0);
command.EndRendering();
RenderExecutorTestAccess::ResetBindings(executor);
};
const auto read_color = [&] {
return ReadCachedTexel(name, context, color.image_id, {}, {extent, extent, 1});
};
draw(filled);
const auto solid_pixels = read_color();
if (packed_vertex_color) {
const auto center = 4 * ((extent / 2) * extent + extent / 2);
for (uint32_t component = 0; component < packed_expected.size(); component++) {
Require(name, "packed vertex fetch and coverage",
std::abs(std::bit_cast<float>(solid_pixels[center + component]) -
packed_expected[component]) < 0.0001f && solid_pixels[component] == 0,
"packed vertex color changed channels, clamped HDR, or filled outside the triangle");
}
} else if (depth_feedback) {
// Consecutive draws in the same submission must see each earlier depth write.
draw(filled);
draw(filled);
depth.depth_write_enable = false;
draw(filled); // Reset feedback in the same command buffer and preserve the depth writes.
const auto repeated_pixels = read_color();
const auto stored_depth = ReadCachedTexel(
name, context, depth.image_id, {}, {extent, extent, 1});
for (uint32_t y = 0; y < extent; y++) {
for (uint32_t x = 0; x < extent; x++) {
const auto index = y * extent + x;
Require(name, "sampled and stored depth",
solid_pixels[index * 4] == std::bit_cast<uint32_t>(initial_depth(x, y)) &&
repeated_pixels[index * 4] ==
std::bit_cast<uint32_t>(initial_depth(x, y) + 0.375f) &&
stored_depth[index] == repeated_pixels[index * 4],
"depth feedback lost a prior write or the read-only draw changed depth");
}
}
depth.depth_test_enable = false;
for (uint32_t pass = 0; pass < 3; pass++) {
depth.depth_bounds_test_enable = pass != 2;
registers.SetDepthBoundsMin(pass == 0 ? 0.5f : 0.625f);
registers.SetDepthBoundsMax(pass == 0 ? 0.5625f : 0.6875f);
Require(name, "dynamic depth bounds pipeline reuse",
pipeline(true, 2, 2).pipeline == filled.pipeline,
"changing depth bounds or their enable state created another pipeline");
draw(filled);
const auto bounded_pixels = read_color();
for (size_t index = 0; index < stored_depth.size(); index++) {
const auto z = std::bit_cast<float>(stored_depth[index]);
const bool passes = !depth.depth_bounds_test_enable ||
(z >= registers.GetDepthBoundsMin() &&
z <= registers.GetDepthBoundsMax());
Require(name, "dynamic stored-depth bounds",
bounded_pixels[index * 4] == (passes ? stored_depth[index] : 0u),
"depth bounds did not filter stored depth independently of the depth test");
}
}
} else {
auto &wireframe = pipeline(true, 1, 1);
Require(name, "pipeline cache", filled.pipeline != wireframe.pipeline &&
pipeline(true, 2, 2).pipeline == filled.pipeline &&
pipeline(false, 1, 0).pipeline == filled.pipeline,
"polygon mode was lost from the cache key or dormant face modes were applied");
draw(wireframe);
const auto line_pixels = read_color();
const auto interior = 4 * (16 * extent + 16);
Require(name, "GPU coverage", solid_pixels[interior] == 0x3f800000u &&
line_pixels[interior] == 0 &&
std::ranges::any_of(line_pixels, [](u32 value) { return value == 0x3f800000u; }),
"wireframe did not preserve triangle edges while leaving its interior empty");
auto &last_vertex = pipeline(true, 2, 2, true);
Require(name, "provoking vertex pipeline cache",
last_vertex.pipeline != filled.pipeline &&
pipeline(true, 2, 2, true).pipeline == last_vertex.pipeline &&
pipeline(true, 2, 2).pipeline == filled.pipeline,
"first and last provoking vertices did not keep distinct cached pipelines");
draw(last_vertex);
const auto last_pixels = read_color();
for (size_t i = 0; i < solid_pixels.size(); i++) {
Require(name, "flat provoking vertex output",
last_pixels[i] == (solid_pixels[i] == 0 ? 0 : 0x3e800000u),
"last-vertex flat shading changed coverage or did not use vertex two");
}
const auto blend_pipeline = [&](bool enabled, bool bypass,
bool alternate = false) -> PipelineCache::Pipeline & {
auto blend = registers.GetBlendControl(0);
blend.enable = enabled;
blend.color_srcblend = static_cast<uint8_t>(alternate ? Prospero::BlendFactor::kOne
: Prospero::BlendFactor::kZero);
blend.alpha_srcblend = blend.color_srcblend;
blend.separate_alpha_blend = alternate;
registers.SetBlendControl(0, blend);
auto target = registers.GetRenderTarget(0).info;
target.blend_bypass = bypass;
registers.SetColorInfo(0, target);
return pipeline(true, 2, 2);
};
auto &disabled_blend = blend_pipeline(false, false);
Require(name, "effective blend pipeline cache",
blend_pipeline(false, true).pipeline == disabled_blend.pipeline &&
blend_pipeline(true, true).pipeline == disabled_blend.pipeline &&
blend_pipeline(false, false, true).pipeline == disabled_blend.pipeline &&
blend_pipeline(true, true, true).pipeline == disabled_blend.pipeline,
"disabled and bypassed blending did not share the same host pipeline");
auto &enabled_blend = blend_pipeline(true, false);
Require(name, "enabled blend pipeline cache",
enabled_blend.pipeline != disabled_blend.pipeline &&
blend_pipeline(true, false).pipeline == enabled_blend.pipeline,
"active blending was lost from the pipeline key or missed its cached pipeline");
auto dormant_alpha = registers.GetBlendControl(0);
dormant_alpha.alpha_srcblend = static_cast<uint8_t>(Prospero::BlendFactor::kOne);
registers.SetBlendControl(0, dormant_alpha);
Require(name, "dormant alpha blend pipeline cache",
pipeline(true, 2, 2).pipeline == enabled_blend.pipeline,
"inactive separate-alpha state recompiled the pipeline");
draw(enabled_blend);
const auto blended_pixels = read_color();
Require(name, "active blend output",
std::ranges::all_of(blended_pixels, [](u32 value) { return value == 0; }),
"zero-factor blending did not clear the fragment color");
draw(blend_pipeline(true, true));
Require(name, "bypassed blend output", read_color() == solid_pixels,
"blend bypass did not restore the unblended fragment color and coverage");
HW::DepthRenderTarget stencil_target{};
stencil_target.z_info.format = Prospero::DepthFormat::kZ32F;
stencil_target.stencil_info.format = Prospero::StencilFormat::k8UInt;
stencil_target.stencil_info.htile_stencil_disabled = true;
stencil_target.z_read_base_addr = stencil_target.z_write_base_addr = depth_address;
stencil_target.stencil_read_base_addr = stencil_target.stencil_write_base_addr =
depth_address + 0x20000;
stencil_target.size = {static_cast<uint16_t>(extent - 1),
static_cast<uint16_t>(extent - 1), true};
registers.SetDepthRenderTarget(stencil_target);
HW::DepthControl stencil_control{};
// Keep the same depth/stencil attachment active when stencil testing is disabled.
stencil_control.z_enable = true;
stencil_control.zfunc = static_cast<uint8_t>(vk::CompareOp::eAlways);
stencil_control.stencil_enable = stencil_control.backface_enable = true;
stencil_control.stencilfunc = stencil_control.stencilfunc_bf =
static_cast<uint8_t>(vk::CompareOp::eAlways);
registers.SetDepthControl(stencil_control);
// PPSA01325 sets bit 7 on front faces and clears it on back faces.
registers.SetStencilControl({3, 3, 3, 4, 4, 4});
registers.SetStencilMask({0x80, 0, 0x80, 1, 0x80, 0, 0x80, 1});
RenderExecutorTestAccess::ResolveRenderDepthTarget(executor, scheduler.Current(), depth);
auto stencil_readback = CreateHostBuffer(name, extent * extent,
vk::BufferUsageFlagBits::eTransferDst, {});
const auto read_stencil = [&] {
const vk::BufferImageCopy copy{0, 0, 0,
{vk::ImageAspectFlagBits::eStencil, 0, 0, 1}, {}, {extent, extent, 1}};
cache.GetImage(depth.image_id).Download(std::span{&copy, 1},
stencil_readback.buffer, 0, stencil_readback.size);
const vk::MemoryBarrier2 barrier{
.srcStageMask = vk::PipelineStageFlagBits2::eCopy,
.srcAccessMask = vk::AccessFlagBits2::eTransferWrite,
.dstStageMask = vk::PipelineStageFlagBits2::eHost,
.dstAccessMask = vk::AccessFlagBits2::eHostRead};
vk::DependencyInfo dependency{};
dependency.memoryBarrierCount = 1;
dependency.pMemoryBarriers = &barrier;
scheduler.Current().Handle().pipelineBarrier2(dependency);
scheduler.Finish();
return ReadBuffer(name, stencil_readback, extent * extent / 4);
};
const std::array stencil_pipelines{
pipeline(true, 2, 2, false, true).pipeline,
pipeline(true, 2, 2, false, false).pipeline};
const auto check_stencil = [&](const char *label, bool back_face,
uint8_t initial, uint8_t written) {
vk::ClearValue clear{};
clear.depthStencil = vk::ClearDepthStencilValue{0.625f, initial};
TextureCacheTestAccess::ClearImage(cache, scheduler.Current(), depth.image_id,
{vk::ImageAspectFlagBits::eDepth | vk::ImageAspectFlagBits::eStencil,
0, 1, 0, 1}, clear);
const auto &selected = pipeline(true, 2, 2, false, !back_face);
Require(name, "dynamic stencil pipeline reuse",
selected.pipeline == stencil_pipelines[back_face],
"stencil enable, operations or references changed the graphics pipeline");
draw(selected);
const auto result = read_stencil();
const auto *stencil_bytes = reinterpret_cast<const uint8_t *>(result.data());
for (uint32_t texel = 0; texel < extent * extent; ++texel) {
const uint8_t expected = solid_pixels[texel * 4] != 0 ? written : initial;
Require(name, label,
stencil_bytes[texel] == expected,
"captured stencil operations changed masked bits or triangle coverage");
}
};
check_stencil("front-face ReplaceTest", false, 0x35, 0xb5);
check_stencil("back-face masked ReplaceOp", true, 0xb5, 0x35);
stencil_control.stencil_enable = false;
registers.SetDepthControl(stencil_control);
RenderExecutorTestAccess::ResolveRenderDepthTarget(executor, scheduler.Current(), depth);
check_stencil("disabled front stencil", false, 0x35, 0x35);
check_stencil("disabled back stencil", true, 0xb5, 0xb5);
stencil_control.stencil_enable = true;
registers.SetDepthControl(stencil_control);
RenderExecutorTestAccess::ResolveRenderDepthTarget(executor, scheduler.Current(), depth);
check_stencil("re-enabled front stencil", false, 0x35, 0xb5);
check_stencil("re-enabled back stencil", true, 0xb5, 0x35);
// Frame 6115 uses separate test/op replacement values with Always comparisons.
stencil_control.z_enable = stencil_control.z_write_enable = true;
stencil_control.zfunc = static_cast<uint8_t>(vk::CompareOp::eLessOrEqual);
registers.SetDepthControl(stencil_control);
registers.SetStencilControl({0, 3, 9, 0, 4, 8});
registers.SetStencilMask({8, 0xff, 0xff, 1, 8, 0xff, 0xff, 1});
RenderExecutorTestAccess::ResolveRenderDepthTarget(executor, scheduler.Current(), depth);
Require(name, "independent stencil references",
depth.stencil_front.reference == 8 &&
depth.stencil_back.reference == 1,
"always-pass faces must preserve their distinct replacement values");
check_stencil("front test-value replacement", false, 0x35, 8);
check_stencil("back operation-value replacement", true, 0x35, 1);
stencil_control.zfunc = static_cast<uint8_t>(vk::CompareOp::eNever);
registers.SetDepthControl(stencil_control);
RenderExecutorTestAccess::ResolveRenderDepthTarget(executor, scheduler.Current(), depth);
check_stencil("front depth-fail subtract wrap", false, 0, 0xff);
check_stencil("back depth-fail add wrap", true, 0xff, 0);
// A compare on bit 3 and replacement on bit 0 share one Vulkan reference
// without changing either native operation or the untouched stencil bits.
stencil_control.z_enable = stencil_control.z_write_enable = false;
stencil_control.stencilfunc = stencil_control.stencilfunc_bf =
static_cast<uint8_t>(vk::CompareOp::eEqual);
registers.SetDepthControl(stencil_control);
registers.SetStencilControl({0, 4, 0, 0, 4, 0});
registers.SetStencilMask({8, 8, 1, 1, 8, 8, 1, 1});
RenderExecutorTestAccess::ResolveRenderDepthTarget(executor, scheduler.Current(), depth);
Require(name, "masked compare and replace reference",
depth.stencil_front.reference == 9 &&
depth.stencil_back.reference == 9,
"disjoint comparison and replacement bits must combine into one reference");
check_stencil("masked compare passes", false, 0xb8, 0xb9);
check_stencil("masked compare fails", false, 0xb0, 0xb0);
// SPI_PS_IN_CONTROL must select the native wave width through the actual
// program cache. In wave32 a low-word compare preserves scalar VCC_HI.
static const auto native_vertex = [] {
std::vector<u32> code;
AppendVMovLiteral(&code, 1, 0xbf800000u);
AppendVMovLiteral(&code, 2, 0x40400000u);
code.push_back(EncodeVopc(0xc2, InlineU32(1), 5));
code.push_back(EncodeVop2(0x01, 3, Vgpr(1), 2));
code.push_back(EncodeVopc(0xc2, InlineU32(2), 5));
code.push_back(EncodeVop2(0x01, 4, Vgpr(1), 2));
AppendVMovU32(&code, 0, 0);
AppendVMovLiteral(&code, 6, 0x3f800000u);
code.push_back(EncodeExp0(0x0c, 0xf));
code.push_back(EncodeExp1(3, 4, 0, 6));
for (u32 parameter = 0; parameter < 8; parameter++) {
code.push_back(EncodeExp0(0x20 + parameter, 0xf));
code.push_back(EncodeExp1(0, 0, 0, 0));
}
AppendEnd(&code);
return code;
}();
static const auto native_pixel = [] {
std::vector<u32> code;
AppendVMovU32(&code, 0, 0);
AppendSMovLiteral(&code, 107, 0x3e800000u);
code.push_back(EncodeVopc(0xc2, InlineU32(1), 0));
code.push_back(EncodeVop1(0x01, 1, 107));
AppendVMovLiteral(&code, 2, 0x3f800000u);
code.push_back(EncodeExp0(0x00, 0xf));
code.push_back(EncodeExp1(1, 1, 1, 2));
AppendEnd(&code);
return code;
}();
static ShaderUserData native_user_data{};
const auto vertex_address = reinterpret_cast<uint64_t>(native_vertex.data());
const auto pixel_address = reinterpret_cast<uint64_t>(native_pixel.data());
ShaderMapUserData(vertex_address,
{.type = Prospero::ShaderBinaryType::kGs,
.user_data = &native_user_data,
.code_size_bytes = static_cast<uint32_t>(native_vertex.size() * sizeof(u32))});
ShaderMapUserData(pixel_address,
{.type = Prospero::ShaderBinaryType::kPs,
.code_size_bytes = static_cast<uint32_t>(native_pixel.size() * sizeof(u32))});
HW::VertexShaderInfo native_vertex_regs{};
native_vertex_regs.es_regs.data_addr = vertex_address;
HW::PixelShaderInfo native_pixel_regs{};
native_pixel_regs.ps_regs.data_addr = pixel_address;
const auto owned_vertex_shader = vertex_shader;
const auto owned_pixel_shader = pixel_shader;
depth.stencil_test_enable = false;
registers.SetViewportTransformControl(0x300);
user_config.SetPrimitiveType(Prospero::PrimitiveType::kTriList);
std::array<Prospero::ColorComponentMapping, 8> export_mapping{};
std::array<ShaderVertexInputInfo, 3> native_vertex_info{};
std::array<uint64_t, 2> wave_ids{};
std::array<std::vector<u32>, 2> wave_keys;
struct PixelWaveCase { uint32_t control, width, color; };
for (const auto wave : {PixelWaveCase{0x0008, 64, 0},
PixelWaveCase{0x8008, 32, 0x3e800000},
PixelWaveCase{0x0008, 64, 0}}) {
registers.SetPsInControl(wave.control);
const auto programs = context.GetPipelineCache().GetGraphicsPrograms(
native_vertex_regs, native_pixel_regs, registers.GetShaderRegisters(),
registers, user_config, export_mapping, true, native_vertex_info, pixel);
Require(name, "pixel wave metadata propagation",
pixel.input_num == 8 && pixel.stage.program->wave_size == wave.width,
"SPI_PS_IN_CONTROL width did not reach the compiled pixel program");
const auto index = static_cast<uint32_t>(wave.width == 32);
std::vector<u32> key;
BuildStageStaticKey(pixel, key);
if (wave_ids[index] != 0) {
Require(name, "pixel wave cache reuse",
programs.pixel.id == wave_ids[index] && key == wave_keys[index],
"restoring wave64 did not reuse its original program");
}
wave_ids[index] = programs.pixel.id;
wave_keys[index] = std::move(key);
vertex_shader = programs.vertex[0];
pixel_shader = programs.pixel;
vertex = native_vertex_info[0];
draw(pipeline(true, 2, 2));
const auto wave_pixels = read_color();
for (size_t component = 0; component < wave_pixels.size(); component++) {
const auto expected = component % 4 == 3 ? 0x3f800000u : wave.color;
Require(name, "pixel wave VCC_HI readback", wave_pixels[component] == expected,
"pixel comparison clobbered VCC_HI in wave32 or retained it in wave64");
}
}
Require(name, "pixel wave cache distinction",
wave_ids[0] != wave_ids[1] && wave_keys[0] != wave_keys[1],
"wave32 and wave64 pixel programs shared a cache key");
static const auto waterfall_vertex = [&] {
std::vector<u32> code;
AppendVMovU32(&code, 9, 0);
// d3e1ae284c4dea3b reconstructs EXEC twice from packed NGG s3.
code.insert(code.end(), {0xbefe04c1u, 0x906a8803u, 0x81ea6a80u,
0x90fe6ac1u, 0x81ea0380u, 0x90fe6ac1u,
EncodeSop1(0x04, 10, 126)});
const auto loop = code.size();
code.push_back(EncodeSop1(0x14, 4, 10));
AppendVop3(&code, 0x360, 6, Vgpr(5), 4);
code.push_back(EncodeVopc(0xc2, 6, 5));
code.push_back(EncodeSop1(0x24, 18, 106));
AppendVMovLiteral(&code, 9, 0x3f800000u);
code.push_back(EncodeSop2(0x15, 10, 10, 106));
code.push_back(EncodeSop1(0x04, 126, 18));
code.push_back(EncodeSopp(0x05, static_cast<int16_t>(loop - code.size() - 1u)));
const auto base = code.size();
code.insert(code.end(), native_vertex.begin(), native_vertex.end());
const auto position = std::find(code.begin() + base, code.end(), EncodeExp0(0x0c, 0xf));
// Unprocessed vertices keep W=0, making waterfall progress observable.
*(position + 1) = EncodeExp1(3, 4, 0, 9);
return code;
}();
native_vertex_regs.es_regs.data_addr = reinterpret_cast<uint64_t>(waterfall_vertex.data());
ShaderMapUserData(native_vertex_regs.es_regs.data_addr,
{.type = Prospero::ShaderBinaryType::kGs, .user_data = &native_user_data,
.code_size_bytes = static_cast<u32>(waterfall_vertex.size() * sizeof(u32))});
registers.SetPsInControl(0x8008);
const auto saved_stages = registers.GetShaderStages();
for (const u32 wave : {32u, 64u}) {
registers.SetShaderStages((saved_stages & ~0x00400000u) |
(wave == 32u ? 0x00400000u : 0u));
const auto programs = context.GetPipelineCache().GetGraphicsPrograms(
native_vertex_regs, native_pixel_regs, registers.GetShaderRegisters(),
registers, user_config, export_mapping, true, native_vertex_info, pixel);
vertex_shader = programs.vertex[0];
pixel_shader = programs.pixel;
vertex = native_vertex_info[0];
Require(name, "native NGG waterfall wave",
vertex.stage.program->stage == ShaderType::Vertex &&
vertex.stage.program->wave_size == wave,
"NGG waterfall bypassed the native vertex path or lost its guest wave width");
for (const u32 count : {3u, 6u, 33u}) {
draw(pipeline(true, 2, 2), count);
const auto pixels = read_color();
for (size_t component = 0; component < pixels.size(); ++component)
Require(name, "NGG partial-wave waterfall output",
pixels[component] == (component % 4 == 3 ? 0x3f800000u : 0x3e800000u),
"native vertex waterfall did not terminate with every input vertex processed");
}
}
registers.SetShaderStages(saved_stages);
native_vertex_regs.es_regs.data_addr = vertex_address;
// Reversed RGBA targets must use logical Sa for RGB, with a separate
// zero or unit source factor for separate alpha attenuation or accumulation.
static const auto blend_pixel = [] {
std::vector<u32> code;
constexpr std::array<float, 4> source{0.8f, 0.6f, 0.4f, 0.5f};
for (u32 component = 0; component < source.size(); component++) {
AppendVMovLiteral(&code, component, std::bit_cast<u32>(source[component]));
}
code.push_back(EncodeExp0(0, 0xf));
code.push_back(EncodeExp1(0, 1, 2, 3));
AppendEnd(&code);
return code;
}();
native_pixel_regs.ps_regs.data_addr = reinterpret_cast<uint64_t>(blend_pixel.data());
ShaderMapUserData(native_pixel_regs.ps_regs.data_addr,
{.type = Prospero::ShaderBinaryType::kPs,
.code_size_bytes = static_cast<uint32_t>(blend_pixel.size() * sizeof(u32))});
const auto saved_blend = registers.GetBlendControl(0);
const auto saved_target_info = registers.GetRenderTarget(0).info;
const auto saved_output_mode = registers.GetShaderRegisters().target_output_mode[0];
registers.SetTargetOutputMode(0, 4);
auto blend_target_info = saved_target_info;
blend_target_info.blend_bypass = false;
registers.SetColorInfo(0, blend_target_info);
export_mapping[0] = color.export_mapping = Prospero::ColorMappingAbgr;
HW::BlendControl alpha_blend{};
alpha_blend.enable = true;
alpha_blend.color_srcblend = static_cast<uint8_t>(Prospero::BlendFactor::kSrcAlpha);
alpha_blend.alpha_srcblend = static_cast<uint8_t>(Prospero::BlendFactor::kOne);
alpha_blend.color_destblend = alpha_blend.alpha_destblend =
static_cast<uint8_t>(Prospero::BlendFactor::kOneMinusSrcAlpha);
struct AlphaBlendCase {
bool separate;
Prospero::BlendFactor alpha_source;
ShaderAlphaBlendSource mode;
float alpha;
};
uint64_t broadcast_program = 0;
vk::Pipeline broadcast_pipeline{};
for (const auto test : {
AlphaBlendCase{false, Prospero::BlendFactor::kOne, ShaderAlphaBlendSource::SourceAlpha, 0.4f},
AlphaBlendCase{true, Prospero::BlendFactor::kSrcAlpha, ShaderAlphaBlendSource::SourceAlpha, 0.4f},
AlphaBlendCase{true, Prospero::BlendFactor::kOne, ShaderAlphaBlendSource::SourceAlphaOne, 0.65f},
AlphaBlendCase{true, Prospero::BlendFactor::kZero, ShaderAlphaBlendSource::SourceAlphaZero, 0.15f}}) {
alpha_blend.separate_alpha_blend = test.separate;
alpha_blend.alpha_srcblend = static_cast<uint8_t>(test.alpha_source);
registers.SetBlendControl(0, alpha_blend);
const auto programs = context.GetPipelineCache().GetGraphicsPrograms(
native_vertex_regs, native_pixel_regs, registers.GetShaderRegisters(),
registers, user_config, export_mapping, true, native_vertex_info, pixel);
Require(name, "logical alpha blend program", pixel.alpha_blend_source == test.mode &&
pixel.dual_source_blending,
"the separate alpha equation did not reach the compiled pixel program");
vertex_shader = programs.vertex[0];
pixel_shader = programs.pixel;
vertex = native_vertex_info[0];
auto &selected = pipeline(true, 2, 2);
if (test.mode == ShaderAlphaBlendSource::SourceAlpha) {
Require(name, "equal blend equation cache reuse",
broadcast_program == 0 || (programs.pixel.id == broadcast_program &&
selected.pipeline == broadcast_pipeline),
"equivalent separate alpha state created another shader or pipeline");
broadcast_program = programs.pixel.id;
broadcast_pipeline = selected.pipeline;
}
constexpr std::array<float, 4> destination{0.3f, 0.3f, 0.1f, 0.2f};
draw(selected, 3, std::bit_cast<std::array<u32, 4>>(destination));
const auto blend_pixels = read_color();
const std::array<float, 4> expected{test.alpha, 0.35f, 0.35f, 0.5f};
for (size_t component = 0; component < blend_pixels.size(); component++) {
Require(name, "logical alpha blend readback",
std::abs(std::bit_cast<float>(blend_pixels[component]) -
expected[component % 4]) < 0.00001f,
"reversed blending changed logical RGB or the separate alpha equation");
}
}
native_pixel_regs.ps_regs.data_addr = pixel_address;
export_mapping[0] = color.export_mapping = {};
registers.SetBlendControl(0, saved_blend);
registers.SetColorInfo(0, saved_target_info);
registers.SetTargetOutputMode(0, saved_output_mode);
// Captured Playroom strips contain the fullscreen triangle followed by an
// out-of-bounds fetch exporting (0,0,0,0). Additive color reveals any extra triangle.
static constexpr std::array<std::array<u32, 4>, 4> fourth_positions{{
{0, 0, 0, 0}, {0x80000000u, 0, 0x80000000u, 0x80000000u},
{0, 0, 0, 0x3f800000u}, {0xbf800000u, 0xbf800000u, 0, 0}}};
static const auto position_shaders = [&] {
std::array<std::vector<u32>, fourth_positions.size()> result;
for (size_t i = 0; i < result.size(); i++) {
auto &code = result[i];
const auto position_export = std::ranges::find(native_vertex, EncodeExp0(0x0c, 0xf));
code.assign(native_vertex.begin(), position_export);
code.push_back(EncodeVopc(0xc2, InlineU32(3), 5));
constexpr std::array<u32, 4> position_registers{3, 4, 0, 6};
for (size_t component = 0; component < position_registers.size(); component++) {
AppendVMovLiteral(&code, 7, fourth_positions[i][component]);
const auto reg = position_registers[component];
code.push_back(EncodeVop2(0x01, reg, Vgpr(reg), 7));
}
code.insert(code.end(), position_export, native_vertex.end());
}
return result;
}();
auto additive_blend = registers.GetBlendControl(0);
additive_blend.enable = true;
additive_blend.color_srcblend = additive_blend.color_destblend =
additive_blend.alpha_srcblend = additive_blend.alpha_destblend =
static_cast<uint8_t>(Prospero::BlendFactor::kOne);
registers.SetBlendControl(0, additive_blend);
auto target_info = registers.GetRenderTarget(0).info;
target_info.blend_bypass = false;
registers.SetColorInfo(0, target_info);
registers.SetPsInControl(0x8008);
user_config.SetPrimitiveType(Prospero::PrimitiveType::kTriStrip);
for (size_t i = 0; i < position_shaders.size(); i++) {
const auto &code = position_shaders[i];
native_vertex_regs.es_regs.data_addr = reinterpret_cast<uint64_t>(code.data());
ShaderMapUserData(native_vertex_regs.es_regs.data_addr,
{.type = Prospero::ShaderBinaryType::kGs,
.user_data = &native_user_data,
.code_size_bytes = static_cast<uint32_t>(code.size() * sizeof(u32))});
const auto programs = context.GetPipelineCache().GetGraphicsPrograms(
native_vertex_regs, native_pixel_regs, registers.GetShaderRegisters(),
registers, user_config, export_mapping, true, native_vertex_info, pixel);
vertex_shader = programs.vertex[0];
pixel_shader = programs.pixel;
vertex = native_vertex_info[0];
draw(pipeline(true, 2, 2, false, false, vk::PrimitiveTopology::eTriangleStrip), 4);
const auto pixels = read_color();
bool second_triangle = false;
for (size_t component = 0; component < pixels.size(); component += 4) {
Require(name, "fullscreen triangle preserved",
pixels[component] == 0x3e800000u || pixels[component] == 0x3f000000u,
"clipping-error culling removed valid fullscreen coverage");
second_triangle |= pixels[component] == 0x3f000000u;
}
Require(name, "zero homogeneous position culling", second_triangle == (i >= 2),
"zero positions drew an extra triangle, or a valid position was culled");
}
// A NULL-export stencil pass must ignore smaller stale color targets,
// materialize the whole pending HTile clear, and still execute pixel discard.
static const auto null_pixels = [] {
std::array<std::vector<u32>, 2> result;
for (uint32_t discard = 0; discard < result.size(); discard++) {
auto &code = result[discard];
if (discard != 0) {
code.push_back(EncodeSop1(0x04, 126, InlineU32(0)));
}
code.push_back(EncodeExp0(0x09, 0, true, false, true));
code.push_back(EncodeExp1(0, 0, 0, 0));
AppendEnd(&code);
}
return result;
}();
registers.Reset();
registers.SetViewportTransformControl(0x300);
registers.SetViewportScaleOffset(0, extent / 2, extent / 2,
extent / 2, extent / 2, 1, 0);
registers.SetViewportZMax(0, 1);
registers.SetScreenScissor(0, 0, extent, extent);
registers.SetWindowScissor(0, 0, extent, extent, false);
registers.SetGenericScissor(0, 0, extent, extent, false);
registers.SetViewportScissor(0, 0, 0, extent, extent, false);
registers.SetRenderTargetMask(0xf00f);
registers.SetShaderMask(0xf00f);
registers.SetPsInControl(0x8000);
registers.SetDepthShaderControl({.shader_kill_enable = true});
for (const auto slot : {0u, 3u}) {
registers.SetColorBase(slot,
{.addr = depth_address + (slot == 0 ? 0x38000 : 0x10000)});
registers.SetColorInfo(slot,
{.format = Prospero::ChannelLayout::k32_32_32_32,
.channel_type = Prospero::ChannelType::kFloat,
.channel_order = Prospero::ChannelOrder::kStandard});
const auto side = slot == 0 ? extent / 4 : extent;
registers.SetColorAttrib2(slot, {.height = side - 1, .width = side - 1});
registers.SetColorAttrib3(slot,
{.tile_mode = Prospero::TileMode::kLinear, .dimension = 1});
}
registers.SetTargetOutputMode(0, 4);
registers.SetTargetOutputMode(1, 4);
stencil_target.z_info.htile_acceleration = true;
stencil_target.htile_data_base_addr = depth_address + 0x30000;
registers.SetDepthRenderTarget(stencil_target);
registers.SetDepthClearValue(1);
stencil_control = {};
stencil_control.stencil_enable = true;
stencil_control.stencilfunc = static_cast<uint8_t>(vk::CompareOp::eAlways);
registers.SetDepthControl(stencil_control);
registers.SetStencilControl({3, 3, 3, 3, 3, 3});
registers.SetStencilMask({0x80, 0, 0x80, 1, 0x80, 0, 0x80, 1});
user_config.SetPrimitiveType(Prospero::PrimitiveType::kTriList);
shaders.SetEsShaderBase(vertex_address);
for (uint32_t discard = 0; discard < null_pixels.size(); discard++) {
const auto &code = null_pixels[discard];
const auto address = reinterpret_cast<uint64_t>(code.data());
ShaderMapUserData(address,
{.type = Prospero::ShaderBinaryType::kPs,
.code_size_bytes = static_cast<uint32_t>(code.size() * sizeof(u32))});
shaders.SetPsShaderBase(address);
RenderExecutorTestAccess::ResolveRenderDepthTarget(executor, scheduler.Current(), depth);
(void)cache.FindDepthTarget(depth.image_id, depth.desc);
vk::ClearValue zero{};
TextureCacheTestAccess::ClearImage(cache, scheduler.Current(), depth.image_id,
{vk::ImageAspectFlagBits::eDepth | vk::ImageAspectFlagBits::eStencil,
0, 1, 0, 1}, zero);
Require(name, "pending stencil-pass HTile clear",
cache.ClearMeta(stencil_target.htile_data_base_addr),
"the stencil pass did not retain its HTile metadata");
RenderExecutorTestAccess::DrawAuto(
executor, scheduler.Current(), {.vertex_count = 3, .instance_count = 1});
const auto depth_pixels = ReadCachedTexel(name, context, depth.image_id,
{}, {extent, extent, 1});
Require(name, "NULL-export full depth clear",
std::ranges::all_of(depth_pixels, [](u32 v) { return v == 0x3f800000u; }),
"an unexported color target clipped the pending depth clear");
const auto stencil = read_stencil();
const auto expected = discard == 0 ? 0x80808080u : 0u;
Require(name, "NULL-export stencil coverage and discard",
std::ranges::all_of(stencil, [=](u32 v) { return v == expected; }),
"a stale color target clipped stencil coverage or the NULL-export shader was skipped");
}
// A depth-only draw retains the previous PS address and write masks, but
// SPI_SHADER_COL_FORMAT disables its exports. Compiling this stale PS
// with the new zero-input metadata would fail before the depth draw.
static const auto stale_pixel = [] {
std::vector<u32> code{EncodeVintrp(0, 0, 0, 0, 0),
EncodeVintrp(1, 0, 0, 0, 1),
EncodeExp0(0, 0xf), EncodeExp1(0, 0, 0, 0)};
AppendEnd(&code);
return code;
}();
const auto stale_address = reinterpret_cast<uint64_t>(stale_pixel.data());
ShaderMapUserData(stale_address,
{.type = Prospero::ShaderBinaryType::kPs,
.code_size_bytes = static_cast<u32>(stale_pixel.size() * sizeof(u32))});
shaders.SetPsShaderBase(stale_address);
registers.SetPsInControl(0);
registers.SetDepthShaderControl({});
registers.SetDepthControl({.z_enable = true, .z_write_enable = true,
.zfunc = static_cast<uint8_t>(vk::CompareOp::eAlways)});
for (const auto slot : {0u, 1u}) {
registers.SetTargetOutputMode(slot, 0);
}
RenderExecutorTestAccess::DrawAuto(
executor, scheduler.Current(), {.vertex_count = 3, .instance_count = 1});
const auto depth_only_pixels = ReadCachedTexel(
name, context, depth.image_id, {}, {extent, extent, 1});
Require(name, "disabled stale PS preserves fixed-function depth",
std::ranges::all_of(depth_only_pixels, [](u32 v) { return v == 0; }),
"disabled color exports invoked the stale PS or clipped fixed-function depth");
registers.SetPsInControl(0x8000);
registers.SetTargetOutputMode(0, 4);
// Export ordinals follow CB_SHADER_MASK. A disabled CB_TARGET_MASK slot
// still consumes an ordinal, and stale MRT0 must not clip physical MRT3.
static const auto sparse_pixels_code = [] {
std::array<std::vector<u32>, 2> result{native_pixel, native_pixel};
auto &code = result[1];
const auto exp = std::ranges::find(code, EncodeExp0(0, 0xf));
*exp = EncodeExp0(0, 0xf, false);
*(exp + 1) = EncodeExp1(0, 0, 0, 0);
code.insert(exp + 2, {EncodeExp0(1, 0xf), EncodeExp1(1, 1, 1, 2)});
return result;
}();
registers.SetRenderTargetMask(0xf000);
registers.SetDepthControl({});
registers.SetDepthShaderControl({});
RenderColorInfo sparse_color{};
for (uint32_t index = 0; index < sparse_pixels_code.size(); index++) {
const auto &code = sparse_pixels_code[index];
const auto address = reinterpret_cast<uint64_t>(code.data());
ShaderMapUserData(address,
{.type = Prospero::ShaderBinaryType::kPs,
.code_size_bytes = static_cast<uint32_t>(code.size() * sizeof(u32))});
shaders.SetPsShaderBase(address);
registers.SetShaderMask(index == 0 ? 0xf000 : 0xf00f);
registers.SetTargetOutputMode(1, index == 0 ? 0 : 4);
RenderExecutorTestAccess::ResolveRenderColorTarget(
executor, scheduler.Current(), sparse_color, 3);
TextureCacheTestAccess::ClearImage(cache, scheduler.Current(), sparse_color.image_id,
{vk::ImageAspectFlagBits::eColor, 0, 1, 0, 1}, {});
RenderExecutorTestAccess::DrawAuto(
executor, scheduler.Current(), {.vertex_count = 3, .instance_count = 1});
const auto pixels = ReadCachedTexel(name, context, sparse_color.image_id,
{}, {extent, extent, 1});
for (size_t component = 0; component < pixels.size(); component++) {
const auto expected = component % 4 == 3 ? 0x3f800000u : 0x3e800000u;
Require(name, "compact export to sparse MRT3", pixels[component] == expected,
"shader output routing used the write mask or clipped MRT3 with stale MRT0");
}
}
static const auto snorm_pixel = [] {
std::vector<u32> code;
AppendVMovLiteral(&code, 0, 0xc0008000u);
AppendVMovLiteral(&code, 1, 0x7fff4000u);
code.insert(code.end(), {EncodeExp0(0, 0xf, true, true), EncodeExp1(0, 1, 0, 0)});
AppendEnd(&code);
return code;
}();
const auto snorm_address = reinterpret_cast<uint64_t>(snorm_pixel.data());
ShaderMapUserData(snorm_address,
{.type = Prospero::ShaderBinaryType::kPs,
.code_size_bytes = static_cast<uint32_t>(snorm_pixel.size() * sizeof(u32))});
shaders.SetPsShaderBase(snorm_address);
registers.SetShaderMask(0xf000);
registers.SetTargetOutputMode(0, 6);
registers.SetTargetOutputMode(1, 0);
RenderExecutorTestAccess::DrawAuto(
executor, scheduler.Current(), {.vertex_count = 3, .instance_count = 1});
const auto snorm_pixels = ReadCachedTexel(name, context, sparse_color.image_id,
{}, {extent, extent, 1});
constexpr std::array<float, 4> snorm_expected{-1, -16384.0f / 32767,
16384.0f / 32767, 1};
for (size_t component = 0; component < snorm_pixels.size(); component++) {
Require(name, "compressed SNORM16 color export",
std::abs(std::bit_cast<float>(snorm_pixels[component]) -
snorm_expected[component % 4]) < 0.00001f,
"SNORM16 export lost its sign, normalization or negative endpoint clamp");
}
registers.SetTargetOutputMode(0, 4);
// Indirect arguments also reach s8/s9. Native VS prologs must add them
// exactly once, for both indexed and automatically generated indices.
static const auto indirect_vertex = [&] {
auto code = native_vertex;
for (auto &word : code) {
if (word == EncodeVopc(0xc2, InlineU32(1), 5))
word = EncodeVopc(0xc2, InlineU32(8), 5);
else if (word == EncodeVopc(0xc2, InlineU32(2), 5))
word = EncodeVopc(0xc2, InlineU32(9), 5);
}
code.insert(code.begin(), {EncodeVop2(0x25, 5, 8, 5),
EncodeVop2(0x25, 8, 9, 8)});
const auto position = std::ranges::find(code, EncodeExp0(0x0c, 0xf));
code.insert(position, {EncodeVopc(0xc2, InlineU32(5), 8),
EncodeVop2(0x01, 3, Vgpr(1), 3),
EncodeVop2(0x01, 4, Vgpr(1), 4)});
return code;
}();
const auto indirect_vs = reinterpret_cast<uint64_t>(indirect_vertex.data());
ShaderMapUserData(indirect_vs,
{.type = Prospero::ShaderBinaryType::kGs, .user_data = &native_user_data,
.code_size_bytes = static_cast<uint32_t>(indirect_vertex.size() * sizeof(u32))});
shaders.SetEsShaderBase(indirect_vs);
shaders.SetPsShaderBase(pixel_address);
shaders.SetGsShaderResource2({.user_sgpr = 2});
shaders.SetGsUserSgpr(0, 7, HW::UserSgprType::Unknown);
shaders.SetGsUserSgpr(1, 5, HW::UserSgprType::Unknown);
user_config.SetPrimitiveType(Prospero::PrimitiveType::kTriList);
for (const bool indexed : {false, true}) {
TextureCacheTestAccess::ClearImage(cache, scheduler.Current(), sparse_color.image_id,
{vk::ImageAspectFlagBits::eColor, 0, 1, 0, 1}, {});
if (indexed) {
RenderExecutorTestAccess::DrawIndex(executor, scheduler.Current(),
{.index_count = 3, .index_addr = reinterpret_cast<const void *>(index_address),
.instance_count = 1, .index_type_and_size = 1, .base_vertex = 7, .first_instance = 5,
.offset_source = DrawOffsetSource::IndirectArgs});
} else {
RenderExecutorTestAccess::DrawAuto(executor, scheduler.Current(),
{.vertex_count = 3, .instance_count = 1, .first_vertex = 7, .first_instance = 5,
.offset_source = DrawOffsetSource::IndirectArgs});
}
const auto pixels = ReadCachedTexel(name, context, sparse_color.image_id,
{}, {extent, extent, 1});
for (size_t component = 0; component < pixels.size(); component++) {
Require("NativeIndirectDrawOffsets", indexed ? "indexed" : "auto",
pixels[component] == (component % 4 == 3 ? 0x3f800000u : 0x3e800000u),
"native vertex or instance offset was applied twice");
}
}
// A fourth VS invocation reads beyond the descriptor instead of reconstructing the corner.
const ShaderBufferResource rect_buffer{{
static_cast<u32>(rect_address),
static_cast<u32>(rect_address >> 32u) | (32u << 16u), 3,
DstSel(4, 5, 6, 7) |
(static_cast<u32>(Prospero::BufferFormat::k32Float) << 12u)}};
static const auto rect_vertex_code = [] {
std::vector<u32> code{
EncodeMubuf0(0x0d, 0, true, false), EncodeMubuf1(0, 2, 5),
EncodeMubuf0(0x0d, 16, true, false), EncodeMubuf1(2, 2, 5)};
AppendVMovU32(&code, 4, 0);
AppendVMovLiteral(&code, 6, 0x3f800000u);
code.insert(code.end(), {EncodeExp0(0x0c, 0xf), EncodeExp1(0, 1, 4, 6),
EncodeExp0(0x20, 0xf), EncodeExp1(2, 3, 4, 6)});
AppendEnd(&code);
return code;
}();
static const auto rect_pixel_code = [] {
std::vector<u32> code;
for (u32 component = 0; component < 2; component++) {
code.push_back(EncodeVintrp(0, 20 + component, 0, component, 0));
code.push_back(EncodeVintrp(1, 20 + component, 0, component, 1));
}
AppendVMovU32(&code, 22, 0);
AppendVMovLiteral(&code, 23, 0x3f800000u);
code.insert(code.end(), {EncodeExp0(0, 0xf), EncodeExp1(20, 21, 22, 23)});
AppendEnd(&code);
return code;
}();
const auto rect_vs = reinterpret_cast<uint64_t>(rect_vertex_code.data());
const auto rect_ps = reinterpret_cast<uint64_t>(rect_pixel_code.data());
ShaderMapUserData(rect_vs,
{.type = Prospero::ShaderBinaryType::kGs, .user_data = &native_user_data,
.code_size_bytes = static_cast<u32>(rect_vertex_code.size() * sizeof(u32))});
ShaderMapUserData(rect_ps,
{.type = Prospero::ShaderBinaryType::kPs,
.code_size_bytes = static_cast<u32>(rect_pixel_code.size() * sizeof(u32))});
shaders.SetEsShaderBase(rect_vs);
shaders.SetPsShaderBase(rect_ps);
shaders.SetGsShaderResource2({.user_sgpr = 4});
for (u32 i = 0; i < 4; i++) {
shaders.SetGsUserSgpr(i, rect_buffer.fields[i], HW::UserSgprType::Vsharp);
}
registers.SetPsInControl(0x8001);
registers.SetPsInputEna(2);
registers.SetPsInputAddr(2);
registers.SetPsInputSettings(0, 0);
for (const auto primitive : {Prospero::PrimitiveType::kRectList,
Prospero::PrimitiveType::kRectListLegacy}) {
const char *rect_name = primitive == Prospero::PrimitiveType::kRectList
? "RectListThreeRecords" : "LegacyRectListThreeRecords";
user_config.SetPrimitiveType(primitive);
registers.SetModeControl({});
TextureCacheTestAccess::ClearImage(cache, scheduler.Current(), sparse_color.image_id,
{vk::ImageAspectFlagBits::eColor, 0, 1, 0, 1}, {});
RenderExecutorTestAccess::DrawAuto(
executor, scheduler.Current(), {.vertex_count = 3, .instance_count = 1});
const auto pixels = ReadCachedTexel(name, context, sparse_color.image_id,
{}, {extent, extent, 1});
for (u32 y = 0; y < extent; y++) {
for (u32 x = 0; x < extent; x++) {
const bool inside = x >= 4 && x < 28 && y >= 4 && y < 28;
const auto offset = 4 * (y * extent + x);
const std::array<float, 4> expected{
inside ? (x + 0.5f - 4) / 24 : 0,
inside ? (y + 0.5f - 4) / 24 : 0, 0, inside ? 1.0f : 0};
for (u32 component = 0; component < 4; component++) {
Require(rect_name, "three-vertex rectangle coverage and UV",
std::abs(std::bit_cast<float>(pixels[offset + component]) -
expected[component]) < 0.0001f,
"rectangle reconstruction lost its fourth corner or interpolated UVs");
}
}
}
}
// Captured format30 uses uncompressed mode9 at MRT2. Keep a blended
// floating-point MRT live beside it, including sparse export routing.
constexpr uint64_t packed_address = depth_address + 0x28000;
static const auto packed_pixels = [] {
std::array<std::vector<u32>, 2> result;
constexpr std::array<std::array<u32, 3>, 2> inputs{{
{0x3aa01403u, 0x3ae01c04u, 0x3c282a0bu}, // just above 2.5/3.5/10.5 LSB
{0xbf800000u, 0x40000000u, 0x3e800000u}}};
for (u32 i = 0; i < result.size(); ++i) {
auto &code = result[i];
for (u32 component = 0; component < 3; ++component)
AppendVMovLiteral(&code, component, inputs[i][component]);
AppendVMovLiteral(&code, 4, 0x38004400u); // FP16 RG: 4, 0.5
AppendVMovLiteral(&code, 5, 0x4000bc00u); // FP16 BA: -1, 2
for (u32 ordinal = 0; ordinal < 7; ++ordinal) {
code.push_back(EncodeExp0(ordinal, 0xf, ordinal == 6, ordinal != 2));
code.push_back(ordinal == 2 ? EncodeExp1(0, 1, 2, 0) : EncodeExp1(4, 5, 0, 0));
}
AppendEnd(&code);
}
return result;
}();
TestCase packed_read;
packed_read.name = "PackedUnormRenderTargetRead";
packed_read.has_user_data = true;
packed_read.image_descriptor_swizzle = DstSel(4, 5, 6, 1);
const ShaderTextureResource packed_descriptor{{static_cast<u32>(packed_address >> 8u),
(static_cast<u32>(Prospero::BufferFormat::k11_11_10UNorm) << 20u) | (3u << 30u),
((extent - 1u) >> 2u) | ((extent - 1u) << 14u),
packed_read.image_descriptor_swizzle |
(static_cast<u32>(Prospero::ImageType::kColor2D) << 28u),
0, 0x00700000u, 0, 0}};
std::copy_n(packed_descriptor.fields, 8, packed_read.user_data.begin());
packed_read.user_data[8] = 0x92u;
packed_read.user_data[9] = 0x00fff000u;
packed_read.user_data[10] = 0x0a500000u; // Existing packed-UNorm bilinear clamp contract.
packed_read.user_data[50] = 8 * sizeof(u32);
AppendVMovU32(&packed_read.code, 20, extent / 2);
AppendVMovU32(&packed_read.code, 21, extent / 2);
packed_read.code.insert(packed_read.code.end(),
{EncodeMimg0(0x00, 0xf), EncodeMimg1(0, 20)});
AppendVMovLiteral(&packed_read.code, 20, 0x3f000000u);
AppendVMovLiteral(&packed_read.code, 21, 0x3f000000u);
packed_read.code.insert(packed_read.code.end(),
{EncodeMimg0(0x27, 0xf), EncodeMimg1(4, 20, 0, 2)});
for (u32 component = 0; component < 8; ++component)
AppendStoreVgpr(&packed_read.code, component, component);
AppendEnd(&packed_read.code);
const auto packed_reader = CompileCase(packed_read, SubgroupSize());
auto packed_output = CreateStorageBuffer(name, {}, 8);
const auto packed_sampler = CreateSampler(name);
uint64_t half_up_program = 0, truncate_program = 0;
struct PackedDraw { u32 slot, wave; bool truncate, clamp; };
for (const auto test : {PackedDraw{2, 64, false, false}, {2, 64, true, false},
{2, 64, false, false}, {3, 32, false, false},
{3, 32, false, true}}) {
const u32 float_slot = test.slot == 2 ? 6 : 7;
registers.Reset();
registers.SetViewportTransformControl(0x300);
registers.SetViewportScaleOffset(0, extent / 2, extent / 2,
extent / 2, extent / 2, 1, 0);
registers.SetViewportZMax(0, 1);
registers.SetScreenScissor(0, 0, extent, extent);
registers.SetWindowScissor(0, 0, extent, extent, false);
registers.SetGenericScissor(0, 0, extent, extent, false);
registers.SetViewportScissor(0, 0, 0, extent, extent, false);
registers.SetRenderTargetMask((7u << (4u * test.slot)) | (0xfu << (4u * float_slot)));
registers.SetShaderMask(test.slot == 2 ? 0x0fffffffu : 0xfffff0ffu);
registers.SetPsInControl(test.wave == 32 ? 0x8000 : 0);
constexpr std::array<u32, 7> modes{4, 4, 9, 5, 4, 4, 4};
for (u32 ordinal = 0; ordinal < modes.size(); ++ordinal)
registers.SetTargetOutputMode(ordinal, modes[ordinal]);
for (const auto slot : {test.slot, float_slot}) {
const bool packed = slot == test.slot;
registers.SetColorBase(slot, {.addr = packed ? packed_address : depth_address + 0x10000});
registers.SetColorInfo(slot,
{.blend_clamp = packed, .round_mode = packed && test.truncate,
.format = packed ? Prospero::ChannelLayout::k11_11_10
: Prospero::ChannelLayout::k32_32_32_32,
.channel_type = packed ? Prospero::ChannelType::kUNorm : Prospero::ChannelType::kFloat,
.channel_order = Prospero::ChannelOrder::kStandard});
registers.SetColorAttrib2(slot, {.height = extent - 1, .width = extent - 1});
registers.SetColorAttrib3(slot, {.tile_mode = Prospero::TileMode::kLinear, .dimension = 1});
}
HW::BlendControl additive{};
additive.enable = true;
additive.color_srcblend = additive.color_destblend =
additive.alpha_srcblend = additive.alpha_destblend =
static_cast<uint8_t>(Prospero::BlendFactor::kOne);
registers.SetBlendControl(float_slot, additive);
const auto &code = packed_pixels[test.clamp];
const auto ps_address = reinterpret_cast<uint64_t>(code.data());
ShaderMapUserData(ps_address, {.type = Prospero::ShaderBinaryType::kPs,
.code_size_bytes = static_cast<u32>(code.size() * sizeof(u32))});
shaders.SetEsShaderBase(vertex_address);
shaders.SetGsShaderResource2({});
shaders.SetPsShaderBase(ps_address);
user_config.SetPrimitiveType(Prospero::PrimitiveType::kTriList);
native_vertex_regs.es_regs.data_addr = vertex_address;
native_pixel_regs.ps_regs.data_addr = ps_address;
const auto programs = context.GetPipelineCache().GetGraphicsPrograms(
native_vertex_regs, native_pixel_regs, registers.GetShaderRegisters(),
registers, user_config, export_mapping, true, native_vertex_info, pixel);
Require(name, "packed format follows export ordinal",
pixel.target_output_mode[2] == 9 && pixel.wave_size == test.wave &&
pixel.target_packed_unorm_mask == 4u &&
pixel.target_round_mode == (test.truncate ? 4u : 0u),
"packed conversion or rounding contaminated another MRT or followed a physical slot");
if (test.slot == 2) {
auto &id = test.truncate ? truncate_program : half_up_program;
Require(name, "packed rounding program reuse", id == 0 || id == programs.pixel.id,
"restoring half-up rounding failed to reuse its program");
id = programs.pixel.id;
}
RenderColorInfo packed_target{}, float_target{};
RenderExecutorTestAccess::ResolveRenderColorTarget(
executor, scheduler.Current(), packed_target, test.slot);
RenderExecutorTestAccess::ResolveRenderColorTarget(
executor, scheduler.Current(), float_target, float_slot);
Require(name, "packed target storage", packed_target.desc.info.bytes_per_block == 4 &&
packed_target.desc.info.pixel_format == vk::Format::eR32Uint,
"packed target expanded its physical storage or used floating-point bit encoding");
vk::ClearValue packed_clear{}, float_clear{};
packed_clear.color.uint32[0] = 0xdeadbeefu;
float_clear.color.float32 = std::array<float, 4>{-2.f, 0.25f, 3.f, 0.5f};
const vk::ImageSubresourceRange range{vk::ImageAspectFlagBits::eColor, 0, 1, 0, 1};
TextureCacheTestAccess::ClearImage(cache, scheduler.Current(), packed_target.image_id, range, packed_clear);
TextureCacheTestAccess::ClearImage(cache, scheduler.Current(), float_target.image_id, range, float_clear);
RenderExecutorTestAccess::DrawAuto(
executor, scheduler.Current(), {.vertex_count = 3, .instance_count = 1});
const std::array<u32, 3> channels = test.clamp ? std::array<u32, 3>{0, 2047, 256}
: test.truncate ? std::array<u32, 3>{2, 3, 10} : std::array<u32, 3>{3, 4, 11};
const u32 word = channels[0] | (channels[1] << 11u) | (channels[2] << 22u);
Require(name, "packed MRT words",
ReadCachedTexel(name, context, packed_target.image_id, {}, {extent, extent, 1}) ==
std::vector<u32>(extent * extent, word),
"mode9 export lost packed widths, half-up/truncate rounding, clamp or RGB write mask");
const auto floats = ReadCachedTexel(name, context, float_target.image_id, {}, {extent, extent, 1});
constexpr std::array<float, 4> blended{2.f, 0.75f, 2.f, 2.5f};
for (size_t i = 0; i < floats.size(); ++i)
Require(name, "unconverted MRT blending", floats[i] == std::bit_cast<u32>(blended[i % 4]),
"packed MRT handling changed another target's floating-point export or blending");
ShaderRecompiler::IR::DescriptorValue value{};
value.dword_count = 8;
std::copy_n(packed_descriptor.fields, 8, value.dwords.begin());
const auto binding = RenderExecutorTestAccess::ResolveTexture(
executor, packed_reader.program.info.images.at(0), value);
Require(name, "packed target sampled alias", binding.image_id == packed_target.image_id,
"sampling the packed target allocated a different image or lost the rendered data");
Image sampled;
sampled.view = cache.FindTexture(binding.image_id, binding.desc);
sampled.layout = cache.GetImage(binding.image_id).backing.state.layout;
scheduler.Finish();
Dispatch(packed_read, packed_reader, packed_output, nullptr, &sampled,
nullptr, nullptr, packed_sampler);
const auto loaded = ReadBuffer(name, packed_output, 8);
for (u32 i = 0; i < loaded.size(); ++i) {
const auto component = i % 4;
const float expected = component == 3 ? 1.f
: channels[component] / (component == 2 ? 1023.f : 2047.f);
Require(name, "rendered packed load and sample",
std::abs(std::bit_cast<float>(loaded[i]) - expected) < 0.000001f,
"packed rendered contents lost normalization, ordering or texture visibility");
}
}
Require(name, "packed rounding program distinction", half_up_program != truncate_program,
"half-up and truncate exports shared one shader specialization");
DestroyBuffer(&packed_output);
m_device.destroySampler(packed_sampler);
DestroyBuffer(&stencil_readback);
vertex_shader = owned_vertex_shader;
pixel_shader = owned_pixel_shader;
}
resources.UnmapMemory(depth_address, allocation_size);
scheduler.Finish();
DestroyBuffer(&buffer);
m_device.destroyShaderModule(pixel_shader.module);
m_device.destroyShaderModule(vertex_shader.module);
Require(name, "release depth memory",
Libs::LibKernel::Memory::KernelMunmap(depth_address, allocation_size) == 0 &&
Libs::LibKernel::Memory::KernelReleaseDirectMemory(
direct_offset, allocation_size) == 0,
"depth mapping or allocation release failed");
std::printf("[gpu] %-32s ok\n", name);
}
std::vector<u32> RenderFragment(const GraphicsCase &test,
const CompiledShader &fragment) {
const bool object_id = !test.object_ids.empty();
const auto vertex_spirv =
TestSpv::MakePassthroughVertexSpirv(test.layers > 1, test.vertex_clip_w);
ValidateSpirv(test.name, vertex_spirv);
Image target =
CreateImageMips(test.name, test.width, 1, vk::Format::eR32G32B32A32Sfloat,
vk::ImageUsageFlagBits::eColorAttachment, {}, 4,
vk::ImageLayout::eGeneral, vk::ImageType::e2D,
test.layers > 1 ? vk::ImageViewType::e2DArray : vk::ImageViewType::e2D,
test.layers, 0, 0, test.samples);
Image resolved;
if (test.samples != vk::SampleCountFlagBits::e1) {
resolved = CreateImageMips(
test.name, test.width, 1, target.format, vk::ImageUsageFlagBits::eColorAttachment,
{}, 4, vk::ImageLayout::eGeneral, vk::ImageType::e2D,
test.layers > 1 ? vk::ImageViewType::e2DArray : vk::ImageViewType::e2D,
test.layers);
}
Image primitive_target;
if (object_id) {
primitive_target = CreateImageMips(
test.name, test.width, 1, vk::Format::eR32Uint,
vk::ImageUsageFlagBits::eColorAttachment, {}, 1,
vk::ImageLayout::eGeneral, vk::ImageType::e2D,
test.layers > 1 ? vk::ImageViewType::e2DArray : vk::ImageViewType::e2D,
test.layers);
}
const std::vector<u32> default_vertices = {
0xbf800000u, 0xbf800000u, 0x3e800000u, 0x3f000000u, 0x3f400000u,
0x3f800000u, 0x40400000u, 0xbf800000u, 0x3e800000u, 0x3f000000u,
0x3f400000u, 0x3f800000u, 0xbf800000u, 0x40400000u, 0x3e800000u,
0x3f000000u, 0x3f400000u, 0x3f800000u,
};
const auto &vertices =
test.vertices.empty() ? default_vertices : test.vertices;
Require(test.name, "graphics", !vertices.empty() && vertices.size() % 18u == 0,
"graphics vertex buffer must contain pos2/color4 triangle lists");
auto vertex_buffer =
CreateHostBuffer(test.name, vertices.size() * sizeof(u32),
vk::BufferUsageFlagBits::eVertexBuffer, vertices);
vk::ShaderModule vertex_module = CreateShaderModule(test.name, vertex_spirv);
vk::ShaderModule fragment_module = CreateShaderModule(test.name, fragment.spirv);
const auto &fragment_bind = fragment.program.bindings;
vk::PushConstantRange push_constant_range{};
if (fragment_bind.UsesPushData()) {
Require(test.name, "graphics",
test.push_constants.size() * sizeof(u32) ==
fragment_bind.ShaderDataDwords() * sizeof(u32),
"fragment push constant data size does not match reflection");
}
if (fragment_bind.UsesPushData() || object_id) {
push_constant_range.stageFlags = vk::ShaderStageFlagBits::eFragment;
push_constant_range.size = ShaderRecompiler::IR::NativePushConstantSize;
}
vk::PipelineLayoutCreateInfo pipeline_layout_info{};
pipeline_layout_info.sType = vk::StructureType::ePipelineLayoutCreateInfo;
pipeline_layout_info.pushConstantRangeCount =
push_constant_range.size != 0 ? 1u : 0u;
pipeline_layout_info.pPushConstantRanges =
push_constant_range.size != 0 ? &push_constant_range : nullptr;
vk::PipelineLayout pipeline_layout = nullptr;
RequireVk(test.name, "graphics",
m_device.createPipelineLayout(&pipeline_layout_info, nullptr,
&pipeline_layout),
"vkCreatePipelineLayout");
vk::PipelineShaderStageCreateInfo stages[2] = {};
stages[0].sType = vk::StructureType::ePipelineShaderStageCreateInfo;
stages[0].stage = vk::ShaderStageFlagBits::eVertex;
stages[0].module = vertex_module;
stages[0].pName = "main";
stages[1].sType = vk::StructureType::ePipelineShaderStageCreateInfo;
stages[1].stage = vk::ShaderStageFlagBits::eFragment;
stages[1].module = fragment_module;
stages[1].pName = "main";
vk::VertexInputBindingDescription vertex_binding{};
vertex_binding.binding = 0;
vertex_binding.stride = 6u * sizeof(float);
vertex_binding.inputRate = vk::VertexInputRate::eVertex;
vk::VertexInputAttributeDescription attributes[2] = {};
attributes[0].location = 0;
attributes[0].binding = 0;
attributes[0].format = vk::Format::eR32G32Sfloat;
attributes[0].offset = 0;
attributes[1].location = 1;
attributes[1].binding = 0;
attributes[1].format = vk::Format::eR32G32B32A32Sfloat;
attributes[1].offset = 2u * sizeof(float);
vk::PipelineVertexInputStateCreateInfo vertex_input{};
vertex_input.sType = vk::StructureType::ePipelineVertexInputStateCreateInfo;
vertex_input.vertexBindingDescriptionCount = 1;
vertex_input.pVertexBindingDescriptions = &vertex_binding;
vertex_input.vertexAttributeDescriptionCount = 2;
vertex_input.pVertexAttributeDescriptions = attributes;
vk::PipelineInputAssemblyStateCreateInfo input_assembly{};
input_assembly.sType =
vk::StructureType::ePipelineInputAssemblyStateCreateInfo;
input_assembly.topology = vk::PrimitiveTopology::eTriangleList;
vk::Viewport viewport{};
viewport.x = 0.0f;
viewport.y = 0.0f;
viewport.width = static_cast<float>(test.width);
viewport.height = 1.0f;
viewport.minDepth = 0.0f;
viewport.maxDepth = 1.0f;
vk::Rect2D scissor{};
scissor.extent.width = test.width;
scissor.extent.height = 1;
vk::PipelineViewportStateCreateInfo viewport_state{};
viewport_state.sType = vk::StructureType::ePipelineViewportStateCreateInfo;
viewport_state.viewportCount = 1;
viewport_state.pViewports = &viewport;
viewport_state.scissorCount = 1;
viewport_state.pScissors = &scissor;
vk::PipelineRasterizationStateCreateInfo raster{};
raster.sType = vk::StructureType::ePipelineRasterizationStateCreateInfo;
raster.polygonMode = vk::PolygonMode::eFill;
raster.cullMode = vk::CullModeFlagBits::eNone;
raster.frontFace = vk::FrontFace::eCounterClockwise;
raster.lineWidth = 1.0f;
vk::PipelineRasterizationProvokingVertexStateCreateInfoEXT provoking_vertex{};
if (test.provoking_vtx_last) {
provoking_vertex.provokingVertexMode = vk::ProvokingVertexModeEXT::eLastVertex;
raster.pNext = &provoking_vertex;
}
vk::PipelineMultisampleStateCreateInfo multisample{};
multisample.sType = vk::StructureType::ePipelineMultisampleStateCreateInfo;
multisample.rasterizationSamples = test.samples;
std::array<vk::PipelineColorBlendAttachmentState, 8> color_attachments{};
for (auto &attachment : color_attachments) {
attachment.colorWriteMask =
vk::ColorComponentFlagBits::eR | vk::ColorComponentFlagBits::eG |
vk::ColorComponentFlagBits::eB | vk::ColorComponentFlagBits::eA;
}
vk::PipelineColorBlendStateCreateInfo color_blend{};
color_blend.sType = vk::StructureType::ePipelineColorBlendStateCreateInfo;
color_blend.attachmentCount = object_id ? 8 : 1;
color_blend.pAttachments = color_attachments.data();
vk::GraphicsPipelineCreateInfo pipeline_info{};
std::array<vk::Format, 8> color_formats{};
color_formats[0] = target.format;
color_formats[7] = primitive_target.format;
vk::PipelineRenderingCreateInfo rendering_pipeline{};
rendering_pipeline.sType = vk::StructureType::ePipelineRenderingCreateInfo;
rendering_pipeline.colorAttachmentCount = color_blend.attachmentCount;
rendering_pipeline.pColorAttachmentFormats = color_formats.data();
pipeline_info.sType = vk::StructureType::eGraphicsPipelineCreateInfo;
pipeline_info.pNext = &rendering_pipeline;
pipeline_info.stageCount = 2;
pipeline_info.pStages = stages;
pipeline_info.pVertexInputState = &vertex_input;
pipeline_info.pInputAssemblyState = &input_assembly;
pipeline_info.pViewportState = &viewport_state;
pipeline_info.pRasterizationState = &raster;
pipeline_info.pMultisampleState = &multisample;
pipeline_info.pColorBlendState = &color_blend;
pipeline_info.layout = pipeline_layout;
vk::Pipeline pipeline = nullptr;
RequireVk(test.name, "graphics",
m_device.createGraphicsPipelines(nullptr, 1, &pipeline_info,
nullptr, &pipeline),
"vkCreateGraphicsPipelines");
std::array<vk::RenderingAttachmentInfo, 8> colors{};
colors[0].imageView = target.view;
colors[0].imageLayout = vk::ImageLayout::eGeneral;
colors[0].loadOp = vk::AttachmentLoadOp::eClear;
colors[0].storeOp = vk::AttachmentStoreOp::eStore;
if (resolved.image != nullptr) {
colors[0].resolveMode = vk::ResolveModeFlagBits::eAverage;
colors[0].resolveImageView = resolved.view;
colors[0].resolveImageLayout = vk::ImageLayout::eGeneral;
}
if (object_id) {
colors[7].imageView = primitive_target.view;
colors[7].imageLayout = vk::ImageLayout::eGeneral;
colors[7].loadOp = vk::AttachmentLoadOp::eClear;
colors[7].storeOp = vk::AttachmentStoreOp::eStore;
colors[7].clearValue.color.uint32[0] = 0xfeedfaceu;
}
vk::RenderingInfo rendering{};
rendering.sType = vk::StructureType::eRenderingInfo;
rendering.renderArea.extent = {test.width, 1};
rendering.layerCount = test.layers;
rendering.colorAttachmentCount = color_blend.attachmentCount;
rendering.pColorAttachments = colors.data();
std::vector<u32> pixel;
// Reuse the same shader and pipeline while changing only dynamic object ID.
for (size_t draw = 0; draw < std::max<size_t>(test.object_ids.size(), 1); ++draw) {
vk::CommandBuffer cmd = BeginCommands(test.name, "graphics");
for (auto *image : {&target, &primitive_target}) {
if (image->image != nullptr && image->layout != vk::ImageLayout::eGeneral) {
AddImageBarrier(cmd, image->image, image->layout,
vk::ImageLayout::eGeneral,
vk::PipelineStageFlagBits::eTransfer,
vk::PipelineStageFlagBits::eColorAttachmentOutput,
vk::AccessFlagBits::eTransferRead,
vk::AccessFlagBits::eColorAttachmentWrite,
image->mip_levels, image->layers);
image->layout = vk::ImageLayout::eGeneral;
}
}
cmd.beginRendering(rendering);
cmd.bindPipeline(vk::PipelineBindPoint::eGraphics, pipeline);
if (push_constant_range.size != 0) {
ShaderRecompiler::IR::PushData push_data;
if (object_id) {
push_data.dwords[0] = test.object_ids[draw];
}
if (fragment_bind.UsesPushData()) {
std::copy(test.push_constants.begin(), test.push_constants.end(),
push_data.dwords.begin() + fragment_bind.push_data_start_dword);
}
cmd.pushConstants(pipeline_layout, vk::ShaderStageFlagBits::eFragment, 0,
sizeof(push_data), push_data.dwords.data());
}
vk::DeviceSize offset = 0;
cmd.bindVertexBuffers(0, 1, &vertex_buffer.buffer, &offset);
cmd.draw(static_cast<u32>(vertices.size() / 6), test.layers, 0, 0);
cmd.endRendering();
EndSubmitAndFree(test.name, "graphics", cmd);
auto color_words = ReadImage(test.name, resolved.image != nullptr ? &resolved : &target);
pixel.insert(pixel.end(), color_words.begin(), color_words.end());
if (object_id) {
auto id_words = ReadImage(test.name, &primitive_target);
pixel.insert(pixel.end(), id_words.begin(), id_words.end());
}
}
m_device.destroyPipeline(pipeline, nullptr);
m_device.destroyPipelineLayout(pipeline_layout, nullptr);
m_device.destroyShaderModule(fragment_module, nullptr);
m_device.destroyShaderModule(vertex_module, nullptr);
DestroyBuffer(&vertex_buffer);
DestroyImage(&primitive_target);
DestroyImage(&resolved);
DestroyImage(&target);
return pixel;
}
void CheckGpuTilerCpuParity() {
constexpr const char *name = "GpuTilerCpuParity";
EnsureRuntimeContext();
CommandScheduler scheduler(Renderer(), m_runtime_context);
HW::Context registers{};
HW::UserConfig user_config{};
HW::Shader shaders{};
scheduler.Begin(registers, user_config, shaders);
StreamBuffer parameters(m_runtime_context, scheduler, MemoryUsage::Stream,
1u << 20);
TileManager tile_manager(m_runtime_context, scheduler, parameters);
constexpr uint64_t conversion_limit = 128ull << 20;
Require(name, "D16 conversion chunk boundaries",
TileManagerTestAccess::ConversionRows(0, 32ull << 10, 32ull << 10,
4096, 256, conversion_limit,
65535) == 4096 &&
TileManagerTestAccess::ConversionRows(
0, 32ull << 10, 32ull << 10, 4097, 256, conversion_limit,
65535) == 4096 &&
TileManagerTestAccess::ConversionRows(
257, 32ull << 10, 32ull << 10, 4097, 256, conversion_limit,
65535) == 4095,
"D16 conversion planner exceeds descriptor or dispatch limits");
const auto align_dword = [](uint64_t size) {
return (size + sizeof(u32) - 1u) & ~(uint64_t{sizeof(u32) - 1u});
};
const auto to_dwords = [&](const std::vector<uint8_t> &bytes,
uint64_t padded_size) {
std::vector<u32> words(static_cast<size_t>(padded_size / sizeof(u32)), 0);
std::memcpy(words.data(), bytes.data(), bytes.size());
return words;
};
const auto read_bytes = [&](const Buffer &buffer, uint64_t size) {
const auto words = ReadBuffer(
name, buffer, static_cast<size_t>(align_dword(size) / sizeof(u32)));
std::vector<uint8_t> bytes(static_cast<size_t>(size));
std::memcpy(bytes.data(), words.data(), bytes.size());
return bytes;
};
const auto host_barrier = [&](vk::Buffer buffer, uint64_t size,
vk::PipelineStageFlags source_stage,
vk::AccessFlags source_access) {
vk::BufferMemoryBarrier barrier{};
barrier.sType = vk::StructureType::eBufferMemoryBarrier;
barrier.srcAccessMask = source_access;
barrier.dstAccessMask = vk::AccessFlagBits::eHostRead;
barrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
barrier.buffer = buffer;
barrier.offset = 0;
barrier.size = size;
scheduler.Current().Handle().pipelineBarrier(
source_stage, vk::PipelineStageFlagBits::eHost, {}, 0, nullptr, 1,
&barrier, 0, nullptr);
};
auto scratch_input = CreateHostBuffer(name, 32, AllFlags,
{0x00020001u, 0x00040003u, 0x00060005u, 0x00080007u,
0x000a0009u, 0x000c000bu, 0x000e000du, 0x0010000fu});
auto scratch_output = CreateHostBuffer(name, 56, AllFlags, {});
const auto copy_scratch = [&](TileManager::Result result, uint64_t offset) {
const vk::BufferCopy copy{result.offset, offset, result.size};
scheduler.Current().Handle().copyBuffer(result.buffer, scratch_output.buffer, 1, &copy);
};
// Keep all four conversions in flight to check growth, overwrite ordering,
// the requested span on reuse, and two simultaneously live workspaces.
const auto first = tile_manager.TransformColor(
{scratch_input.buffer, 0, 8}, ColorTransform::SwapBgra16, true);
copy_scratch(first, 0);
const auto grown = tile_manager.TransformColor(
{scratch_input.buffer, 0, 32}, ColorTransform::SwapBgra16, true);
copy_scratch(grown, 8);
const auto reused = tile_manager.TransformColor(
{scratch_input.buffer, 8, 8}, ColorTransform::SwapBgra16, true);
copy_scratch(reused, 40);
const auto paired = tile_manager.TransformColor(reused, ColorTransform::SwapBgra16, true);
copy_scratch(paired, 48);
Require(name, "scratch workspace reuse",
first.buffer != grown.buffer && grown.buffer == reused.buffer &&
reused.buffer != paired.buffer && reused.size == 8 && paired.size == 8,
"scratch growth, input exclusion, or requested span was lost");
host_barrier(scratch_output.buffer, scratch_output.size,
vk::PipelineStageFlagBits::eTransfer, vk::AccessFlagBits::eTransferWrite);
scheduler.Finish();
Require(name, "scratch workspace GPU contents",
ReadBuffer(name, scratch_output, 14) == std::vector<u32>{
0x00020003u, 0x00040001u,
0x00020003u, 0x00040001u, 0x00060007u, 0x00080005u,
0x000a000bu, 0x000c0009u, 0x000e000fu, 0x0010000du,
0x00060007u, 0x00080005u, 0x00060005u, 0x00080007u},
"scratch reuse overwrote an earlier result or its live conversion input");
DestroyBuffer(&scratch_output);
DestroyBuffer(&scratch_input);
const auto gpu_detile =
[&](const std::vector<uint8_t> &tiled, std::vector<uint8_t> *linear,
uint64_t tiled_capacity, uint64_t linear_capacity,
const ImageInfo &info, TileManager *manager = nullptr,
uint64_t source_offset = 0) {
const uint64_t padded_tiled = align_dword(tiled_capacity);
const uint64_t padded_linear = align_dword(linear_capacity);
std::vector<uint8_t> source(source_offset + padded_tiled, 0x7b);
std::copy(tiled.begin(), tiled.end(), source.begin() + source_offset);
auto input = CreateHostBuffer(name, source.size(), AllFlags,
to_dwords(source, source.size()));
auto output = CreateHostBuffer(
name, padded_linear, AllFlags,
std::vector<u32>(static_cast<size_t>(padded_linear / sizeof(u32)),
0));
const auto result = (manager ? *manager : tile_manager)
.Detile(input.buffer, source_offset, info);
Require(name, "detile capacity", result.size <= padded_linear,
"canonical transfer size exceeded the fixture output");
const vk::BufferCopy copy{result.offset, 0, align_dword(result.size)};
scheduler.Current().Handle().copyBuffer(result.buffer, output.buffer,
1, &copy);
host_barrier(output.buffer, padded_linear,
vk::PipelineStageFlagBits::eTransfer,
vk::AccessFlagBits::eTransferWrite);
scheduler.Finish();
*linear = read_bytes(output, linear_capacity);
DestroyBuffer(&output);
DestroyBuffer(&input);
};
const auto gpu_tile = [&](const std::vector<uint8_t> &linear,
std::vector<uint8_t> *tiled,
uint64_t tiled_capacity, uint64_t linear_capacity,
const ImageInfo &info,
TileManager *manager = nullptr,
uint32_t levels = 0, uint64_t source_offset = 0,
uint64_t target_offset = 0) {
const uint64_t padded_tiled = align_dword(tiled_capacity);
const uint64_t padded_linear = align_dword(linear_capacity);
std::vector<uint8_t> source(source_offset + padded_linear, 0x7b);
std::copy(linear.begin(), linear.end(), source.begin() + source_offset);
auto input = CreateHostBuffer(name, source.size(), AllFlags,
to_dwords(source, source.size()));
auto output = CreateHostBuffer(
name, target_offset + padded_tiled, AllFlags,
std::vector<u32>(
static_cast<size_t>((target_offset + padded_tiled) / sizeof(u32)),
0xabababab));
(manager ? *manager : tile_manager)
.Tile(input.buffer, source_offset, padded_linear, output.buffer,
target_offset, padded_tiled, info, levels);
host_barrier(output.buffer, target_offset + padded_tiled,
vk::PipelineStageFlagBits::eComputeShader,
vk::AccessFlagBits::eShaderWrite);
scheduler.Finish();
const auto observed = read_bytes(output, target_offset + tiled_capacity);
Require(name, "nonzero transfer target base",
std::all_of(observed.begin(), observed.begin() + target_offset,
[](uint8_t value) { return value == 0xab; }),
"tiling changed bytes preceding its target offset");
tiled->assign(observed.begin() + target_offset, observed.end());
DestroyBuffer(&output);
DestroyBuffer(&input);
};
for (const bool d32 : std::array{true, false}) {
constexpr uint32_t width = 5;
constexpr uint32_t height = 3;
constexpr uint32_t pitch = 8;
constexpr uint32_t layers = 2;
constexpr uint64_t d16_row = pitch * sizeof(uint16_t);
constexpr uint64_t d32_row = pitch * sizeof(uint32_t);
constexpr uint64_t d16_slice = d16_row * height;
constexpr uint64_t d32_slice = d32_row * height;
constexpr uint64_t d16_size = d16_slice * layers;
constexpr uint64_t d32_size = d32_slice * layers;
StreamBuffer conversion_upload(m_runtime_context, scheduler,
MemoryUsage::Upload, 1u << 20);
auto [prefix, prefix_offset] = conversion_upload.Map(4096, 256);
Require(name, "D16 conversion prefix",
prefix != nullptr && prefix_offset == 0,
"failed to advance the D16 conversion upload ring");
std::memset(prefix, 0x7b, 4096);
conversion_upload.Commit();
auto [source_bytes, source_offset] = conversion_upload.Map(d16_size, 4);
Require(name, "D16 conversion source",
source_bytes != nullptr && source_offset != 0,
"failed to reserve a nonzero D16 conversion source");
std::memset(source_bytes, 0x3c, d16_size);
for (uint32_t layer = 0; layer < layers; layer++) {
for (uint32_t y = 0; y < height; y++) {
auto *row = reinterpret_cast<uint16_t *>(
source_bytes + layer * d16_slice + y * d16_row);
for (uint32_t x = 0; x < width; x++) {
row[x] = static_cast<uint16_t>(
0x1111u * (1u + x + y * width + layer * width * height));
}
}
}
conversion_upload.Commit();
auto promoted = CreateHostBuffer(
name, d32_size, AllFlags,
std::vector<u32>(static_cast<size_t>(d32_size / sizeof(u32)), 0));
auto round_trip = CreateHostBuffer(
name, d16_size, AllFlags,
std::vector<u32>(static_cast<size_t>(d16_size / sizeof(u32)),
0xa5a5a5a5u));
const TileManager::D16Layout promote_layout{
.width = width,
.height = height,
.layers = layers,
.source_row_stride = d16_row,
.target_row_stride = d32_row,
.source_slice_stride = d16_slice,
.target_slice_stride = d32_slice,
};
tile_manager.ConvertD16({conversion_upload.Handle(), source_offset,
conversion_upload.Size() - source_offset},
{promoted.buffer, 0, d32_size},
TileManager::D16Direction::Promote, d32,
promote_layout);
tile_manager.ConvertD16({promoted.buffer, 0, d32_size},
{round_trip.buffer, 0, d16_size},
TileManager::D16Direction::Demote, d32,
{.width = width,
.height = height,
.layers = layers,
.source_row_stride = d32_row,
.target_row_stride = d16_row,
.source_slice_stride = d32_slice,
.target_slice_stride = d16_slice});
host_barrier(round_trip.buffer, d16_size,
vk::PipelineStageFlagBits::eComputeShader,
vk::AccessFlagBits::eShaderWrite);
scheduler.Finish();
const auto result = read_bytes(round_trip, d16_size);
bool conversion_matches = true;
for (uint32_t layer = 0; layer < layers; layer++) {
for (uint32_t y = 0; y < height; y++) {
for (uint32_t x = 0; x < pitch; x++) {
const auto offset =
layer * d16_slice + y * d16_row + x * sizeof(uint16_t);
conversion_matches &=
x < width
? std::memcmp(result.data() + offset, source_bytes + offset,
sizeof(uint16_t)) == 0
: result[offset] == 0xa5 && result[offset + 1] == 0xa5;
}
}
}
Require(name,
d32 ? "D32 conversion round trip" : "D24 conversion round trip",
conversion_matches,
"D16 conversion lost rows, layers, active values, or padding");
DestroyBuffer(&round_trip);
DestroyBuffer(&promoted);
}
auto fill = [](std::vector<uint8_t> *bytes, u32 salt) {
for (size_t i = 0; i < bytes->size(); i++) {
uint64_t value =
i + static_cast<uint64_t>(salt) * 0x9e3779b97f4a7c15ull;
value = (value ^ (value >> 30u)) * 0xbf58476d1ce4e5b9ull;
value = (value ^ (value >> 27u)) * 0x94d049bb133111ebull;
(*bytes)[i] = static_cast<uint8_t>((value ^ (value >> 31u)) >> 56u);
}
};
auto compare = [&](const char *stage, const std::vector<uint8_t> &expected,
const std::vector<uint8_t> &actual) {
if (expected != actual) {
const auto mismatch = static_cast<size_t>(
std::mismatch(expected.begin(), expected.end(), actual.begin())
.first -
expected.begin());
std::ostringstream out;
out << "first mismatch at " << mismatch << " of " << expected.size();
Fail(name, stage, out.str());
}
};
auto convert_reference = [&](bool to_tiled, std::vector<uint8_t> *dst,
const std::vector<uint8_t> &src,
const ImageInfo &info, uint32_t level = 0) {
const auto &block = info.tiling.block;
const auto &mip = info.mip_layout[level];
const auto active = info.MipExtent(level);
const uint32_t width = (active.width + info.tiling.texel_width - 1u) /
info.tiling.texel_width;
const uint32_t height = (active.height + info.tiling.texel_height - 1u) /
info.tiling.texel_height;
const uint64_t columns =
(mip.pitch + block.block_width - 1u) / block.block_width;
const uint64_t slice = info.linear_slice_stride != 0
? info.linear_slice_stride
: mip.linear_size;
const bool tail = level >= info.first_tail_level;
for (u32 z = 0; z < active.depth; ++z) {
for (u32 y = 0; y < height; ++y) {
for (u32 x = 0; x < width; ++x) {
const u32 bx = tail ? 0 : x / block.block_width;
const u32 by = tail ? 0 : y / block.block_height;
const u32 bz = z / block.block_depth;
const u32 lx = tail ? x + mip.tail_x : x % block.block_width;
const u32 ly = tail ? y + mip.tail_y : y % block.block_height;
const u32 lz = z % block.block_depth;
u32 local = 0, block_xor = 0;
Require(name, "reference offset",
TileGetBlockOffset(block, lx, ly, lz, local) &&
TileGetBlockXor(block, bx, by, mip.surface_z + bz,
block_xor),
"CPU reference address lookup failed");
const uint64_t tiled =
mip.offset + uint64_t{bz} * info.tiled_slice_stride +
(by * columns + bx) * block.block_size + (local ^ block_xor);
const uint64_t linear =
mip.linear_offset + uint64_t{z} * slice +
(uint64_t{y} * mip.linear_pitch + x) * block.bytes_per_element;
const uint64_t dst_offset = to_tiled ? tiled : linear;
const uint64_t src_offset = to_tiled ? linear : tiled;
Require(name, "reference range",
dst_offset + block.bytes_per_element <= dst->size() &&
src_offset + block.bytes_per_element <= src.size(),
"CPU reference address escaped storage");
if (info.GetColorTransform() == ColorTransform::Reverse10_11_11) {
uint32_t packed = 0;
std::memcpy(&packed, src.data() + src_offset, sizeof(packed));
const std::array<uint32_t, 3> widths =
to_tiled ? std::array<uint32_t, 3>{11, 11, 10}
: std::array<uint32_t, 3>{10, 11, 11};
std::array<uint32_t, 3> channels{};
for (uint32_t channel = 0; channel < channels.size(); ++channel) {
channels[channel] = packed & ((1u << widths[channel]) - 1u);
packed >>= widths[channel];
}
uint32_t shift = 0;
for (int channel = 2; channel >= 0; --channel) {
packed |= channels[channel] << shift;
shift += widths[channel];
}
std::memcpy(dst->data() + dst_offset, &packed, sizeof(packed));
} else if (info.GetColorTransform() == ColorTransform::SwapBgra16) {
std::array<uint16_t, 4> channels{};
std::memcpy(channels.data(), src.data() + src_offset,
sizeof(channels));
std::swap(channels[0], channels[2]);
std::memcpy(dst->data() + dst_offset, channels.data(),
sizeof(channels));
} else {
std::memcpy(dst->data() + dst_offset, src.data() + src_offset,
block.bytes_per_element);
}
}
}
}
};
const auto block_image = [](const TileBlockLayout &block, u32 width,
u32 height, u32 depth, u32 pitch,
uint64_t guest_size) {
ImageInfo info{};
info.extent = {width, height, block.block_depth > 1 ? depth : 1u};
info.type = block.block_depth > 1 ? Prospero::ImageType::kColor3D
: Prospero::ImageType::kColor2D;
info.resources = {1, block.block_depth > 1 ? 1u : depth};
info.tile_mode = Prospero::TileMode::kStandard4KB;
info.bytes_per_block = block.bytes_per_element;
info.tiling = {block, 1, 1};
info.data.size = guest_size;
info.first_tail_level = 16;
const uint64_t columns =
(pitch + block.block_width - 1u) / block.block_width;
const uint64_t rows =
(height + block.block_height - 1u) / block.block_height;
info.tiled_slice_stride = columns * rows * block.block_size;
info.mip_layout[0] = {
0, info.IsVolume() ? info.tiled_slice_stride : guest_size, pitch,
static_cast<uint32_t>(rows * block.block_height)};
info.mip_layout[0].linear_pitch = pitch;
info.mip_layout[0].linear_size =
uint64_t{pitch} * height * block.bytes_per_element;
return info;
};
struct FamilyCase {
TileBlockFamily family;
u32 max_bpe;
};
constexpr FamilyCase families[] = {
{TileBlockFamily::Standard256B, 16},
{TileBlockFamily::Standard4KB, 16},
{TileBlockFamily::Standard4KB3D, 16},
{TileBlockFamily::Standard64KB, 16},
{TileBlockFamily::Standard64KB3D, 16},
{TileBlockFamily::Prt64KB, 16},
{TileBlockFamily::Prt64KB3D, 16},
{TileBlockFamily::RenderTarget64KB, 16},
{TileBlockFamily::Depth64KB, 8},
};
{
TileBlockLayout standard{}, prt{}, standard_3d{}, prt_3d{}, color{},
depth{};
u32 standard_offset = 0, prt_offset = 0, standard_3d_offset = 0,
prt_3d_offset = 0, color_z = 0, depth_z = 0;
bool fixed_addresses =
TileGetBlockLayout(TileBlockFamily::Standard64KB, 4, standard) &&
TileGetBlockLayout(TileBlockFamily::Prt64KB, 4, prt) &&
TileGetBlockLayout(TileBlockFamily::Standard64KB3D, 4, standard_3d) &&
TileGetBlockLayout(TileBlockFamily::Prt64KB3D, 4, prt_3d) &&
TileGetBlockLayout(TileBlockFamily::RenderTarget64KB, 4, color) &&
TileGetBlockLayout(TileBlockFamily::Depth64KB, 4, depth) &&
TileGetBlockOffset(standard, 64, 0, 0, standard_offset) &&
TileGetBlockOffset(prt, 64, 0, 0, prt_offset) &&
TileGetBlockOffset(standard_3d, 16, 0, 0, standard_3d_offset) &&
TileGetBlockOffset(prt_3d, 16, 0, 0, prt_3d_offset) &&
TileGetBlockXor(color, 0, 0, 1, color_z) &&
TileGetBlockXor(depth, 0, 0, 15, depth_z);
Require(name, "fixed address vectors",
fixed_addresses && standard_offset == 0x8000 &&
prt_offset == 0x8100 && standard_3d_offset == 0x8000 &&
prt_3d_offset == 0x8400 && standard_3d.block_width == 32 &&
standard_3d.block_height == 32 &&
standard_3d.block_depth == 16 && color_z == 0x800 &&
depth_z == 0xf00,
"a fixed block address changed");
constexpr auto format = Prospero::BufferFormat::k32Float;
constexpr u32 levels = 7;
TileSizeAlign total{};
TileSizeOffset mip[levels]{};
TilePaddedSize padded[levels]{};
TileGetTextureSize(format, 65, 33, levels,
Prospero::TileMode::kStandard256B, &total, mip,
padded);
constexpr u32 offsets[levels] = {0x1a00, 0xb00, 0x500, 0x300,
0x200, 0x100, 0};
constexpr u32 sizes[levels] = {0x2d00, 0xf00, 0x600, 0x200,
0x100, 0x100, 0x100};
bool layout_matches = total.size == 0x4700 && total.align == 0x100 &&
padded[0].width == 72 && padded[0].height == 40;
for (u32 level = 0; level < levels; ++level) {
layout_matches &= mip[level].offset == offsets[level] &&
mip[level].size == sizes[level];
}
Require(name, "fixed mip vector", layout_matches,
"the reverse-packed small-block mip layout changed");
}
u32 case_index = 0;
auto check_round_trip = [&](const char *stage, uint64_t tiled_size,
const ImageInfo &info,
TileManager *manager = nullptr) {
const uint64_t linear_size = info.LinearSize();
std::vector<uint8_t> tiled(tiled_size);
std::vector<uint8_t> cpu(linear_size, 0);
std::vector<uint8_t> gpu(linear_size, 0xab);
fill(&tiled, ++case_index);
for (uint32_t level = 0; level < info.resources.levels; ++level) {
convert_reference(false, &cpu, tiled, info, level);
}
gpu_detile(tiled, &gpu, tiled_size, linear_size, info, manager);
compare((std::string(stage) + " detile bytes").c_str(), cpu, gpu);
std::vector<uint8_t> linear(linear_size);
std::vector<uint8_t> cpu_tiled(tiled_size, 0xab);
std::vector<uint8_t> gpu_tiled(tiled_size, 0xab);
fill(&linear, 0x280u + case_index);
for (uint32_t level = 0; level < info.resources.levels; ++level) {
convert_reference(true, &cpu_tiled, linear, info, level);
}
gpu_tile(linear, &gpu_tiled, tiled_size, linear_size, info, manager);
compare((std::string(stage) + " tile bytes").c_str(), cpu_tiled,
gpu_tiled);
};
{
constexpr auto format = Prospero::BufferFormat::k8UNorm;
constexpr auto tile = Prospero::TileMode::kStandard256B;
constexpr u32 layers = 261;
const auto info =
MakeTilingImage(format, 1, 1, 1, layers, tile, 0, false);
const auto regions = info.BufferCopies();
Require(name, "large array mip record",
regions.size() == 1 &&
regions[0].imageSubresource.layerCount == layers &&
info.MipExtent(0).depth == layers &&
info.tiled_slice_stride == 256 &&
info.data.size == uint64_t{layers} * 256,
"array layers expanded the mip table or lost their guest stride");
check_round_trip("large array", info.data.size, info);
}
{
// Adjacent subword mips share output dwords and a guest tail block.
TileBlockLayout block{};
Require(name, "packed mip block",
TileGetBlockLayout(TileBlockFamily::Standard4KB, 1, block),
"packed mip block is unavailable");
auto info = block_image(block, 1, 1, 1, 1, 4096);
info.resources.levels = 16;
info.first_tail_level = 0;
for (u32 mip = 0; mip < info.resources.levels; ++mip) {
info.mip_layout[mip] = {0, 4096, block.block_width, block.block_height};
info.mip_layout[mip].linear_offset = mip;
info.mip_layout[mip].linear_size = 1;
info.mip_layout[mip].linear_pitch = 1;
info.mip_layout[mip].tail_x = mip;
}
check_round_trip("16 packed subword mip records", 4096, info);
const auto atom =
std::max<uint64_t>(m_runtime_context.GetPhysicalDeviceProperties()
.limits.nonCoherentAtomSize,
1);
for (const auto [mips, bytes] :
std::array{std::pair{1u, 64u}, std::pair{2u, 112u}}) {
StreamBuffer compact_parameters(m_runtime_context, scheduler,
MemoryUsage::Stream,
Common::AlignUp<uint64_t>(bytes, atom));
TileManager compact_tiler(m_runtime_context, scheduler,
compact_parameters);
auto compact_info = info;
compact_info.resources.levels = mips;
check_round_trip("compact parameter buffer", 4096, compact_info,
&compact_tiler);
}
}
{
constexpr auto format = Prospero::BufferFormat::k32Float;
constexpr auto tile = Prospero::TileMode::kStandard256B;
constexpr u32 levels = 7, layers = 3;
constexpr uint64_t guest_stride = 0x4800,
guest_size = guest_stride * layers;
constexpr uint64_t offsets[levels] = {0x1a00, 0xb00, 0x500, 0x300,
0x200, 0x100, 0};
const auto info = MakeTilingImage(format, 65, 33, levels, layers, tile,
guest_size, false);
const auto regions = info.BufferCopies();
bool valid =
info.tiled_slice_stride == guest_stride && regions.size() == levels;
for (u32 mip = 0; mip < levels && valid; ++mip) {
valid &= info.mip_layout[mip].offset == offsets[mip] &&
info.MipExtent(mip).depth == layers &&
regions[mip].imageSubresource.layerCount == layers;
}
Require(name, "reverse mip array addresses", valid,
"mip order or padded guest layer stride changed");
check_round_trip("reverse mip array", guest_size, info);
std::vector<uint8_t> tiled(guest_size),
expected_linear(info.LinearSize(), 0), observed_linear;
fill(&tiled, ++case_index);
for (u32 mip = 0; mip < levels; ++mip)
convert_reference(false, &expected_linear, tiled, info, mip);
gpu_detile(tiled, &observed_linear, guest_size, info.LinearSize(), info,
nullptr, 260);
compare("detile nonzero source base", expected_linear, observed_linear);
const auto saved_mips = info.mip_layout;
const auto prefix = info.BufferCopies(256, 2);
Require(name, "partial mip array addresses",
prefix.size() == 2 && prefix[0].bufferOffset == 256 &&
prefix[1].bufferOffset ==
info.mip_layout[1].linear_offset + 256 &&
info.mip_layout[0].offset == 0x1a00 &&
info.mip_layout[1].offset == 0xb00,
"partial mip transfer recomputed the full-chain guest stride");
std::vector<uint8_t> linear(info.LinearSize());
fill(&linear, ++case_index);
for (const u32 count : {2u, levels, 1u, levels}) {
std::vector<uint8_t> expected(guest_size, 0xab),
observed(guest_size, 0xab);
for (u32 mip = 0; mip < count; ++mip)
convert_reference(true, &expected, linear, info, mip);
gpu_tile(linear, &observed, guest_size, info.LinearSize(), info,
nullptr, count, count == levels ? 516 : 260,
count == levels ? 520 : 264);
compare("repeated partial/full transfer", expected, observed);
Require(
name, "immutable transfer metadata",
info.mip_layout == saved_mips && info.resources.levels == levels &&
info.tiled_slice_stride == guest_stride,
"partial or repeated transfer changed canonical image metadata");
}
}
for (const auto family : families) {
for (u32 bpe = 1; bpe <= family.max_bpe; bpe <<= 1u) {
TileBlockLayout block{};
Require(name, "block layout",
TileGetBlockLayout(family.family, bpe, block),
"admitted family/BPE has no block layout");
const bool volume = block.block_depth > 1;
const u32 width =
block.block_width * 3u + std::min(block.block_width, 3u);
const u32 height =
block.block_height * 3u + std::min(block.block_height, 3u);
const u32 depth = volume ? block.block_depth + 1u : 1u;
const u32 pitch = block.block_width * 4u;
const uint64_t block_columns =
(pitch + block.block_width - 1u) / block.block_width;
const uint64_t block_rows =
(height + block.block_height - 1u) / block.block_height;
const uint64_t block_slices =
(depth + block.block_depth - 1u) / block.block_depth;
const uint64_t storage_size =
block_columns * block_rows * block_slices * block.block_size;
std::vector<uint8_t> tiled(storage_size);
std::vector<uint8_t> cpu(storage_size, 0);
std::vector<uint8_t> gpu(storage_size, 0xab);
fill(&tiled, ++case_index);
auto info =
block_image(block, width, height, depth, pitch, storage_size);
info.mip_layout[0].surface_z =
family.family == TileBlockFamily::RenderTarget64KB ||
family.family == TileBlockFamily::Depth64KB
? 3
: 0;
convert_reference(false, &cpu, tiled, info);
gpu_detile(tiled, &gpu, storage_size, storage_size, info);
const auto family_label = [&](const char *operation) {
std::ostringstream out;
out << operation << " family=" << static_cast<u32>(family.family)
<< " bpe=" << bpe;
return out.str();
};
compare(family_label("detile bytes").c_str(), cpu, gpu);
{
std::vector<uint8_t> linear(storage_size);
std::vector<uint8_t> cpu_tiled(storage_size, 0xab);
std::vector<uint8_t> gpu_tiled(storage_size, 0xab);
fill(&linear, 0x80u + case_index);
convert_reference(true, &cpu_tiled, linear, info);
gpu_tile(linear, &gpu_tiled, storage_size, storage_size, info);
compare(family_label("tile bytes").c_str(), cpu_tiled, gpu_tiled);
}
}
}
// Exercise the real texture-layout/info-building seam for every format
// admitted by each texture tile mode. Formats sharing a byte/block width
// intentionally share a shader, but this loop still validates their
// texel-to-element conversion (notably every BCn format).
struct TextureMode {
Prospero::TileMode tile;
TileBlockFamily family;
};
constexpr TextureMode texture_modes[] = {
{Prospero::TileMode::kStandard256B, TileBlockFamily::Standard256B},
{Prospero::TileMode::kStandard4KB, TileBlockFamily::Standard4KB},
{Prospero::TileMode::kStandard64KB, TileBlockFamily::Standard64KB},
{Prospero::TileMode::kPrt, TileBlockFamily::Prt64KB},
{Prospero::TileMode::kRenderTarget, TileBlockFamily::RenderTarget64KB},
{Prospero::TileMode::kDepth, TileBlockFamily::Depth64KB},
};
struct TextureTileFormatCase {
Prospero::BufferFormat format;
u32 bytes_per_element;
bool depth_supported;
};
constexpr TextureTileFormatCase texture_tile_formats[] = {
{Prospero::BufferFormat::k8Srgb, 1, true},
{Prospero::BufferFormat::k8_8Srgb, 2, true},
{Prospero::BufferFormat::k9_9_9_5Float, 4, true},
{Prospero::BufferFormat::kBc1Srgb, 8, false},
{Prospero::BufferFormat::kBc7Srgb, 16, false},
{Prospero::BufferFormat::k32_32_32_32Float, 16, false},
};
for (const auto &test : texture_tile_formats) {
for (const auto tile :
{Prospero::TileMode::kDepth, Prospero::TileMode::kRenderTarget}) {
TileTextureBlockLayout texture{};
const bool supported =
TileGetTextureBlockLayout(test.format, tile, false, texture);
const bool expected = tile == Prospero::TileMode::kRenderTarget ||
test.depth_supported;
Require(name, "texture tile format policy",
supported == expected &&
(!supported || texture.block.bytes_per_element ==
test.bytes_per_element),
"texture tile policy used attachment format restrictions");
}
}
{
TileSurfaceLayout invalid{};
const TileSurfaceDescription description{
Prospero::BufferFormat::k32Float,
Prospero::TileMode::kStandard256B,
TileSurfaceDimension::Dim3D,
16,
16,
16,
1,
1,
};
Require(name, "3D tile policy",
!TileGetTiledTextureLayout(description, invalid),
"Standard256B volume silently fell back to a 2D array layout");
}
Require(
name, "FMASK policy",
Prospero::IsFmaskTextureFormat(Prospero::BufferFormat::kFmask8_S4_F4),
"FMASK sample metadata lost its explicit tiler exclusion "
"classification");
u32 format_cases = 0;
for (u32 raw_format = 1;
raw_format <= static_cast<uint32_t>(Prospero::BufferFormat::kBc7Srgb);
++raw_format) {
const auto format = static_cast<Prospero::BufferFormat>(raw_format);
// FMASK contains sample-to-fragment metadata, not texels for the GPU tiler.
if (Prospero::IsFmaskTextureFormat(format)) {
continue;
}
for (const auto &mode : texture_modes) {
TileTextureBlockLayout texture{};
if (!TileGetTextureBlockLayout(format, mode.tile, false, texture)) {
continue;
}
const u32 bpe = texture.block.bytes_per_element;
Require(name, "format block", texture.block.family == mode.family,
"CPU-supported format selected the wrong block family");
constexpr u32 width = 67;
constexpr u32 height = 51;
TileSizeAlign total{};
TileGetTextureSize(format, width, height, 1, mode.tile, &total, nullptr,
nullptr);
Require(name, "format size", total.size != 0,
"supported format has an empty layout");
const auto layout = MakeTilingImage(format, width, height, 1, 1,
mode.tile, total.size, false);
const auto regions = layout.BufferCopies();
Require(name, "format family",
layout.resources.levels == 1 &&
layout.tiling.block.family == mode.family &&
layout.tiling.block.bytes_per_element == bpe,
"texture info selected the wrong shader family");
std::vector<uint8_t> tiled(total.size);
std::vector<uint8_t> cpu(total.size, 0);
std::vector<uint8_t> gpu(total.size, 0xab);
fill(&tiled, ++case_index);
convert_reference(false, &cpu, tiled, layout);
gpu_detile(tiled, &gpu, total.size, total.size, layout);
compare("format bytes", cpu, gpu);
++format_cases;
}
}
Require(name, "format coverage", format_cases != 0,
"no CPU-supported texture formats were tested");
{
constexpr auto format = Prospero::BufferFormat::kBc1Srgb;
constexpr auto tile = Prospero::TileMode::kRenderTarget;
constexpr u32 layers = 8;
const auto captured =
MakeTilingImage(format, 2048, 2048, 1, layers, tile, 0, false);
Require(name, "PPSA19577 BC1 array footprint",
captured.data.size == 0x1000000 &&
captured.tiled_slice_stride == 0x200000 &&
captured.first_tail_level == 1 &&
captured.tiling.block.block_width == 128 &&
captured.tiling.block.block_height == 64 &&
captured.mip_layout[0].pitch == 512 &&
captured.mip_layout[0].height == 512,
"R64 BC1 texture lost its compressed blocks or array stride");
const auto small_array =
MakeTilingImage(format, 67, 51, 1, layers, tile, 0, false);
check_round_trip("R64 BC1 array", small_array.data.size, small_array);
}
{
constexpr auto format = Prospero::BufferFormat::k11_11_10UInt;
constexpr auto tile = Prospero::TileMode::kStandard64KB;
constexpr u32 width = 128, height = 128, levels = 8;
TileSurfaceLayout surface{};
const TileSurfaceDescription description{
format, tile, TileSurfaceDimension::Dim2D, width, height, 1,
levels, 1};
TileSizeAlign total{};
TileGetTextureTotalSize(format, width, height, 1, levels, tile, false,
total);
const bool built = TileGetTiledTextureLayout(description, surface);
const auto surface_format = TextureGetSurfaceFormatInfo(format);
constexpr u32 tail_xy[7][2] = {{64, 0}, {0, 64}, {32, 0}, {0, 32},
{16, 0}, {8, 16}, {0, 24}};
bool valid = built && Prospero::NumBytesPerElement(format) == 4 &&
Prospero::BlockCompressedBytesPerBlock(format) == 0 &&
Prospero::SampledTextureNumericClass(format) ==
Prospero::TextureNumericClass::Uint &&
Prospero::RemapTextureFormat(format) ==
Prospero::BufferFormat::k32UInt &&
surface_format.vk_format == vk::Format::eR32Uint &&
surface_format.conversion_format == format &&
VulkanFormat(format) == vk::Format::eUndefined &&
surface.texture.block.block_width == 128 &&
surface.texture.block.block_height == 128 &&
surface.texture.block.block_depth == 1 &&
surface.first_tail_level == 1 &&
surface.mips[0].offset == 0x10000 && total.size == 0x20000 &&
total.align == 0x10000;
for (u32 level = 1; level < levels && valid; level++) {
valid &= surface.mips[level].offset == 0 &&
surface.mips[level].tail_x == tail_xy[level - 1][0] &&
surface.mips[level].tail_y == tail_xy[level - 1][1];
}
Require(name, "11_11_10 uint layout", valid,
"PS5 packed integer texture metadata or Standard64KB mip tail "
"changed");
const auto signed_format = Prospero::BufferFormat::k32SInt;
const auto signed_surface = TextureGetSurfaceFormatInfo(signed_format);
Require(name, "signed sampled surface mapping",
Prospero::SampledTextureNumericClass(signed_format) ==
Prospero::TextureNumericClass::Sint &&
Prospero::RemapTextureFormat(signed_format) ==
signed_format &&
signed_surface.vk_format == vk::Format::eR32Sint &&
signed_surface.conversion_format ==
Prospero::BufferFormat::kInvalid,
"signed sampled texture class or native backing changed");
}
{
constexpr auto format = Prospero::BufferFormat::k32_32_32Float;
TileTextureElementLayout element{};
Require(name, "RGB32 texture policy",
!TileGetTextureElementLayout(format, element),
"12-byte texel layouts must be rejected");
}
{
constexpr auto format = Prospero::BufferFormat::kBc1UNorm;
constexpr auto tile = Prospero::TileMode::kStandard64KB;
constexpr u32 width = 256, height = 256, levels = 9;
TileSizeAlign total{};
TileGetTextureSize(format, width, height, levels, tile, &total, nullptr,
nullptr);
const auto layout = MakeTilingImage(format, width, height, levels, 1,
tile, total.size, false);
const auto regions = layout.BufferCopies();
const auto linear_size = layout.LinearSize();
Require(name, "BC1 mip-tail capacities",
total.size == 0x10000 && layout.first_tail_level == 0 &&
linear_size == 0x15560 && linear_size > total.size,
"BC1 mip tail conflated tiled and linear capacities");
check_round_trip("BC1 mip tail", total.size, layout);
}
{
constexpr auto format = Prospero::BufferFormat::k32Float;
constexpr auto tile = Prospero::TileMode::kRenderTarget;
constexpr u32 width = 129, height = 65, layers = 3;
TileSizeAlign total{};
TileGetTextureTotalSize(format, width, height, layers, 1, tile, false,
total);
const auto layout = MakeTilingImage(format, width, height, 1, layers,
tile, total.size, false);
const auto regions = layout.BufferCopies();
Require(name, "array infos",
layout.resources.levels == 1 &&
layout.MipExtent(0).depth == layers &&
layout.mip_layout[0].surface_z == 0 &&
layout.tiled_slice_stride == total.size / layers &&
regions.size() == 1 &&
regions[0].imageSubresource.layerCount == layers,
"array mip lost its layer count or absolute surface Z origin");
check_round_trip("array", total.size, layout);
}
for (const auto &mode : texture_modes) {
constexpr auto format = Prospero::BufferFormat::k32Float;
constexpr u32 levels = 2;
TileBlockLayout block{};
Require(name, "odd mip block", TileGetBlockLayout(mode.family, 4, block),
"odd multi-mip format has no block layout");
const u32 width = block.block_width * 2u + 1u;
const u32 height = block.block_height * 2u + 1u;
TileSizeAlign total{};
TileGetTextureSize(format, width, height, levels, mode.tile, &total,
nullptr, nullptr);
const auto layout = MakeTilingImage(format, width, height, levels, 1,
mode.tile, total.size, false);
const auto regions = layout.BufferCopies();
Require(name, "odd mip infos",
layout.resources.levels == 2 &&
layout.mip_layout[1].pitch >=
layout.mip_layout[1].linear_pitch &&
layout.mip_layout[1].height >=
((layout.MipExtent(1).height +
layout.tiling.texel_height - 1u) /
layout.tiling.texel_height) &&
(mode.family == TileBlockFamily::Standard256B ||
layout.mip_layout[1].height >
((layout.MipExtent(1).height +
layout.tiling.texel_height - 1u) /
layout.tiling.texel_height)),
"odd multi-mip physical stride collapsed to the active linear "
"extent");
check_round_trip("odd multi-mip", total.size, layout);
}
for (const bool sparse : {false, true}) {
const auto format = sparse ? Prospero::BufferFormat::k32_32_32_32UInt
: Prospero::BufferFormat::k16_16_16_16Float;
constexpr auto tile = Prospero::TileMode::kStandard4KB;
const uint32_t extent = sparse ? 2 : 16;
auto view = MakeTilingImage(format, extent, extent, 1, 1, tile, 0, false);
view.data.address = sparse ? 0x50322e3500 : 0x5032b23800;
view.UpdateSize();
Require(name, "rebased mip footprint",
view.data.size == (sparse ? 0x60 : 0x800) &&
view.DataAlignment() == (sparse ? 0x100 : 0x800) &&
view.mip_layout[0].pitch == (sparse ? 16 : 32) &&
view.mip_layout[0].height == 16 &&
view.tiled_slice_stride == 0x1000,
"a compact view changed its allocation geometry");
check_round_trip("rebased mip exact range", view.data.size, view);
check_round_trip("rebased mip adjacent canary", 0x1000, view);
std::vector<uint8_t> linear(view.LinearSize());
fill(&linear, ++case_index);
std::vector<uint8_t> expected(0x1000, 0xab), observed;
convert_reference(true, &expected, linear, view);
gpu_tile(linear, &observed, 0x1000, linear.size(), view, nullptr, 0,
0x100, 0x800);
compare("rebased mip prefix and neighboring mip", expected, observed);
std::vector<uint8_t> round_trip;
gpu_detile(observed, &round_trip, observed.size(), linear.size(), view,
nullptr, 0x800);
std::vector<uint8_t> expected_linear(linear.size(), 0);
convert_reference(false, &expected_linear, expected, view);
compare("rebased mip nonzero source", expected_linear, round_trip);
}
struct TailMode {
Prospero::TileMode tile;
TileBlockFamily family;
};
constexpr TailMode tail_modes[] = {
{Prospero::TileMode::kStandard4KB, TileBlockFamily::Standard4KB},
{Prospero::TileMode::kStandard64KB, TileBlockFamily::Standard64KB},
{Prospero::TileMode::kPrt, TileBlockFamily::Prt64KB},
{Prospero::TileMode::kRenderTarget, TileBlockFamily::RenderTarget64KB},
{Prospero::TileMode::kDepth, TileBlockFamily::Depth64KB},
};
for (const auto &mode : tail_modes) {
constexpr auto format = Prospero::BufferFormat::k32Float;
constexpr u32 levels = 7;
TileBlockLayout block{};
Require(name, "2D tail block", TileGetBlockLayout(mode.family, 4, block),
"2D tail mode has no block layout");
const u32 width = block.block_width * 2u;
const u32 height = block.block_height * 2u;
TileSizeAlign total{};
TileGetTextureSize(format, width, height, levels, mode.tile, &total,
nullptr, nullptr);
const auto layout = MakeTilingImage(format, width, height, levels, 1,
mode.tile, total.size, false);
const auto regions = layout.BufferCopies();
Require(name, "2D mip tail seam",
layout.first_tail_level == 2 && layout.resources.levels == levels,
"2D mip chain lost its linear/tiled tail boundary");
check_round_trip("2D mip tail seam", total.size, layout);
}
{
constexpr auto format = Prospero::BufferFormat::k32Float;
constexpr auto tile = Prospero::TileMode::kStandard4KB;
constexpr u32 width = 65;
constexpr u32 height = 129;
constexpr u32 depth = 17;
constexpr u32 levels = 2;
TileSizeAlign total{};
TileGetTextureTotalSize(format, width, height, depth, levels, tile, true,
total);
const auto layout = MakeTilingImage(format, width, height, levels, depth,
tile, total.size, true);
const auto regions = layout.BufferCopies();
Require(
name, "3D mip infos",
regions.size() == levels && layout.resources.levels == levels &&
regions[0].imageExtent.depth == depth &&
regions[1].imageExtent.depth == (depth >> 1u) &&
layout.MipExtent(0).depth == 17 &&
layout.MipExtent(1).depth == 8 &&
layout.mip_layout[0].offset == 0x19000 &&
layout.tiled_slice_stride == 0x6a000 &&
layout.mip_layout[0].offset + layout.tiled_slice_stride ==
0x83000 &&
layout.mip_layout[0].offset + 2 * layout.tiled_slice_stride ==
0xed000 &&
layout.mip_layout[1].offset == 0 &&
layout.mip_layout[1].linear_pitch == 32 &&
((layout.MipExtent(1).height + layout.tiling.texel_height - 1u) /
layout.tiling.texel_height) == 64 &&
layout.mip_layout[1].pitch == 40 &&
layout.mip_layout[1].height == 80,
"Standard4KB3D mip depth, packing, or physical stride changed");
check_round_trip("3D mip", total.size, layout);
}
{
constexpr auto format = Prospero::BufferFormat::kBc1UNorm;
constexpr auto tile = Prospero::TileMode::kStandard4KB;
constexpr u32 width = 65;
constexpr u32 height = 129;
constexpr u32 depth = 17;
constexpr u32 levels = 2;
TileSizeAlign total{};
TileGetTextureTotalSize(format, width, height, depth, levels, tile, true,
total);
const auto layout = MakeTilingImage(format, width, height, levels, depth,
tile, total.size, true);
const auto regions = layout.BufferCopies();
Require(
name, "3D BC mip infos",
layout.resources.levels == levels &&
layout.MipExtent(0).depth == depth &&
layout.MipExtent(1).depth == (depth >> 1u) &&
layout.mip_layout[1].linear_pitch == 8 &&
((layout.MipExtent(1).height + layout.tiling.texel_height - 1u) /
layout.tiling.texel_height) == 16 &&
layout.mip_layout[1].pitch == 16 &&
layout.mip_layout[1].height == 24,
"block-compressed 3D mip lost its physical row or slice stride");
check_round_trip("3D BC mip", total.size, layout);
}
{
constexpr auto format = Prospero::BufferFormat::kBc3UNorm;
constexpr auto tile = Prospero::TileMode::kLinear;
constexpr u32 width = 8, height = 8, levels = 4;
TileSizeAlign total{};
TileGetTextureSize(format, width, height, levels, tile, &total, nullptr,
nullptr);
const auto layout = MakeTilingImage(format, width, height, levels, 1,
tile, total.size, false);
const auto regions = layout.BufferCopies();
Require(name, "linear BC native regions",
regions.size() == levels &&
std::all_of(regions.begin(), regions.end(),
[](const auto &region) {
return region.bufferRowLength >=
region.imageExtent.width &&
region.bufferImageHeight >=
region.imageExtent.height &&
region.bufferRowLength % 4 == 0 &&
region.bufferImageHeight % 4 == 0;
}),
"linear BC mip tails emitted invalid Vulkan copy strides");
}
struct VolumeModeCase {
Prospero::TileMode tile;
TileBlockFamily family;
Prospero::BufferFormat format;
};
constexpr VolumeModeCase volume_modes[] = {
{Prospero::TileMode::kStandard64KB, TileBlockFamily::Standard64KB3D,
Prospero::BufferFormat::k32Float},
{Prospero::TileMode::kPrt, TileBlockFamily::Prt64KB3D,
Prospero::BufferFormat::k32Float},
{Prospero::TileMode::kRenderTarget, TileBlockFamily::RenderTarget64KB,
Prospero::BufferFormat::k32Float},
{Prospero::TileMode::kDepth, TileBlockFamily::Depth64KB,
Prospero::BufferFormat::k32Float},
{Prospero::TileMode::kStandard64KB, TileBlockFamily::Standard64KB3D,
Prospero::BufferFormat::kBc1UNorm},
{Prospero::TileMode::kPrt, TileBlockFamily::Prt64KB3D,
Prospero::BufferFormat::kBc3UNorm},
};
for (const auto test : volume_modes) {
constexpr u32 width = 65, height = 33, depth = 37, levels = 6;
TileSizeAlign total{};
TileGetTextureTotalSize(test.format, width, height, depth, levels,
test.tile, true, total);
const auto layout = MakeTilingImage(test.format, width, height, levels,
depth, test.tile, total.size, true);
const auto regions = layout.BufferCopies();
const bool uses_z = test.family == TileBlockFamily::RenderTarget64KB ||
test.family == TileBlockFamily::Depth64KB;
Require(name, "volume family infos",
layout.resources.levels != 0 &&
layout.tiling.block.family == test.family &&
layout.resources.levels == levels &&
layout.MipExtent(0).depth == depth &&
(!uses_z || (layout.mip_layout[0].surface_z == 0 &&
layout.tiled_slice_stride != 0)),
"volume mode selected the wrong family or lost its surface Z");
check_round_trip("volume family", total.size, layout);
}
struct VolumeTailCase {
Prospero::BufferFormat format;
u32 bytes;
};
constexpr VolumeTailCase volume_tails[] = {
{Prospero::BufferFormat::k8UNorm, 1},
{Prospero::BufferFormat::k16UNorm, 2},
{Prospero::BufferFormat::k32Float, 4},
{Prospero::BufferFormat::k16_16_16_16Float, 8},
{Prospero::BufferFormat::k32_32_32_32Float, 16},
{Prospero::BufferFormat::kBc1UNorm, 8},
{Prospero::BufferFormat::kBc3UNorm, 16},
};
constexpr u32 volume_tail_xy[5][5][2] = {
{{0, 8}, {8, 4}, {8, 0}, {0, 4}, {0, 0}},
{{0, 8}, {4, 4}, {4, 0}, {0, 4}, {0, 0}},
{{0, 8}, {4, 4}, {4, 0}, {0, 4}, {0, 0}},
{{0, 4}, {4, 2}, {4, 0}, {0, 2}, {0, 0}},
{{0, 4}, {2, 2}, {2, 0}, {0, 2}, {0, 0}},
};
for (const auto &tail : volume_tails) {
constexpr auto tile = Prospero::TileMode::kStandard4KB;
constexpr u32 levels = 5;
TileBlockLayout block{};
Require(
name, "3D tail block",
TileGetBlockLayout(TileBlockFamily::Standard4KB3D, tail.bytes, block),
"Standard4KB3D tail format has no block layout");
const bool compressed =
Prospero::BlockCompressedBytesPerBlock(tail.format) != 0;
const u32 width = block.block_width * (compressed ? 4u : 1u);
const u32 height = block.block_height / 2u * (compressed ? 4u : 1u);
const u32 depth = block.block_depth + 1u;
TileSizeAlign total{};
TileGetTextureTotalSize(tail.format, width, height, depth, levels, tile,
true, total);
const auto layout = MakeTilingImage(tail.format, width, height, levels,
depth, tile, total.size, true);
const auto regions = layout.BufferCopies();
bool valid = total.size == 8192 && regions.size() == levels &&
layout.resources.levels == levels;
const u32 table = std::countr_zero(tail.bytes);
for (u32 level = 0; level < levels && valid; level++) {
const auto &mip = layout.mip_layout[level];
valid &=
level >= layout.first_tail_level &&
mip.tail_x == volume_tail_xy[table][level][0] &&
mip.tail_y == volume_tail_xy[table][level][1] && mip.offset == 0 &&
layout.MipExtent(level).depth == std::max(depth >> level, 1u) &&
layout.tiled_slice_stride == 4096;
}
if (valid) {
valid = layout.MipExtent(0).depth == block.block_depth + 1u &&
layout.mip_layout[0].offset + layout.tiled_slice_stride == 4096;
}
Require(
name, "3D mip tail infos", valid,
"Standard4KB3D mip tail coordinates or block-slice packing changed");
check_round_trip("3D mip tail", total.size, layout);
}
for (const auto family :
{TileBlockFamily::Standard4KB, TileBlockFamily::Standard64KB,
TileBlockFamily::Prt64KB, TileBlockFamily::RenderTarget64KB,
TileBlockFamily::Depth64KB}) {
for (u32 bpe = 1; bpe <= 16; bpe <<= 1u) {
if (family == TileBlockFamily::Depth64KB && bpe == 16)
continue;
TileBlockLayout block{};
Require(name, "tail layout", TileGetBlockLayout(family, bpe, block),
"tail family/BPE has no block layout");
const u32 width = std::max(block.block_width / 4u, 1u);
const u32 height = std::max(block.block_height / 4u, 1u);
const u32 x = block.block_width / 2u;
const u32 y = block.block_height / 2u;
const u32 pitch = width;
const uint64_t linear_size =
static_cast<uint64_t>(pitch) * height * bpe;
std::vector<uint8_t> tiled(block.block_size);
std::vector<uint8_t> cpu(linear_size, 0xcd);
std::vector<uint8_t> gpu(linear_size, 0xab);
fill(&tiled, ++case_index);
auto info =
block_image(block, width, height, 1, pitch, block.block_size);
info.first_tail_level = 0;
info.mip_layout[0].tail_x = x;
info.mip_layout[0].tail_y = y;
info.mip_layout[0].surface_z =
family == TileBlockFamily::RenderTarget64KB ||
family == TileBlockFamily::Depth64KB
? 2
: 0;
convert_reference(false, &cpu, tiled, info);
gpu_detile(tiled, &gpu, block.block_size, linear_size, info);
compare("tail bytes", cpu, gpu);
std::vector<uint8_t> linear(linear_size);
std::vector<uint8_t> cpu_tiled(block.block_size, 0xab);
std::vector<uint8_t> gpu_tiled(block.block_size, 0xab);
fill(&linear, 0x400u + case_index);
convert_reference(true, &cpu_tiled, linear, info);
gpu_tile(linear, &gpu_tiled, block.block_size, linear_size, info);
compare("tail tile bytes", cpu_tiled, gpu_tiled);
}
}
TileBlockLayout small_block{};
Require(name, "small layout",
TileGetBlockLayout(TileBlockFamily::Standard256B, 4, small_block),
"small fixture layout is unavailable");
std::vector<uint8_t> small_input(small_block.block_size);
std::vector<uint8_t> small_expected(small_block.block_size, 0);
std::vector<uint8_t> small_output(small_block.block_size, 0xab);
fill(&small_input, 0xee);
auto small_info = block_image(small_block, 1, 1, 1, small_block.block_width,
small_input.size());
convert_reference(false, &small_expected, small_input, small_info);
gpu_detile(small_input, &small_output, small_input.size(),
small_output.size(), small_info);
compare("small detile", small_expected, small_output);
std::fill(small_output.begin(), small_output.end(), 0xab);
gpu_detile(small_input, &small_output, small_input.size(),
small_output.size(), small_info);
compare("scheduler-owned reuse", small_expected, small_output);
constexpr auto volume_format = Prospero::BufferFormat::k32UInt;
constexpr auto volume_tile = Prospero::TileMode::kLinear;
constexpr u32 volume_width = 8, volume_height = 4, volume_depth = 5;
constexpr u32 volume_levels = 3;
TileSizeAlign volume_size{};
TileGetTextureTotalSize(volume_format, volume_width, volume_height,
volume_depth, volume_levels, volume_tile, true,
volume_size);
const auto volume_layout = MakeTilingImage(
volume_format, volume_width, volume_height, volume_levels, volume_depth,
volume_tile, volume_size.size, true);
const auto volume_copies = volume_layout.BufferCopies();
std::vector<u32> volume_source(volume_size.size / sizeof(u32), 0);
std::vector<std::pair<size_t, u32>> volume_probes;
for (const auto &copy : volume_copies) {
for (u32 z = 0; z < copy.imageExtent.depth; ++z) {
for (u32 y = 0; y < copy.imageExtent.height; ++y) {
for (u32 x = 0; x < copy.imageExtent.width; ++x) {
const auto index = copy.bufferOffset / sizeof(u32) +
(uint64_t{z} * copy.bufferImageHeight + y) *
copy.bufferRowLength +
x;
const u32 value =
0xa0000000u | (copy.imageSubresource.mipLevel << 24u) |
((static_cast<u32>(copy.imageOffset.z) + z) << 16u) |
(y << 8u) | x;
Require(name, "linear volume native bounds",
index < volume_source.size(),
"linear volume copy escaped its padded guest layout");
volume_source[index] = value;
volume_probes.emplace_back(static_cast<size_t>(index), value);
}
}
}
}
auto volume_upload =
CreateHostBuffer(name, volume_size.size,
vk::BufferUsageFlagBits::eTransferSrc, volume_source);
auto volume_download = CreateHostBuffer(
name, volume_size.size, vk::BufferUsageFlagBits::eTransferDst,
std::vector<u32>(volume_source.size(), 0));
auto volume_info = volume_layout;
volume_info.data.address = 0x10000;
volume_info.pixel_format = vk::Format::eR32Uint;
Libs::Graphics::Image volume_image(m_runtime_context, scheduler,
volume_info);
volume_image.Upload(volume_copies, volume_upload.buffer, 0,
volume_size.size);
volume_image.Download(volume_copies, volume_download.buffer, 0,
volume_size.size);
scheduler.Finish();
const auto volume_observed =
ReadBuffer(name, volume_download, volume_source.size());
Require(name, "linear volume native content",
std::ranges::all_of(volume_probes,
[&](const auto &probe) {
return volume_observed[probe.first] ==
probe.second;
}),
"native upload/download did not preserve padded 3D mip rows");
DestroyBuffer(&volume_download);
DestroyBuffer(&volume_upload);
std::printf("[gpu] %-32s ok (%u cases, %u format/mode pairs)\n", name,
case_index, format_cases);
}
private:
bool m_rasterization_supported = true;
u32 m_skipped_cases = 0;
RenderContext &Renderer() {
EXIT_IF(m_renderer == nullptr);
return *m_renderer;
}
void EnsureRuntimeContext() {
if (m_runtime_context.allocator != nullptr) {
return;
}
namespace Exception = Common::HostException;
Require("VulkanHarness", "guest fault handler",
Exception::InstallHandler([](const Exception::ExceptionInfo &info) {
if (info.type != Exception::ExceptionType::AccessViolation ||
(info.access_violation_type != Exception::AccessViolationType::Read &&
info.access_violation_type != Exception::AccessViolationType::Write)) {
return false;
}
const auto access =
info.access_violation_type == Exception::AccessViolationType::Write
? PageFaultAccess::Write : PageFaultAccess::Read;
return LibKernel::Memory::HandleGpuFault(access, info.access_violation_vaddr);
}),
"failed to install the production guest-memory fault route");
m_runtime_context.instance = m_instance;
m_runtime_context.physical_device = m_physical_device;
m_runtime_context.device = m_device;
m_physical_device.getProperties(
&m_runtime_context.physical_device_properties);
m_runtime_context.physical_device_memory_properties = m_memory_properties;
m_runtime_context.queue_family = m_queue_family;
m_runtime_context.queue = m_queue;
m_runtime_context.shader_image_int64_atomics_enabled = true;
m_runtime_context.maintenance8_enabled = true;
m_runtime_context.attachment_feedback_loop_enabled = m_rasterization_supported;
m_runtime_context.provoking_vertex_last_enabled = m_rasterization_supported;
const vk::PhysicalDeviceImageFormatInfo2 block_texel_view_info{
.format = vk::Format::eBc1RgbaUnormBlock,
.type = vk::ImageType::e2D,
.tiling = vk::ImageTiling::eOptimal,
.usage = vk::ImageUsageFlagBits::eSampled,
.flags = vk::ImageCreateFlagBits::eBlockTexelViewCompatible,
};
const auto block_texel_view_props =
m_physical_device.getImageFormatProperties2(block_texel_view_info);
m_runtime_context.supports_block_texel_view =
block_texel_view_props.result == vk::Result::eSuccess;
VmaVulkanFunctions functions{};
functions.vkGetInstanceProcAddr =
VULKAN_HPP_DEFAULT_DISPATCHER.vkGetInstanceProcAddr;
functions.vkGetDeviceProcAddr =
VULKAN_HPP_DEFAULT_DISPATCHER.vkGetDeviceProcAddr;
VmaAllocatorCreateInfo allocator_info{};
allocator_info.instance = m_instance;
allocator_info.physicalDevice = m_physical_device;
allocator_info.device = m_device;
allocator_info.pVulkanFunctions = &functions;
allocator_info.vulkanApiVersion = VK_API_VERSION_1_3;
allocator_info.flags = VMA_ALLOCATOR_CREATE_BUFFER_DEVICE_ADDRESS_BIT;
RequireVk("VulkanHarness", "runtime context",
static_cast<vk::Result>(vmaCreateAllocator(
&allocator_info, &m_runtime_context.allocator)),
"vmaCreateAllocator");
m_renderer = std::make_unique<RenderContext>(m_runtime_context);
}
void Init() {
static vk::detail::DynamicLoader loader;
const auto get_instance_proc_addr =
loader.getProcAddress<PFN_vkGetInstanceProcAddr>(
"vkGetInstanceProcAddr");
Require("VulkanHarness", "dispatch", get_instance_proc_addr != nullptr,
"could not load the Vulkan loader");
VULKAN_HPP_DEFAULT_DISPATCHER.init(get_instance_proc_addr);
vk::ApplicationInfo app{};
app.sType = vk::StructureType::eApplicationInfo;
app.pApplicationName = "ShaderRecompilerComputeTests";
app.apiVersion = VK_API_VERSION_1_3;
vk::InstanceCreateInfo instance_info{};
instance_info.sType = vk::StructureType::eInstanceCreateInfo;
instance_info.pApplicationInfo = &app;
RequireVk("VulkanHarness", "dispatch",
vk::createInstance(&instance_info, nullptr, &m_instance),
"vkCreateInstance");
VULKAN_HPP_DEFAULT_DISPATCHER.init(m_instance);
u32 physical_count = 0;
RequireVk("VulkanHarness", "dispatch",
m_instance.enumeratePhysicalDevices(&physical_count, nullptr),
"vkEnumeratePhysicalDevices");
Require("VulkanHarness", "dispatch", physical_count != 0,
"no Vulkan physical devices");
std::vector<vk::PhysicalDevice> physical_devices(physical_count);
RequireVk("VulkanHarness", "dispatch",
m_instance.enumeratePhysicalDevices(&physical_count,
physical_devices.data()),
"vkEnumeratePhysicalDevices");
for (auto physical : physical_devices) {
u32 queue_count = 0;
physical.getQueueFamilyProperties(&queue_count, nullptr);
std::vector<vk::QueueFamilyProperties> queues(queue_count);
physical.getQueueFamilyProperties(&queue_count, queues.data());
for (u32 i = 0; i < queue_count; i++) {
if ((queues[i].queueFlags &
(vk::QueueFlagBits::eCompute | vk::QueueFlagBits::eGraphics)) ==
(vk::QueueFlagBits::eCompute | vk::QueueFlagBits::eGraphics)) {
vk::PhysicalDeviceVulkan11Features features11{};
vk::PhysicalDeviceVulkan12Features features12{};
features12.sType = vk::StructureType::ePhysicalDeviceVulkan12Features;
vk::PhysicalDeviceWorkgroupMemoryExplicitLayoutFeaturesKHR workgroup_layout{};
features12.pNext = &features11;
features11.pNext = &workgroup_layout;
vk::PhysicalDeviceShaderImageAtomicInt64FeaturesEXT image_atomic64{};
workgroup_layout.pNext = &image_atomic64;
vk::PhysicalDeviceFragmentShaderBarycentricFeaturesKHR barycentric{};
barycentric.sType = vk::StructureType::ePhysicalDeviceFragmentShaderBarycentricFeaturesKHR;
barycentric.pNext = &features12;
vk::PhysicalDeviceFeatures2 features{};
features.sType = vk::StructureType::ePhysicalDeviceFeatures2;
features.pNext = &barycentric;
physical.getFeatures2(&features);
if (barycentric.fragmentShaderBarycentric != true ||
features.features.shaderInt64 != true ||
features11.storageBuffer16BitAccess != true ||
features12.storageBuffer8BitAccess != true ||
features12.samplerMirrorClampToEdge != true ||
features12.shaderOutputViewportIndex != true ||
features12.shaderBufferInt64Atomics != true ||
features12.shaderSampledImageArrayNonUniformIndexing != true ||
features12.shaderStorageImageArrayNonUniformIndexing != true ||
features12.shaderStorageBufferArrayNonUniformIndexing != true ||
features12.shaderSharedInt64Atomics != true ||
image_atomic64.shaderImageInt64Atomics != true ||
workgroup_layout.workgroupMemoryExplicitLayout != true ||
features12.bufferDeviceAddress != true) {
continue;
}
m_physical_device = physical;
m_queue_family = i;
break;
}
}
if (m_physical_device != nullptr) {
break;
}
}
Require("VulkanHarness", "dispatch", m_physical_device != nullptr,
"no Vulkan graphics+compute device with fragment barycentrics and 64-bit LDS/image atomics");
m_physical_device.getMemoryProperties(&m_memory_properties);
vk::PhysicalDeviceFeatures available_features{};
m_physical_device.getFeatures(&available_features);
vk::PhysicalDeviceVulkan11Features available_features11{};
vk::PhysicalDeviceVulkan12Features available_features12{};
available_features12.pNext = &available_features11;
available_features12.sType =
vk::StructureType::ePhysicalDeviceVulkan12Features;
vk::PhysicalDeviceVulkan13Features available_features13{};
available_features13.sType =
vk::StructureType::ePhysicalDeviceVulkan13Features;
available_features13.pNext = &available_features12;
vk::PhysicalDeviceComputeShaderDerivativesFeaturesKHR available_derivatives{};
available_derivatives.pNext = &available_features13;
vk::PhysicalDeviceDepthClipEnableFeaturesEXT available_depth_clip{};
available_depth_clip.pNext = &available_derivatives;
vk::PhysicalDeviceDepthClipControlFeaturesEXT available_clip_control{};
available_clip_control.pNext = &available_depth_clip;
vk::PhysicalDeviceColorWriteEnableFeaturesEXT available_color_write{};
available_color_write.pNext = &available_clip_control;
vk::PhysicalDeviceAttachmentFeedbackLoopLayoutFeaturesEXT available_feedback_layout{};
available_feedback_layout.pNext = &available_color_write;
vk::PhysicalDeviceAttachmentFeedbackLoopDynamicStateFeaturesEXT available_feedback_dynamic{};
available_feedback_dynamic.pNext = &available_feedback_layout;
vk::PhysicalDeviceProvokingVertexFeaturesEXT available_provoking_vertex{};
available_provoking_vertex.pNext = &available_feedback_dynamic;
vk::PhysicalDeviceImageViewMinLodFeaturesEXT available_min_lod{};
available_min_lod.pNext = &available_provoking_vertex;
vk::PhysicalDeviceMaintenance8FeaturesKHR available_maintenance8{};
available_maintenance8.pNext = &available_min_lod;
vk::PhysicalDeviceRobustness2FeaturesEXT available_robustness2{};
available_robustness2.pNext = &available_maintenance8;
vk::PhysicalDeviceFeatures2 available_features2{};
available_features2.sType = vk::StructureType::ePhysicalDeviceFeatures2;
available_features2.pNext = &available_robustness2;
m_physical_device.getFeatures2(&available_features2);
Require("VulkanHarness", "dispatch", available_robustness2.nullDescriptor == true,
"null descriptors are not supported");
Require("VulkanHarness", "dispatch",
available_features.shaderStorageImageWriteWithoutFormat == true,
"shaderStorageImageWriteWithoutFormat is not supported");
Require("VulkanHarness", "dispatch",
available_features.shaderImageGatherExtended == true &&
available_derivatives.computeDerivativeGroupQuads == true,
"image gather or compute derivative quads are not supported");
Require("VulkanHarness", "dispatch",
available_features11.storageBuffer16BitAccess == true &&
available_features12.storageBuffer8BitAccess == true,
"8-bit and 16-bit storage buffer access are not supported");
Require("VulkanHarness", "dispatch",
available_features12.timelineSemaphore == true,
"timeline semaphores are not supported");
Require("VulkanHarness", "dispatch",
available_features13.dynamicRendering == true,
"dynamic rendering is not supported");
Require("VulkanHarness", "dispatch",
available_features13.synchronization2 == true,
"synchronization2 is not supported");
Require("VulkanHarness", "dispatch",
available_features.sampleRateShading == true,
"sample-rate shading is not supported");
Require("VulkanHarness", "dispatch", available_features.shaderInt64 == true,
"shaderInt64 is not supported");
Require("VulkanHarness", "dispatch",
available_features12.bufferDeviceAddress == true,
"bufferDeviceAddress is not supported");
Require("VulkanHarness", "dispatch",
available_features12.shaderSampledImageArrayNonUniformIndexing == true,
"nonuniform sampled image indexing is not supported");
Require("VulkanHarness", "dispatch",
available_features12.shaderStorageBufferArrayNonUniformIndexing == true,
"nonuniform storage buffer indexing is not supported");
Require("VulkanHarness", "dispatch",
available_features12.shaderStorageImageArrayNonUniformIndexing == true,
"nonuniform storage image indexing is not supported");
Require("VulkanHarness", "dispatch", available_min_lod.minLod == true,
"image view minimum LOD is not supported");
Require("VulkanHarness", "dispatch", available_maintenance8.maintenance8 == true,
"maintenance8 dynamic image offsets are not supported");
Require("VulkanHarness", "graphics", available_features12.shaderOutputLayer == true,
"vertex layer output is not supported");
m_rasterization_supported = available_features.fillModeNonSolid &&
available_features.tessellationShader &&
available_features.depthBounds &&
available_depth_clip.depthClipEnable &&
available_clip_control.depthClipControl &&
available_color_write.colorWriteEnable &&
available_feedback_layout.attachmentFeedbackLoopLayout &&
available_feedback_dynamic.attachmentFeedbackLoopDynamicState &&
available_provoking_vertex.provokingVertexLast;
if (!m_rasterization_supported) {
std::printf(
"[host] ProductionRasterization unavailable, rasterization cases "
"will be skipped\n");
}
float priority = 1.0f;
vk::DeviceQueueCreateInfo queue_info{};
queue_info.sType = vk::StructureType::eDeviceQueueCreateInfo;
queue_info.queueFamilyIndex = m_queue_family;
queue_info.queueCount = 1;
queue_info.pQueuePriorities = &priority;
vk::DeviceCreateInfo device_info{};
device_info.sType = vk::StructureType::eDeviceCreateInfo;
device_info.queueCreateInfoCount = 1;
device_info.pQueueCreateInfos = &queue_info;
auto device_features11 = WindowContext::RequiredVulkan11Features();
auto device_features12 = WindowContext::RequiredVulkan12Features();
device_features12.shaderSharedInt64Atomics = true;
vk::PhysicalDeviceWorkgroupMemoryExplicitLayoutFeaturesKHR workgroup_layout{};
workgroup_layout.workgroupMemoryExplicitLayout = true;
device_features12.pNext = &device_features11;
device_features11.pNext = &workgroup_layout;
vk::PhysicalDeviceShaderImageAtomicInt64FeaturesEXT image_atomic64{};
image_atomic64.shaderImageInt64Atomics = true;
workgroup_layout.pNext = &image_atomic64;
vk::PhysicalDeviceFragmentShaderBarycentricFeaturesKHR barycentric{};
barycentric.sType = vk::StructureType::ePhysicalDeviceFragmentShaderBarycentricFeaturesKHR;
barycentric.pNext = &device_features12;
barycentric.fragmentShaderBarycentric = true;
vk::PhysicalDeviceVulkan13Features device_features13{};
device_features13.sType =
vk::StructureType::ePhysicalDeviceVulkan13Features;
device_features13.pNext = &barycentric;
device_features13.dynamicRendering = true;
device_features13.synchronization2 = true;
vk::PhysicalDeviceComputeShaderDerivativesFeaturesKHR derivatives{};
derivatives.pNext = &device_features13;
derivatives.computeDerivativeGroupQuads = true;
// Requesting a feature the device does not support fails device creation, so the
// rasterization feature chain is only chained in when every part is available.
vk::PhysicalDeviceDepthClipEnableFeaturesEXT depth_clip{};
vk::PhysicalDeviceDepthClipControlFeaturesEXT clip_control{};
vk::PhysicalDeviceColorWriteEnableFeaturesEXT color_write{};
vk::PhysicalDeviceAttachmentFeedbackLoopLayoutFeaturesEXT feedback_layout{};
vk::PhysicalDeviceAttachmentFeedbackLoopDynamicStateFeaturesEXT feedback_dynamic{};
vk::PhysicalDeviceProvokingVertexFeaturesEXT provoking_vertex{};
if (m_rasterization_supported) {
depth_clip.pNext = &derivatives;
depth_clip.depthClipEnable = true;
clip_control.pNext = &depth_clip;
clip_control.depthClipControl = true;
color_write.pNext = &clip_control;
color_write.colorWriteEnable = true;
feedback_layout.pNext = &color_write;
feedback_layout.attachmentFeedbackLoopLayout = true;
feedback_dynamic.pNext = &feedback_layout;
feedback_dynamic.attachmentFeedbackLoopDynamicState = true;
provoking_vertex.pNext = &feedback_dynamic;
provoking_vertex.provokingVertexLast = available_provoking_vertex.provokingVertexLast;
}
vk::PhysicalDeviceImageViewMinLodFeaturesEXT min_lod{};
min_lod.minLod = true;
min_lod.pNext = m_rasterization_supported
? static_cast<void *>(&provoking_vertex)
: static_cast<void *>(&derivatives);
vk::PhysicalDeviceMaintenance8FeaturesKHR maintenance8{};
maintenance8.maintenance8 = true;
maintenance8.pNext = &min_lod;
vk::PhysicalDeviceRobustness2FeaturesEXT robustness2{};
robustness2.nullDescriptor = true;
robustness2.pNext = &maintenance8;
device_info.pNext = &robustness2;
vk::PhysicalDeviceFeatures device_features{};
device_features.shaderStorageImageWriteWithoutFormat = true;
device_features.shaderImageGatherExtended = true;
device_features.sampleRateShading = true;
device_features.shaderInt64 = true;
device_features.shaderFloat64 = available_features.shaderFloat64;
device_features.fillModeNonSolid = m_rasterization_supported;
device_features.tessellationShader = m_rasterization_supported;
device_features.depthBounds = m_rasterization_supported;
device_info.pEnabledFeatures = &device_features;
std::vector<const char *> device_extensions{
VK_KHR_PUSH_DESCRIPTOR_EXTENSION_NAME,
VK_KHR_COMPUTE_SHADER_DERIVATIVES_EXTENSION_NAME,
VK_KHR_FRAGMENT_SHADER_BARYCENTRIC_EXTENSION_NAME,
VK_KHR_WORKGROUP_MEMORY_EXPLICIT_LAYOUT_EXTENSION_NAME,
VK_EXT_SHADER_IMAGE_ATOMIC_INT64_EXTENSION_NAME,
VK_EXT_ROBUSTNESS_2_EXTENSION_NAME,
VK_KHR_MAINTENANCE_8_EXTENSION_NAME,
VK_EXT_IMAGE_VIEW_MIN_LOD_EXTENSION_NAME};
if (m_rasterization_supported) {
device_extensions.push_back(VK_EXT_DEPTH_CLIP_ENABLE_EXTENSION_NAME);
device_extensions.push_back(VK_EXT_DEPTH_CLIP_CONTROL_EXTENSION_NAME);
device_extensions.push_back(VK_EXT_COLOR_WRITE_ENABLE_EXTENSION_NAME);
device_extensions.push_back(VK_EXT_ATTACHMENT_FEEDBACK_LOOP_LAYOUT_EXTENSION_NAME);
device_extensions.push_back(
VK_EXT_ATTACHMENT_FEEDBACK_LOOP_DYNAMIC_STATE_EXTENSION_NAME);
device_extensions.push_back(VK_EXT_PROVOKING_VERTEX_EXTENSION_NAME);
}
device_info.enabledExtensionCount =
static_cast<uint32_t>(device_extensions.size());
device_info.ppEnabledExtensionNames = device_extensions.data();
RequireVk("VulkanHarness", "dispatch",
m_physical_device.createDevice(&device_info, nullptr, &m_device),
"vkCreateDevice");
VULKAN_HPP_DEFAULT_DISPATCHER.init(m_device);
m_device.getQueue(m_queue_family, 0, &m_queue);
vk::CommandPoolCreateInfo pool_info{};
pool_info.sType = vk::StructureType::eCommandPoolCreateInfo;
pool_info.queueFamilyIndex = m_queue_family;
pool_info.flags = vk::CommandPoolCreateFlagBits::eResetCommandBuffer;
RequireVk("VulkanHarness", "dispatch",
m_device.createCommandPool(&pool_info, nullptr, &m_command_pool),
"vkCreateCommandPool");
}
void Destroy() {
if (m_device != nullptr) {
RequireVulkanSuccess(m_device.waitIdle(), "vkDeviceWaitIdle");
DestroyBuffer(&m_fault_buffer);
DestroyBuffer(&m_bda_pagetable_buffer);
if (m_runtime_context.allocator != nullptr) {
m_renderer.reset();
vmaDestroyAllocator(m_runtime_context.allocator);
m_runtime_context.allocator = nullptr;
}
if (m_command_pool != nullptr) {
m_device.destroyCommandPool(m_command_pool, nullptr);
}
m_device.destroy(nullptr);
}
if (m_instance != nullptr) {
m_instance.destroy(nullptr);
}
}
bool FindMemoryType(u32 type_bits, vk::MemoryPropertyFlags required,
u32 *index) const {
for (u32 i = 0; i < m_memory_properties.memoryTypeCount; i++) {
if ((type_bits & (1u << i)) == 0) {
continue;
}
if ((m_memory_properties.memoryTypes[i].propertyFlags & required) ==
required) {
*index = i;
return true;
}
}
return false;
}
static vk::AccessFlags AccessForLayout(vk::ImageLayout layout) {
switch (layout) {
case vk::ImageLayout::eTransferDstOptimal:
return vk::AccessFlagBits::eTransferWrite;
case vk::ImageLayout::eTransferSrcOptimal:
return vk::AccessFlagBits::eTransferRead;
case vk::ImageLayout::eShaderReadOnlyOptimal:
return vk::AccessFlagBits::eShaderRead;
case vk::ImageLayout::eGeneral:
return vk::AccessFlagBits::eShaderRead | vk::AccessFlagBits::eShaderWrite;
default:
return {};
}
}
vk::ShaderModule CreateShaderModule(const char *shader_name, const std::vector<u32> &spirv) {
vk::ShaderModuleCreateInfo info{};
info.codeSize = spirv.size() * sizeof(u32);
info.pCode = spirv.data();
vk::ShaderModule module{};
RequireVk(shader_name, "graphics", m_device.createShaderModule(&info, nullptr, &module),
"vkCreateShaderModule");
return module;
}
vk::CommandBuffer BeginCommands(const char *shader_name, const char *stage) {
vk::CommandBufferAllocateInfo cmd_alloc{};
cmd_alloc.sType = vk::StructureType::eCommandBufferAllocateInfo;
cmd_alloc.commandPool = m_command_pool;
cmd_alloc.level = vk::CommandBufferLevel::ePrimary;
cmd_alloc.commandBufferCount = 1;
vk::CommandBuffer cmd = nullptr;
RequireVk(shader_name, stage,
m_device.allocateCommandBuffers(&cmd_alloc, &cmd),
"vkAllocateCommandBuffers");
vk::CommandBufferBeginInfo begin{};
begin.sType = vk::StructureType::eCommandBufferBeginInfo;
begin.flags = vk::CommandBufferUsageFlagBits::eOneTimeSubmit;
RequireVk(shader_name, stage, cmd.begin(&begin), "vkBeginCommandBuffer");
return cmd;
}
void EndSubmitAndFree(const char *shader_name, const char *stage,
vk::CommandBuffer cmd) {
RequireVk(shader_name, stage, cmd.end(), "vkEndCommandBuffer");
vk::FenceCreateInfo fence_info{};
fence_info.sType = vk::StructureType::eFenceCreateInfo;
vk::Fence fence = nullptr;
RequireVk(shader_name, stage,
m_device.createFence(&fence_info, nullptr, &fence),
"vkCreateFence");
vk::SubmitInfo submit{};
submit.sType = vk::StructureType::eSubmitInfo;
submit.commandBufferCount = 1;
submit.pCommandBuffers = &cmd;
RequireVk(shader_name, stage, m_queue.submit(1, &submit, fence),
"vkQueueSubmit");
RequireVk(shader_name, stage,
m_device.waitForFences(1, &fence, true, UINT64_MAX),
"vkWaitForFences");
m_device.destroyFence(fence, nullptr);
m_device.freeCommandBuffers(m_command_pool, 1, &cmd);
}
void AddImageBarrier(vk::CommandBuffer cmd, vk::Image image,
vk::ImageLayout old_layout, vk::ImageLayout new_layout,
vk::PipelineStageFlags src_stage,
vk::PipelineStageFlags dst_stage,
vk::AccessFlags src_access, vk::AccessFlags dst_access,
u32 mip_levels = 1, u32 layers = 1) {
vk::ImageMemoryBarrier barrier{};
barrier.sType = vk::StructureType::eImageMemoryBarrier;
barrier.srcAccessMask = src_access;
barrier.dstAccessMask = dst_access;
barrier.oldLayout = old_layout;
barrier.newLayout = new_layout;
barrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
barrier.image = image;
barrier.subresourceRange.aspectMask = vk::ImageAspectFlagBits::eColor;
barrier.subresourceRange.baseMipLevel = 0;
barrier.subresourceRange.levelCount = mip_levels;
barrier.subresourceRange.baseArrayLayer = 0;
barrier.subresourceRange.layerCount = layers;
cmd.pipelineBarrier(src_stage, dst_stage, {}, 0, nullptr, 0, nullptr, 1,
&barrier);
}
void TransitionImage(const char *shader_name, Image *image,
vk::ImageLayout new_layout,
vk::PipelineStageFlags src_stage,
vk::PipelineStageFlags dst_stage,
vk::AccessFlags src_access, vk::AccessFlags dst_access) {
vk::CommandBuffer cmd = BeginCommands(shader_name, "dispatch");
AddImageBarrier(cmd, image->image, image->layout, new_layout, src_stage,
dst_stage, src_access, dst_access, image->mip_levels,
image->layers);
EndSubmitAndFree(shader_name, "dispatch", cmd);
image->layout = new_layout;
}
void UploadImage(const char *shader_name, Image *image, vk::Buffer staging,
vk::ImageLayout final_layout) {
UploadImageMips(shader_name, image, staging, final_layout);
}
void UploadImageMips(const char *shader_name, Image *image,
vk::Buffer staging, vk::ImageLayout final_layout) {
vk::CommandBuffer cmd = BeginCommands(shader_name, "dispatch");
AddImageBarrier(cmd, image->image, vk::ImageLayout::eUndefined,
vk::ImageLayout::eTransferDstOptimal,
vk::PipelineStageFlagBits::eTopOfPipe,
vk::PipelineStageFlagBits::eTransfer, {},
vk::AccessFlagBits::eTransferWrite, image->mip_levels,
image->layers);
std::vector<vk::BufferImageCopy> copies;
copies.reserve(image->mip_levels);
vk::DeviceSize offset = 0;
for (u32 level = 0; level < image->mip_levels; level++) {
vk::BufferImageCopy copy{};
copy.bufferOffset = offset;
copy.imageSubresource.aspectMask = vk::ImageAspectFlagBits::eColor;
copy.imageSubresource.mipLevel = level;
copy.imageSubresource.baseArrayLayer = 0;
copy.imageSubresource.layerCount = image->layers;
copy.imageExtent.width = MipExtent(image->width, level);
copy.imageExtent.height = MipExtent(image->height, level);
copy.imageExtent.depth = 1;
copies.push_back(copy);
offset += static_cast<vk::DeviceSize>(
ImageMipDwordCount(image->width, image->height,
image->format, level, image->layers) *
sizeof(u32));
}
cmd.copyBufferToImage(staging, image->image,
vk::ImageLayout::eTransferDstOptimal,
static_cast<u32>(copies.size()), copies.data());
AddImageBarrier(cmd, image->image, vk::ImageLayout::eTransferDstOptimal,
final_layout, vk::PipelineStageFlagBits::eTransfer,
vk::PipelineStageFlagBits::eComputeShader,
vk::AccessFlagBits::eTransferWrite,
AccessForLayout(final_layout), image->mip_levels,
image->layers);
EndSubmitAndFree(shader_name, "dispatch", cmd);
image->layout = final_layout;
}
Buffer CreateDeviceBuffer(const char *shader_name, vk::DeviceSize size,
vk::BufferUsageFlags usage) {
Buffer ret;
ret.size = size;
vk::BufferCreateInfo buffer_info{};
buffer_info.sType = vk::StructureType::eBufferCreateInfo;
buffer_info.size = size;
buffer_info.usage = usage;
buffer_info.sharingMode = vk::SharingMode::eExclusive;
RequireVk(shader_name, "dispatch",
m_device.createBuffer(&buffer_info, nullptr, &ret.buffer),
"vkCreateBuffer");
vk::MemoryRequirements requirements{};
m_device.getBufferMemoryRequirements(ret.buffer, &requirements);
u32 memory_type = 0;
Require(shader_name, "dispatch",
FindMemoryType(requirements.memoryTypeBits,
vk::MemoryPropertyFlagBits::eDeviceLocal,
&memory_type) ||
FindMemoryType(requirements.memoryTypeBits, {}, &memory_type),
"no memory type for BDA support buffer");
vk::MemoryAllocateInfo allocation{};
allocation.sType = vk::StructureType::eMemoryAllocateInfo;
allocation.allocationSize = requirements.size;
allocation.memoryTypeIndex = memory_type;
RequireVk(shader_name, "dispatch",
m_device.allocateMemory(&allocation, nullptr, &ret.memory),
"vkAllocateMemory");
RequireVk(shader_name, "dispatch",
m_device.bindBufferMemory(ret.buffer, ret.memory, 0),
"vkBindBufferMemory");
return ret;
}
void EnsureBdaBuffers(const char *shader_name) {
if (m_bda_pagetable_buffer.buffer != nullptr) {
return;
}
const auto usage = vk::BufferUsageFlagBits::eStorageBuffer |
vk::BufferUsageFlagBits::eTransferDst;
m_bda_pagetable_buffer =
CreateDeviceBuffer(shader_name, BufferCache::BDA_PAGETABLE_SIZE, usage);
m_fault_buffer = CreateDeviceBuffer(
shader_name, BufferCache::CACHING_NUMPAGES / 8, usage | vk::BufferUsageFlagBits::eTransferSrc);
}
Buffer CreateHostBuffer(const char *shader_name, vk::DeviceSize size,
vk::BufferUsageFlags usage,
const std::vector<u32> &contents) {
Buffer ret;
ret.size = std::max<vk::DeviceSize>(size, sizeof(u32));
vk::BufferCreateInfo buffer_info{};
buffer_info.sType = vk::StructureType::eBufferCreateInfo;
buffer_info.size = ret.size;
buffer_info.usage = usage;
buffer_info.sharingMode = vk::SharingMode::eExclusive;
RequireVk(shader_name, "dispatch",
m_device.createBuffer(&buffer_info, nullptr, &ret.buffer),
"vkCreateBuffer");
vk::MemoryRequirements req{};
m_device.getBufferMemoryRequirements(ret.buffer, &req);
u32 memory_type = 0;
ret.coherent = FindMemoryType(req.memoryTypeBits,
vk::MemoryPropertyFlagBits::eHostVisible |
vk::MemoryPropertyFlagBits::eHostCoherent,
&memory_type);
if (!ret.coherent) {
Require(shader_name, "dispatch",
FindMemoryType(req.memoryTypeBits,
vk::MemoryPropertyFlagBits::eHostVisible,
&memory_type),
"no host-visible memory type for staging buffer");
}
vk::MemoryAllocateInfo alloc{};
alloc.sType = vk::StructureType::eMemoryAllocateInfo;
alloc.allocationSize = req.size;
alloc.memoryTypeIndex = memory_type;
RequireVk(shader_name, "dispatch",
m_device.allocateMemory(&alloc, nullptr, &ret.memory),
"vkAllocateMemory");
RequireVk(shader_name, "dispatch",
m_device.bindBufferMemory(ret.buffer, ret.memory, 0),
"vkBindBufferMemory");
if (!contents.empty()) {
WriteBuffer(shader_name, ret, contents);
}
return ret;
}
void WriteBuffer(const char *shader_name, const Buffer &buffer,
const std::vector<u32> &contents) {
void *data = nullptr;
RequireVk(shader_name, "dispatch",
m_device.mapMemory(buffer.memory, 0, buffer.size, {}, &data),
"vkMapMemory");
std::memcpy(data, contents.data(), contents.size() * sizeof(u32));
if (!buffer.coherent) {
vk::MappedMemoryRange range{};
range.sType = vk::StructureType::eMappedMemoryRange;
range.memory = buffer.memory;
range.offset = 0;
range.size = VK_WHOLE_SIZE;
RequireVk(shader_name, "dispatch",
m_device.flushMappedMemoryRanges(1, &range),
"vkFlushMappedMemoryRanges");
}
m_device.unmapMemory(buffer.memory);
}
vk::Instance m_instance = nullptr;
vk::PhysicalDevice m_physical_device = nullptr;
vk::Device m_device = nullptr;
vk::Queue m_queue = nullptr;
vk::CommandPool m_command_pool = nullptr;
u32 m_queue_family = 0;
vk::PhysicalDeviceMemoryProperties m_memory_properties{};
Buffer m_bda_pagetable_buffer;
Buffer m_fault_buffer;
GraphicContext m_runtime_context{};
std::unique_ptr<RenderContext> m_renderer;
};
void CompareWords(const TestCase &test, const char *stage,
const std::vector<u32> &expected,
const std::vector<u32> &actual) {
if (actual == expected) {
return;
}
if (test.expected_float_tolerance > 0.0f &&
std::equal(expected.begin(), expected.end(), actual.begin(), actual.end(),
[&](u32 expected_word, u32 actual_word) {
return std::abs(std::bit_cast<float>(expected_word) -
std::bit_cast<float>(actual_word)) <=
test.expected_float_tolerance;
})) {
return;
}
std::ostringstream out;
out << "expected [";
for (size_t i = 0; i < expected.size(); i++) {
out << (i == 0 ? "" : ", ") << Hex(expected[i]);
}
out << "] actual [";
for (size_t i = 0; i < actual.size(); i++) {
out << (i == 0 ? "" : ", ") << Hex(actual[i]);
}
out << "]";
Fail(test.name, stage, out.str());
}
void CompareGraphicsWords(const GraphicsCase &test,
const std::vector<u32> &actual) {
if (actual == test.expected_pixel) {
return;
}
std::ostringstream out;
out << "expected [";
for (size_t i = 0; i < test.expected_pixel.size(); i++) {
out << (i == 0 ? "" : ", ") << Hex(test.expected_pixel[i]);
}
out << "] actual [";
for (size_t i = 0; i < actual.size(); i++) {
out << (i == 0 ? "" : ", ") << Hex(actual[i]);
}
out << "]";
Fail(test.name, "graphics readback", out.str());
}
void RunCase(VulkanHarness *vulkan, const TestCase &test) {
auto compiled = CompileCase(test, vulkan != nullptr ? vulkan->SubgroupSize() : 64u);
if (test.image_descriptor_swizzle != DstSel(4, 5, 6, 7) &&
!compiled.program.info.images.empty()) {
Require(test.name, "resource specialization",
compiled.program.info.images[0].shader_swizzle ==
test.image_descriptor_swizzle,
"storage image descriptor swizzle did not reach the specialized "
"program");
}
if (test.expected_force_point_sampler) {
Require(test.name, "sampler specialization",
compiled.program.info.samplers.size() == 1u &&
compiled.program.info.samplers[0].force_point_filtering,
"bit-packed sampled image did not force point filtering");
}
if (test.sampler_filter == vk::Filter::eLinear) {
Require(test.name, "sampler specialization",
std::ranges::none_of(compiled.program.info.samplers,
[](const auto &sampler) { return sampler.force_point_filtering; }),
"filterable sampled image incorrectly forced point filtering");
}
if (test.compile_only) {
std::printf("[compute] %-32s ok\n", test.name);
return;
}
using Kind = ShaderRecompiler::IR::DescriptorBindingKind;
auto Has = [&](Kind kind) {
return compiled.program.bindings.descriptor_counts[static_cast<size_t>(kind)] != 0;
};
const auto dwords = std::max<size_t>(
{test.initial.size(), test.expected.size(), static_cast<size_t>(1)});
auto buffer = vulkan->CreateStorageBuffer(
test.name, test.initial, dwords, Has(Kind::BdaPagetable));
VulkanHarness::Image sampled_image;
VulkanHarness::Image storage_image;
VulkanHarness::Image storage_image_uint;
VulkanHarness::Buffer gds_buffer;
vk::Sampler sampler = nullptr;
bool needs_sampled_image = false;
bool needs_storage_image = false;
for (const auto &image : compiled.program.info.images) {
const auto resource_class = image.resource_class;
if (resource_class == ShaderRecompiler::IR::ImageResourceClass::Sampled) {
needs_sampled_image = true;
} else if (resource_class ==
ShaderRecompiler::IR::ImageResourceClass::Storage) {
needs_storage_image = true;
}
}
const bool needs_sampler = Has(Kind::Samplers);
const bool needs_gds = Has(Kind::Gds);
if (needs_gds) {
const auto gds_dwords = std::max<size_t>(
{test.gds_initial.size(), test.expected_gds.size(), 1u});
gds_buffer =
vulkan->CreateStorageBuffer(test.name, test.gds_initial, gds_dwords);
}
if (needs_sampled_image) {
auto sampled_mips = test.sampled_image_rgba_mips;
auto sampled_format = test.sampled_image_format;
auto sampled_dwords_per_pixel = test.sampled_image_dwords_per_pixel;
if (!test.sampled_image_rgba_mips.empty()) {
sampled_format = vk::Format::eR32G32B32A32Sfloat;
sampled_dwords_per_pixel = 4;
} else if (!test.sampled_image_rgba.empty()) {
sampled_mips.push_back(test.sampled_image_rgba);
}
sampled_image = vulkan->CreateImageMips(
test.name, test.image_width, test.image_height, sampled_format,
vk::ImageUsageFlagBits::eSampled, sampled_mips,
sampled_dwords_per_pixel, vk::ImageLayout::eShaderReadOnlyOptimal,
test.sampled_image_type, test.sampled_image_view_type,
test.sampled_image_layers, test.sampled_image_view_base_layer,
test.sampled_image_view_layers, vk::SampleCountFlagBits::e1,
vk::Format::eUndefined, test.sampled_image_view_base_mip);
}
if (needs_storage_image) {
const bool atomic64 = std::ranges::any_of(
compiled.program.info.images,
[](const auto &image) { return image.atomic64; });
storage_image = vulkan->CreateImage2D(
test.name, test.image_width, test.image_height,
test.storage_image_format, vk::ImageUsageFlagBits::eStorage,
test.storage_image_rgba, test.storage_image_dwords_per_pixel,
vk::ImageLayout::eGeneral);
storage_image_uint = vulkan->CreateImage2D(
test.name, test.image_width, test.image_height,
atomic64 ? vk::Format::eR32G32Uint : vk::Format::eR32Uint,
vk::ImageUsageFlagBits::eStorage, test.storage_image_r32ui, atomic64 ? 2 : 1,
vk::ImageLayout::eGeneral,
atomic64 ? vk::Format::eR64Uint : vk::Format::eR32Uint);
}
if (needs_sampler) {
sampler = vulkan->CreateSampler(
test.name, sampled_image.mip_levels - test.sampled_image_view_base_mip,
test.sampler_filter);
}
vulkan->Dispatch(test, compiled, buffer, needs_gds ? &gds_buffer : nullptr,
needs_sampled_image ? &sampled_image : nullptr,
needs_storage_image ? &storage_image : nullptr,
needs_storage_image ? &storage_image_uint : nullptr,
sampler);
auto actual = vulkan->ReadBuffer(test.name, buffer, test.expected.size());
if (!test.expected_gds.empty()) {
const auto gds_actual =
vulkan->ReadBuffer(test.name, gds_buffer, test.expected_gds.size());
CompareWords(test, "GDS readback", test.expected_gds, gds_actual);
}
if (!test.expected_storage_image_rgba.empty()) {
auto image_actual = vulkan->ReadImage(test.name, &storage_image);
image_actual.resize(test.expected_storage_image_rgba.size());
CompareWords(test, "storage image readback",
test.expected_storage_image_rgba, image_actual);
}
if (!test.expected_storage_image_r32ui.empty()) {
auto image_actual = vulkan->ReadImage(test.name, &storage_image_uint);
image_actual.resize(test.expected_storage_image_r32ui.size());
CompareWords(test, "uint storage image readback",
test.expected_storage_image_r32ui, image_actual);
}
if (sampler != nullptr) {
vulkan->Device().destroySampler(sampler, nullptr);
}
vulkan->DestroyImage(&sampled_image);
vulkan->DestroyImage(&storage_image);
vulkan->DestroyImage(&storage_image_uint);
vulkan->DestroyBuffer(&gds_buffer);
vulkan->DestroyBuffer(&buffer);
CompareWords(test, "readback", test.expected, actual);
std::printf("[compute] %-32s ok\n", test.name);
}
void RunGraphicsCase(VulkanHarness *vulkan, const GraphicsCase &test) {
if (vulkan != nullptr && !vulkan->RasterizationSupported()) {
vulkan->SkipRasterizationCases(1);
return;
}
auto compiled = CompileFragmentCase(test);
auto actual = vulkan->RenderFragment(test, compiled);
CompareGraphicsWords(test, actual);
std::printf("[graphics] %-31s ok\n", test.name);
}
enum class CoverageClass {
Covered,
ControlOrMarker,
NeedsAluCase,
NeedsFloatCase,
NeedsMemoryCase,
NeedsImageCase,
NeedsGraphicsStageCase,
};
CoverageClass ClassifyOpcode(ShaderOpcode opcode,
const std::set<ShaderOpcode> &covered) {
using ShaderRecompiler::Decoder::Opcode;
if (covered.contains(opcode)) {
return CoverageClass::Covered;
}
switch (opcode) {
case Opcode::S_GETPC_B64:
case Opcode::S_SETPC_B64:
case Opcode::S_NOP:
case Opcode::S_WAITCNT:
case Opcode::S_BARRIER:
case Opcode::S_BRANCH:
case Opcode::S_CBRANCH_SCC0:
case Opcode::S_CBRANCH_SCC1:
case Opcode::S_CBRANCH_VCCZ:
case Opcode::S_CBRANCH_VCCNZ:
case Opcode::S_CBRANCH_EXECZ:
case Opcode::S_CBRANCH_EXECNZ:
case Opcode::S_SENDMSG:
case Opcode::S_SETREG_B32:
case Opcode::S_SLEEP:
case Opcode::S_TTRACEDATA:
case Opcode::S_INST_PREFETCH:
case Opcode::S_ENDPGM:
return CoverageClass::ControlOrMarker;
case Opcode::V_ADD_F32:
case Opcode::V_SUB_F32:
case Opcode::V_SUBREV_F32:
case Opcode::V_MUL_F32:
case Opcode::V_MAC_F32:
case Opcode::V_MADMK_F32:
case Opcode::V_MADAK_F32:
case Opcode::V_FMAC_F32:
case Opcode::V_FMAMK_F32:
case Opcode::V_FMAAK_F32:
case Opcode::V_MIN_F32:
case Opcode::V_MAX_F32:
case Opcode::V_MAD_F32:
case Opcode::V_FMA_F32:
case Opcode::V_MIN3_F32:
case Opcode::V_MAX3_F32:
case Opcode::V_MED3_F32:
case Opcode::V_DOT2C_F32_F16:
case Opcode::V_CVT_F64_I32:
case Opcode::V_CVT_F32_F64:
case Opcode::V_CVT_F64_F32:
case Opcode::V_CVT_F64_U32:
case Opcode::V_RCP_F64:
case Opcode::V_FRACT_F64:
case Opcode::V_MUL_F64:
case Opcode::V_FMA_F64:
case Opcode::V_CVT_F32_I32:
case Opcode::V_CVT_F32_U32:
case Opcode::V_CVT_U32_F32:
case Opcode::V_CVT_I32_F32:
case Opcode::V_CVT_F16_F32:
case Opcode::V_CVT_F32_F16:
case Opcode::V_CVT_U16_F16:
case Opcode::V_CVT_RPI_I32_F32:
case Opcode::V_CVT_FLR_I32_F32:
case Opcode::V_CVT_OFF_F32_I4:
case Opcode::V_CVT_F32_UBYTE0:
case Opcode::V_CVT_F32_UBYTE1:
case Opcode::V_CVT_F32_UBYTE2:
case Opcode::V_CVT_F32_UBYTE3:
case Opcode::V_RCP_F32:
case Opcode::V_FRACT_F32:
case Opcode::V_TRUNC_F32:
case Opcode::V_CEIL_F32:
case Opcode::V_RNDNE_F32:
case Opcode::V_FLOOR_F32:
case Opcode::V_EXP_F32:
case Opcode::V_LOG_F32:
case Opcode::V_RSQ_F32:
case Opcode::V_SQRT_F32:
case Opcode::V_SIN_F32:
case Opcode::V_COS_F32:
case Opcode::V_CUBEID_F32:
case Opcode::V_CUBESC_F32:
case Opcode::V_CUBETC_F32:
case Opcode::V_CUBEMA_F32:
case Opcode::V_LDEXP_F32:
case Opcode::V_CVT_PK_U8_F32:
case Opcode::V_CVT_PKNORM_I16_F32:
case Opcode::V_CVT_PKNORM_U16_F32:
case Opcode::V_CVT_PKRTZ_F16_F32:
case Opcode::V_PK_ADD_F16:
case Opcode::V_PK_MUL_F16:
case Opcode::V_PK_MIN_F16:
case Opcode::V_PK_MAX_F16:
case Opcode::V_PK_FMA_F16:
case Opcode::V_ADD_F16:
case Opcode::V_SUB_F16:
case Opcode::V_SUBREV_F16:
case Opcode::V_MUL_F16:
case Opcode::V_MAX_F16:
case Opcode::V_MIN_F16:
case Opcode::V_MIN3_F16:
case Opcode::V_MAX3_F16:
case Opcode::V_MED3_F16:
case Opcode::V_RCP_F16:
case Opcode::V_RSQ_F16:
case Opcode::V_LOG_F16:
case Opcode::V_EXP_F16:
case Opcode::V_SIN_F16:
case Opcode::V_COS_F16:
case Opcode::V_MAD_MIXLO_F16:
case Opcode::V_MAD_MIXHI_F16:
case Opcode::DS_MIN_F32:
case Opcode::DS_MAX_F32:
case Opcode::V_CMP_F_F32:
case Opcode::V_CMP_LT_F32:
case Opcode::V_CMP_EQ_F32:
case Opcode::V_CMP_LE_F32:
case Opcode::V_CMP_GT_F32:
case Opcode::V_CMP_LG_F32:
case Opcode::V_CMP_GE_F32:
case Opcode::V_CMP_O_F32:
case Opcode::V_CMP_U_F32:
case Opcode::V_CMP_NGE_F32:
case Opcode::V_CMP_NLG_F32:
case Opcode::V_CMP_NGT_F32:
case Opcode::V_CMP_NLE_F32:
case Opcode::V_CMP_NEQ_F32:
case Opcode::V_CMP_NLT_F32:
case Opcode::V_CMP_TRU_F32:
case Opcode::V_CMPX_LT_F32:
case Opcode::V_CMPX_EQ_F32:
case Opcode::V_CMPX_LE_F32:
case Opcode::V_CMPX_GT_F32:
case Opcode::V_CMPX_LG_F32:
case Opcode::V_CMPX_GE_F32:
case Opcode::V_CMPX_O_F32:
case Opcode::V_CMPX_NGE_F32:
case Opcode::V_CMPX_NLG_F32:
case Opcode::V_CMPX_NGT_F32:
case Opcode::V_CMPX_NLE_F32:
case Opcode::V_CMPX_NEQ_F32:
case Opcode::V_CMPX_NLT_F32:
case Opcode::V_CMP_CLASS_F32:
case Opcode::V_CMPX_CLASS_F32:
case Opcode::V_CMP_CLASS_F16:
case Opcode::V_CMPX_CLASS_F16:
case Opcode::V_CMP_LT_F16:
case Opcode::V_CMP_EQ_F16:
case Opcode::V_CMP_LE_F16:
case Opcode::V_CMP_GT_F16:
case Opcode::V_CMP_LG_F16:
case Opcode::V_CMP_GE_F16:
case Opcode::V_CMP_NGT_F16:
case Opcode::V_CMP_NEQ_F16:
case Opcode::V_CMP_NLT_F16:
case Opcode::V_CMPX_LT_F16:
case Opcode::V_CMPX_EQ_F16:
case Opcode::V_CMPX_LE_F16:
case Opcode::V_CMPX_GT_F16:
case Opcode::V_CMPX_GE_F16:
case Opcode::V_CMPX_NEQ_F16:
case Opcode::V_CMPX_NLT_F16:
return CoverageClass::NeedsFloatCase;
case Opcode::S_LOAD_DWORD:
case Opcode::S_LOAD_DWORDX2:
case Opcode::S_LOAD_DWORDX4:
case Opcode::S_LOAD_DWORDX8:
case Opcode::S_LOAD_DWORDX16:
case Opcode::S_BUFFER_LOAD_DWORD:
case Opcode::S_BUFFER_LOAD_DWORDX2:
case Opcode::S_BUFFER_LOAD_DWORDX4:
case Opcode::S_BUFFER_LOAD_DWORDX8:
case Opcode::S_BUFFER_LOAD_DWORDX16:
case Opcode::BUFFER_LOAD_FORMAT_X:
case Opcode::BUFFER_LOAD_FORMAT_XY:
case Opcode::BUFFER_LOAD_FORMAT_XYZ:
case Opcode::BUFFER_LOAD_FORMAT_XYZW:
case Opcode::BUFFER_STORE_FORMAT_X:
case Opcode::BUFFER_STORE_FORMAT_XY:
case Opcode::BUFFER_STORE_FORMAT_XYZ:
case Opcode::BUFFER_STORE_FORMAT_XYZW:
case Opcode::BUFFER_LOAD_UBYTE:
case Opcode::BUFFER_LOAD_UBYTE_D16:
case Opcode::BUFFER_LOAD_SBYTE:
case Opcode::BUFFER_LOAD_USHORT:
case Opcode::BUFFER_LOAD_SSHORT:
case Opcode::BUFFER_LOAD_SHORT_D16:
case Opcode::BUFFER_LOAD_SHORT_D16_HI:
case Opcode::BUFFER_LOAD_DWORDX2:
case Opcode::BUFFER_LOAD_DWORDX3:
case Opcode::BUFFER_LOAD_DWORDX4:
case Opcode::BUFFER_STORE_BYTE:
case Opcode::BUFFER_STORE_SHORT:
case Opcode::BUFFER_STORE_DWORDX2:
case Opcode::BUFFER_STORE_DWORDX3:
case Opcode::BUFFER_STORE_DWORDX4:
case Opcode::TBUFFER_LOAD_FORMAT_X:
case Opcode::TBUFFER_LOAD_FORMAT_XY:
case Opcode::TBUFFER_LOAD_FORMAT_XYZ:
case Opcode::TBUFFER_LOAD_FORMAT_XYZW:
case Opcode::TBUFFER_STORE_FORMAT_X:
case Opcode::TBUFFER_STORE_FORMAT_XY:
case Opcode::TBUFFER_STORE_FORMAT_XYZ:
case Opcode::TBUFFER_STORE_FORMAT_XYZW:
case Opcode::BUFFER_ATOMIC_SWAP:
case Opcode::BUFFER_ATOMIC_CMPSWAP:
case Opcode::BUFFER_ATOMIC_ADD:
case Opcode::BUFFER_ATOMIC_SUB:
case Opcode::BUFFER_ATOMIC_SMIN:
case Opcode::BUFFER_ATOMIC_UMIN:
case Opcode::BUFFER_ATOMIC_SMAX:
case Opcode::BUFFER_ATOMIC_UMAX:
case Opcode::BUFFER_ATOMIC_AND:
case Opcode::BUFFER_ATOMIC_ADD_X2:
case Opcode::BUFFER_ATOMIC_AND_X2:
case Opcode::BUFFER_ATOMIC_OR:
case Opcode::BUFFER_ATOMIC_XOR:
case Opcode::BUFFER_ATOMIC_FMIN:
case Opcode::BUFFER_ATOMIC_FMAX:
case Opcode::FLAT_LOAD_UBYTE:
case Opcode::FLAT_LOAD_SBYTE:
case Opcode::FLAT_LOAD_USHORT:
case Opcode::FLAT_LOAD_SSHORT:
case Opcode::FLAT_LOAD_SHORT_D16:
case Opcode::FLAT_LOAD_DWORD:
case Opcode::FLAT_LOAD_DWORDX2:
case Opcode::FLAT_LOAD_DWORDX3:
case Opcode::FLAT_LOAD_DWORDX4:
case Opcode::FLAT_STORE_BYTE:
case Opcode::FLAT_STORE_SHORT:
case Opcode::FLAT_STORE_DWORD:
case Opcode::FLAT_STORE_DWORDX2:
case Opcode::FLAT_STORE_DWORDX3:
case Opcode::FLAT_STORE_DWORDX4:
case Opcode::DS_ADD_U32:
case Opcode::DS_ADD_U64:
case Opcode::DS_MAX_RTN_U64:
case Opcode::DS_CMPST_RTN_B32:
case Opcode::DS_CMPST_RTN_B64:
case Opcode::DS_ADD_RTN_U32:
case Opcode::DS_SUB_U32:
case Opcode::DS_SUB_RTN_U32:
case Opcode::DS_INC_RTN_U32:
case Opcode::DS_DEC_RTN_U32:
case Opcode::DS_MIN_I32:
case Opcode::DS_MIN_RTN_I32:
case Opcode::DS_MAX_I32:
case Opcode::DS_MAX_RTN_I32:
case Opcode::DS_MIN_U32:
case Opcode::DS_MIN_RTN_U32:
case Opcode::DS_MAX_U32:
case Opcode::DS_MAX_RTN_U32:
case Opcode::DS_AND_B32:
case Opcode::DS_AND_RTN_B32:
case Opcode::DS_OR_B32:
case Opcode::DS_OR_RTN_B32:
case Opcode::DS_XOR_B32:
case Opcode::DS_XOR_RTN_B32:
case Opcode::DS_MSKOR_B32:
case Opcode::DS_WRXCHG_RTN_B32:
case Opcode::DS_SWIZZLE_B32:
case Opcode::DS_PERMUTE_B32:
case Opcode::DS_BPERMUTE_B32:
case Opcode::DS_READ_I8:
case Opcode::DS_READ_U8:
case Opcode::DS_READ_I16:
case Opcode::DS_READ_U16:
case Opcode::DS_READ_U16_D16:
case Opcode::DS_READ_U16_D16_HI:
case Opcode::DS_READ2_B32:
case Opcode::DS_READ_B32:
case Opcode::DS_READ_B64:
case Opcode::DS_READ2_B64:
case Opcode::DS_READ_B96:
case Opcode::DS_READ_B128:
case Opcode::DS_WRITE_B8:
case Opcode::DS_WRITE_B16:
case Opcode::DS_WRITE_B8_D16_HI:
case Opcode::DS_WRITE_B16_D16_HI:
case Opcode::DS_WRITE2_B32:
case Opcode::DS_WRITE2ST64_B32:
case Opcode::DS_WRITE_B32:
case Opcode::DS_WRITE_B64:
case Opcode::DS_WRITE_B96:
case Opcode::DS_WRITE_B128:
case Opcode::DS_WRITE_ADDTID_B32:
case Opcode::DS_READ_ADDTID_B32:
return CoverageClass::NeedsMemoryCase;
case Opcode::IMAGE_GET_RESINFO:
case Opcode::IMAGE_GET_LOD:
case Opcode::IMAGE_LOAD:
case Opcode::IMAGE_LOAD_MIP:
case Opcode::IMAGE_STORE:
case Opcode::IMAGE_STORE_MIP:
case Opcode::IMAGE_ATOMIC_SWAP:
case Opcode::IMAGE_ATOMIC_ADD:
case Opcode::IMAGE_ATOMIC_SMIN:
case Opcode::IMAGE_ATOMIC_SMAX:
case Opcode::IMAGE_ATOMIC_UMIN:
case Opcode::IMAGE_ATOMIC_AND:
case Opcode::IMAGE_ATOMIC_OR:
case Opcode::IMAGE_ATOMIC_XOR:
case Opcode::IMAGE_ATOMIC_FMIN:
case Opcode::IMAGE_ATOMIC_FMAX:
case Opcode::IMAGE_SAMPLE:
case Opcode::IMAGE_GATHER4_LZ:
case Opcode::IMAGE_GATHER4_C:
case Opcode::IMAGE_GATHER4_C_L:
case Opcode::IMAGE_GATHER4_C_LZ:
case Opcode::IMAGE_GATHER4_LZ_O:
case Opcode::IMAGE_GATHER4_C_O:
case Opcode::IMAGE_GATHER4_C_LZ_O:
return CoverageClass::NeedsImageCase;
case Opcode::V_INTERP_P1_F32:
case Opcode::V_INTERP_P2_F32:
case Opcode::V_INTERP_MOV_F32:
case Opcode::EXP:
return CoverageClass::NeedsGraphicsStageCase;
default:
return CoverageClass::NeedsAluCase;
}
}
const char *CoverageClassName(CoverageClass status) {
switch (status) {
case CoverageClass::Covered:
return "covered";
case CoverageClass::ControlOrMarker:
return "control";
case CoverageClass::NeedsAluCase:
return "alu";
case CoverageClass::NeedsFloatCase:
return "float";
case CoverageClass::NeedsMemoryCase:
return "memory";
case CoverageClass::NeedsImageCase:
return "image";
case CoverageClass::NeedsGraphicsStageCase:
return "graphics";
default:
return "unknown";
}
}
void PrintPendingOpcodes(CoverageClass status,
const std::vector<ShaderOpcode> &opcodes) {
if (opcodes.empty()) {
return;
}
std::printf("[coverage] pending_%s:", CoverageClassName(status));
for (auto opcode : opcodes) {
const auto name = magic_enum::enum_name(opcode);
std::printf(" %.*s", static_cast<int>(name.size()), name.data());
}
std::printf("\n");
}
void CheckOpcodeCoverage(const std::vector<TestCase> &tests,
const std::vector<GraphicsCase> &graphics_tests) {
using ShaderRecompiler::Decoder::Opcode;
std::set<Opcode> covered;
for (const auto &test : tests) {
for (auto opcode : test.opcodes) {
covered.insert(opcode);
}
}
for (const auto &test : graphics_tests) {
for (auto opcode : test.opcodes) {
covered.insert(opcode);
}
}
uint32_t counts[7] = {};
std::vector<ShaderOpcode> pending[7];
for (auto value = static_cast<int>(Opcode::S_MOV_B32);
value < static_cast<int>(Opcode::COUNT); value++) {
const auto opcode = static_cast<Opcode>(value);
const auto status = ClassifyOpcode(opcode, covered);
counts[static_cast<uint32_t>(status)]++;
if (status != CoverageClass::Covered &&
status != CoverageClass::ControlOrMarker) {
pending[static_cast<uint32_t>(status)].push_back(opcode);
}
}
std::printf(
"[coverage] decoder opcodes: covered=%u control=%u alu_pending=%u "
"float_pending=%u memory_pending=%u image_pending=%u "
"graphics_pending=%u\n",
counts[static_cast<uint32_t>(CoverageClass::Covered)],
counts[static_cast<uint32_t>(CoverageClass::ControlOrMarker)],
counts[static_cast<uint32_t>(CoverageClass::NeedsAluCase)],
counts[static_cast<uint32_t>(CoverageClass::NeedsFloatCase)],
counts[static_cast<uint32_t>(CoverageClass::NeedsMemoryCase)],
counts[static_cast<uint32_t>(CoverageClass::NeedsImageCase)],
counts[static_cast<uint32_t>(CoverageClass::NeedsGraphicsStageCase)]);
PrintPendingOpcodes(
CoverageClass::NeedsAluCase,
pending[static_cast<uint32_t>(CoverageClass::NeedsAluCase)]);
PrintPendingOpcodes(
CoverageClass::NeedsFloatCase,
pending[static_cast<uint32_t>(CoverageClass::NeedsFloatCase)]);
PrintPendingOpcodes(
CoverageClass::NeedsMemoryCase,
pending[static_cast<uint32_t>(CoverageClass::NeedsMemoryCase)]);
PrintPendingOpcodes(
CoverageClass::NeedsImageCase,
pending[static_cast<uint32_t>(CoverageClass::NeedsImageCase)]);
PrintPendingOpcodes(
CoverageClass::NeedsGraphicsStageCase,
pending[static_cast<uint32_t>(CoverageClass::NeedsGraphicsStageCase)]);
}
TestCase IntegerAddSubMul() {
using O = ShaderOpcode;
std::vector<u32> code = {
EncodeSMovB32(0, InlineU32(7)),
EncodeSMovB32(1, InlineU32(3)),
EncodeSop2(0x00, 2, 0, 1),
EncodeSop2(0x01, 3, 2, InlineU32(1)),
EncodeSop2(0x26, 4, 3, InlineU32(2)),
EncodeVop1(0x01, 0, 4),
};
AppendBufferStoreDword(&code, 0, 30);
AppendEnd(&code);
return {"IntegerAddSubMul",
code,
{},
{18},
{O::S_MOV_B32, O::S_ADD_U32, O::S_SUB_U32, O::S_MUL_I32, O::V_MOV_B32,
O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}
TestCase BitwiseOps() {
using O = ShaderOpcode;
std::vector<u32> code = {
EncodeSMovB32(0, InlineU32(60)), EncodeSMovB32(1, InlineU32(15)),
EncodeSop2(0x0e, 2, 0, 1), EncodeSop2(0x10, 3, 0, 1),
EncodeSop2(0x12, 4, 3, 2), EncodeVop1(0x37, 0, 4),
};
AppendBufferStoreDword(&code, 0, 30);
AppendEnd(&code);
return {"BitwiseAndOrXorNot",
code,
{},
{~0x33u},
{O::S_MOV_B32, O::S_AND_B32, O::S_OR_B32, O::S_XOR_B32, O::V_NOT_B32,
O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}
TestCase Shifts() {
using O = ShaderOpcode;
std::vector<u32> code = {
EncodeSMovB32(0, InlineU32(3)),
EncodeSop2(0x1e, 1, 0, InlineU32(2)),
EncodeSop2(0x20, 2, 1, InlineU32(1)),
EncodeVop1(0x01, 0, 2),
};
AppendBufferStoreDword(&code, 0, 30);
AppendEnd(&code);
return {"Shifts",
code,
{},
{6},
{O::S_MOV_B32, O::S_LSHL_B32, O::S_LSHR_B32, O::V_MOV_B32,
O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}
TestCase ScalarZeroShiftWithRuntimeCount() {
using O = ShaderOpcode;
std::vector<u32> code = {
EncodeSop2(0x20, 8, InlineU32(0), 4),
EncodeSop2(0x20, 9, 6, 4),
};
AppendStoreSgpr(&code, 8, 0);
AppendStoreSgpr(&code, 9, 1);
AppendEnd(&code);
TestCase test;
test.name = "ScalarZeroShiftWithRuntimeCount";
test.code = std::move(code);
test.expected = {0, 0x40000000u};
test.opcodes = {O::S_LSHR_B32, O::V_MOV_B32, O::BUFFER_STORE_DWORD,
O::S_ENDPGM};
test.user_data = MakeStructuredStorageBufferData(0, 2 * sizeof(u32));
test.user_data[4] = 33;
test.user_data[6] = 0x80000000u;
test.has_user_data = true;
test.ir_counts = {{" = ShiftRightLogical32 ", 1}};
test.required_spirv = {"OpShiftRightLogical"};
return test;
}
TestCase ExactPushConstantExtent() {
using O = ShaderOpcode;
constexpr std::array<u32, 10> values{
0x10203040u, 0x21314151u, 0x32425262u, 0x43536373u, 0x54647484u,
0x65758595u, 0x768696a6u, 0x8797a7b7u, 0x98a8b8c8u, 0xa9b9c9d9u};
std::vector<u32> code;
for (u32 i = 0; i < values.size(); i++) {
AppendStoreSgpr(&code, 4u + i, i);
}
AppendEnd(&code);
TestCase test;
test.name = "ExactPushConstantExtent";
test.code = std::move(code);
test.initial.resize(values.size());
test.expected.assign(values.begin(), values.end());
test.opcodes = {O::V_MOV_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.user_data =
MakeStructuredStorageBufferData(0, values.size() * sizeof(u32));
std::copy(values.begin(), values.end(), test.user_data.begin() + 4);
test.has_user_data = true;
return test;
}
TestCase ScalarShiftCountsMaskLowBits() {
using O = ShaderOpcode;
std::vector<u32> code;
code.push_back(EncodeSMovB32(0, InlineU32(1)));
AppendSMovLiteral(&code, 1, 0x80000000u);
code.push_back(EncodeSMovB32(2, InlineU32(32)));
code.push_back(EncodeSMovB32(3, InlineU32(33)));
code.push_back(EncodeSop2(0x1e, 10, 0, 2));
code.push_back(EncodeSop2(0x1e, 11, 0, 3));
code.push_back(EncodeSop2(0x20, 12, 1, 2));
code.push_back(EncodeSop2(0x20, 13, 1, 3));
code.push_back(EncodeSop2(0x22, 14, 1, 2));
code.push_back(EncodeSop2(0x22, 15, 1, 3));
for (u32 i = 0; i < 6u; i++) {
AppendStoreSgpr(&code, 10u + i, i);
}
AppendEnd(&code);
return {"ScalarShiftCountsMaskLowBits",
code,
{},
{1, 2, 0x80000000u, 0x40000000u, 0x80000000u, 0xc0000000u},
{O::S_MOV_B32, O::S_LSHL_B32, O::S_LSHR_B32, O::S_ASHR_I32,
O::V_MOV_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}
TestCase Rdna2ScalarOpcodes() {
using O = ShaderOpcode;
std::vector<u32> code;
code.push_back(EncodeSMovB32(2, InlineU32(7)));
code.push_back(EncodeSMovB32(106, InlineU32(16)));
code.push_back(EncodeSop1(0x1d, 106, InlineU32(0)));
AppendStoreSgpr(&code, 106, 0);
code.push_back(EncodeSopk(0x13, 106, 0x1019u));
AppendStoreSgpr(&code, 106, 1);
code.push_back(EncodeSop2(0x02, 106, 2, 239u));
AppendStoreSgpr(&code, 106, 2);
code.push_back(EncodeSopp(0x0e, 0));
code.push_back(EncodeSop2(0x02, 106, 239u, 2));
AppendStoreSgpr(&code, 106, 3);
AppendEnd(&code);
return {"Rdna2ScalarOpcodes",
code,
{},
{0x11u, 0x11u, 7u, 7u},
{O::S_MOV_B32, O::S_BITSET1_B32, O::S_SETREG_B32, O::S_ADD_I32,
O::S_SLEEP, O::V_MOV_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}
TestCase ScalarTrapDisabledFallsThroughExactRaw() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 0, 7);
code.push_back(0xbf920001u); // s_trap 1 from the game shader
AppendVMovU32(&code, 0, 42);
AppendStoreVgpr(&code, 0, 0);
AppendEnd(&code);
TestCase test;
test.name = "ScalarTrapDisabledFallsThroughExactRaw";
test.code = code;
test.expected = {42};
test.opcodes = {O::V_MOV_B32, O::S_TRAP, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.decoded_counts = {{"S_TRAP 0x00000001", 1}};
test.compute_info.threads_num[0] = 1;
test.compute_info.threads_num[1] = 1;
test.compute_info.threads_num[2] = 1;
test.compute_info.wave_size = 32;
test.has_compute_info = true;
return test;
}
TestCase ScalarWaitcntDepctrCapturedVmVsrc() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 7, 7);
AppendVMovU32(&code, 31, 0);
AppendBufferStoreDword(&code, 7, 31);
code.push_back(0xbfa3ffe3u); // s_waitcnt_depctr vm_vsrc(0)
AppendVMovU32(&code, 7, 42);
AppendStoreVgpr(&code, 7, 1);
AppendEnd(&code);
TestCase test;
test.name = "ScalarWaitcntDepctrCapturedVmVsrc";
test.code = std::move(code);
test.initial = {0, 0};
test.expected = {7, 42};
test.opcodes = {O::V_MOV_B32, O::BUFFER_STORE_DWORD, O::S_WAITCNT_DEPCTR,
O::S_ENDPGM};
test.decoded_counts = {{"0x00000010: S_WAITCNT_DEPCTR 0x0000ffe3\n", 1}};
test.forbidden_spirv = {"OpControlBarrier", "OpMemoryBarrier"};
return test;
}
TestCase ScalarExtendedArithmetic() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendSMovLiteral(&code, 0, 0xffffffffu);
code.push_back(EncodeSMovB32(1, InlineU32(1)));
code.push_back(EncodeSop2(0x00, 2, 0, 1));
code.push_back(EncodeSMovB32(3, InlineU32(5)));
code.push_back(EncodeSMovB32(4, InlineU32(6)));
code.push_back(EncodeSop2(0x04, 5, 3, 4));
code.push_back(EncodeSop2(0x02, 6, InlineU32(7), InlineU32(8)));
code.push_back(EncodeSop2(0x03, 7, InlineU32(7), InlineU32(9)));
AppendSMovLiteral(&code, 8, 0xfffffffbu);
code.push_back(EncodeSMovB32(9, InlineU32(3)));
code.push_back(EncodeSop2(0x06, 10, 8, 9));
code.push_back(EncodeSop2(0x08, 11, 8, 9));
AppendSMovLiteral(&code, 12, 0xfffffffeu);
code.push_back(EncodeSMovB32(13, InlineU32(3)));
code.push_back(EncodeSop2(0x07, 14, 12, 13));
code.push_back(EncodeSop2(0x09, 15, 12, 13));
code.push_back(EncodeSMovB32(16, 8));
code.push_back(EncodeSop1(0x34, 16, 16));
code.push_back(EncodeSopk(0x00, 17, 0xfff5u));
code.push_back(EncodeSopk(0x10, 17, 0xfffdu));
const u32 results[] = {2, 5, 6, 7, 10, 11, 14, 15, 16, 17};
for (u32 i = 0; i < static_cast<u32>(std::size(results)); i++) {
AppendStoreSgpr(&code, results[i], i);
}
AppendEnd(&code);
return {"ScalarExtendedArithmetic",
code,
{},
{0, 12, 15, 0xfffffffeu, 0xfffffffbu, 3, 3, 0xfffffffeu, 5, 33},
{O::S_MOV_B32, O::S_ADD_U32, O::S_ADDC_U32, O::S_ADD_I32,
O::S_SUB_I32, O::S_MIN_I32, O::S_MAX_I32, O::S_MIN_U32, O::S_MAX_U32,
O::S_ABS_I32, O::S_MOVK_I32, O::S_MULK_I32, O::V_MOV_B32,
O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}
TestCase ScalarArithmeticSccCarryBorrowOverflow() {
using O = ShaderOpcode;
std::vector<u32> code;
auto set_scc = [&](bool value) {
code.push_back(EncodeSopc(0x06, InlineU32(1), InlineU32(value ? 1u : 0u)));
};
auto capture_scc = [&](u32 dst_sgpr) {
code.push_back(EncodeSop2(0x0a, dst_sgpr, InlineU32(1), InlineU32(0)));
};
auto append_case = [&](bool prior_scc, u32 instruction, u32 out_sgpr) {
set_scc(prior_scc);
code.push_back(instruction);
capture_scc(out_sgpr);
};
code.push_back(EncodeSMovB32(0, InlineU32(0)));
code.push_back(EncodeSMovB32(1, InlineU32(1)));
append_case(false, EncodeSop2(0x01, 10, 0, 1), 20);
code.push_back(EncodeSMovB32(2, InlineU32(5)));
code.push_back(EncodeSMovB32(3, InlineU32(3)));
append_case(true, EncodeSop2(0x01, 11, 2, 3), 21);
AppendSMovLiteral(&code, 4, 0x7fffffffu);
code.push_back(EncodeSMovB32(5, InlineU32(1)));
append_case(false, EncodeSop2(0x02, 12, 4, 5), 22);
code.push_back(EncodeSMovB32(6, InlineU32(1)));
code.push_back(EncodeSMovB32(7, InlineU32(2)));
append_case(true, EncodeSop2(0x02, 13, 6, 7), 23);
AppendSMovLiteral(&code, 8, 0x80000000u);
code.push_back(EncodeSMovB32(9, InlineU32(1)));
append_case(false, EncodeSop2(0x03, 14, 8, 9), 24);
code.push_back(EncodeSMovB32(15, InlineU32(5)));
code.push_back(EncodeSMovB32(16, InlineU32(3)));
append_case(true, EncodeSop2(0x03, 17, 15, 16), 25);
AppendSMovLiteral(&code, 18, 0x7fffffffu);
set_scc(false);
code.push_back(EncodeSopk(0x0f, 18, 1));
capture_scc(26);
code.push_back(EncodeSMovB32(19, InlineU32(4)));
set_scc(true);
code.push_back(EncodeSopk(0x0f, 19, 0xfffeu));
capture_scc(27);
for (u32 i = 0; i < 8u; i++) {
AppendStoreSgpr(&code, 20u + i, i);
}
AppendEnd(&code);
return {"ScalarArithmeticSccCarryBorrowOverflow",
code,
{},
{1, 0, 1, 0, 1, 0, 1, 0},
{O::S_MOV_B32, O::S_SUB_U32, O::S_ADD_I32, O::S_SUB_I32,
O::S_CMP_EQ_U32, O::S_CSELECT_B32, O::V_MOV_B32,
O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}
TestCase ScalarMinMaxSccComparisonEdges() {
using O = ShaderOpcode;
std::vector<u32> code;
auto set_scc = [&](bool value) {
code.push_back(EncodeSopc(0x06, InlineU32(1), InlineU32(value ? 1u : 0u)));
};
auto capture_scc = [&](u32 dst_sgpr) {
code.push_back(EncodeSop2(0x0a, dst_sgpr, InlineU32(1), InlineU32(0)));
};
auto append_case = [&](bool prior_scc, u32 instruction, u32 out_sgpr) {
set_scc(prior_scc);
code.push_back(instruction);
capture_scc(out_sgpr);
};
AppendSMovLiteral(&code, 0, 0xfffffffeu);
code.push_back(EncodeSMovB32(1, InlineU32(3)));
append_case(false, EncodeSop2(0x06, 10, 0, 1), 20);
append_case(true, EncodeSop2(0x06, 11, 1, 0), 21);
append_case(false, EncodeSop2(0x08, 12, 1, 0), 22);
append_case(true, EncodeSop2(0x08, 13, 0, 1), 23);
code.push_back(EncodeSMovB32(2, InlineU32(2)));
code.push_back(EncodeSMovB32(3, InlineU32(3)));
append_case(false, EncodeSop2(0x07, 14, 2, 3), 24);
append_case(true, EncodeSop2(0x07, 15, 3, 2), 25);
append_case(false, EncodeSop2(0x09, 16, 3, 2), 26);
append_case(true, EncodeSop2(0x09, 17, 2, 3), 27);
for (u32 i = 0; i < 8u; i++) {
AppendStoreSgpr(&code, 20u + i, i);
}
AppendEnd(&code);
return {"ScalarMinMaxSccComparisonEdges",
code,
{},
{1, 0, 1, 0, 1, 0, 1, 0},
{O::S_MOV_B32, O::S_MIN_I32, O::S_MAX_I32, O::S_MIN_U32, O::S_MAX_U32,
O::S_CMP_EQ_U32, O::S_CSELECT_B32, O::V_MOV_B32,
O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}
TestCase ScalarAbsI32UpdatesScc() {
using O = ShaderOpcode;
std::vector<u32> code;
auto set_scc = [&](bool value) {
code.push_back(EncodeSopc(0x06, InlineU32(1), InlineU32(value ? 1u : 0u)));
};
auto capture_scc = [&](u32 dst_sgpr) {
code.push_back(EncodeSop2(0x0a, dst_sgpr, InlineU32(1), InlineU32(0)));
};
AppendSmemLoadOpcode(&code, 0x08, 16, 0);
set_scc(true);
code.push_back(EncodeSop1(0x34, 17, 16));
capture_scc(18);
AppendSmemLoadOpcode(&code, 0x08, 19, 4);
set_scc(false);
code.push_back(EncodeSop1(0x34, 20, 19));
capture_scc(21);
AppendSmemLoadOpcode(&code, 0x08, 22, 8);
set_scc(false);
code.push_back(EncodeSop1(0x34, 23, 22));
capture_scc(24);
AppendStoreSgpr(&code, 17, 0);
AppendStoreSgpr(&code, 18, 1);
AppendStoreSgpr(&code, 20, 2);
AppendStoreSgpr(&code, 21, 3);
AppendStoreSgpr(&code, 23, 4);
AppendStoreSgpr(&code, 24, 5);
AppendEnd(&code);
return {"ScalarAbsI32UpdatesScc",
code,
{0, 0xfffffffbu, 0x80000000u},
{0, 0, 5, 1, 0x80000000u, 1},
{O::S_BUFFER_LOAD_DWORD, O::S_CMP_EQ_U32, O::S_ABS_I32, O::S_CSELECT_B32,
O::V_MOV_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}
TestCase ScalarAbsdiffBitset64AndSetprio() {
using O = ShaderOpcode;
std::vector<u32> code;
auto capture_scc = [&](u32 dst_sgpr) {
code.push_back(
EncodeSop2(0x0a, dst_sgpr, InlineU32(1), InlineU32(0)));
};
code.push_back(EncodeSMovB32(0, InlineU32(2)));
code.push_back(EncodeSMovB32(1, InlineU32(5)));
code.push_back(EncodeSop2(0x2c, 10, 0, 1));
capture_scc(20);
AppendSMovLiteral(&code, 0, 0xffffffffu);
code.push_back(EncodeSMovB32(1, InlineU32(0)));
code.push_back(EncodeSop2(0x2c, 11, 0, 1));
capture_scc(21);
AppendSMovLiteral(&code, 0, 0x80000000u);
code.push_back(EncodeSop2(0x2c, 12, 0, 1));
capture_scc(22);
code.push_back(EncodeSMovB32(1, InlineU32(1)));
code.push_back(EncodeSop2(0x2c, 13, 0, 1));
capture_scc(23);
code.push_back(EncodeSop2(0x2c, 14, 0, 0));
capture_scc(24);
AppendSMovLiteral(&code, 2, 65u);
AppendSMovLiteral(&code, 3, 101u);
code.push_back(EncodeSMovB32(4, InlineU32(0)));
code.push_back(EncodeSMovB32(5, InlineU32(0)));
code.push_back(EncodeSop1(0x1e, 4, 2));
AppendStoreSgprPair(&code, 4, 10);
code.push_back(EncodeSop1(0x1e, 4, 3));
AppendStoreSgprPair(&code, 4, 12);
code.push_back(EncodeSop1(0x1c, 4, 2));
AppendStoreSgprPair(&code, 4, 14);
code.push_back(EncodeSop1(0x1c, 4, 3));
AppendStoreSgprPair(&code, 4, 16);
code.push_back(EncodeSopp(0x0f, 3));
AppendStoreSgpr(&code, 10, 0);
AppendStoreSgpr(&code, 11, 1);
AppendStoreSgpr(&code, 12, 2);
AppendStoreSgpr(&code, 13, 3);
AppendStoreSgpr(&code, 14, 4);
AppendStoreSgpr(&code, 20, 5);
AppendStoreSgpr(&code, 21, 6);
AppendStoreSgpr(&code, 22, 7);
AppendStoreSgpr(&code, 23, 8);
AppendStoreSgpr(&code, 24, 9);
AppendEnd(&code);
TestCase test;
test.name = "ScalarAbsdiffBitset64AndSetprio";
test.code = std::move(code);
test.expected = {3u, 1u, 0x80000000u, 0x7fffffffu, 0u,
1u, 1u, 1u, 1u, 0u,
2u, 0u, 2u, 32u, 0u,
32u, 0u, 0u};
test.opcodes = {O::S_MOV_B32, O::S_ABSDIFF_I32, O::S_CSELECT_B32,
O::S_BITSET1_B64, O::S_BITSET0_B64, O::S_SETPRIO,
O::V_MOV_B32,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.decoded_counts = {{"S_ABSDIFF_I32", 5},
{"S_BITSET1_B64", 2},
{"S_BITSET0_B64", 2},
{"S_SETPRIO 0x00000003", 1}};
return test;
}
TestCase ScalarShiftLeftAddSccCarryEdges() {
using O = ShaderOpcode;
std::vector<u32> code;
auto set_scc = [&](bool value) {
code.push_back(EncodeSopc(0x06, InlineU32(1), InlineU32(value ? 1u : 0u)));
};
auto capture_scc = [&](u32 dst_sgpr) {
code.push_back(EncodeSop2(0x0a, dst_sgpr, InlineU32(1), InlineU32(0)));
};
auto append_case = [&](bool prior_scc, u32 instruction, u32 out_sgpr) {
set_scc(prior_scc);
code.push_back(instruction);
capture_scc(out_sgpr);
};
AppendSMovLiteral(&code, 0, 0x80000000u);
AppendSMovLiteral(&code, 1, 0x40000000u);
AppendSMovLiteral(&code, 2, 0x20000000u);
AppendSMovLiteral(&code, 3, 0x10000000u);
code.push_back(EncodeSMovB32(4, InlineU32(1)));
code.push_back(EncodeSMovB32(5, InlineU32(0)));
append_case(false, EncodeSop2(0x2e, 10, 0, 5), 20);
append_case(true, EncodeSop2(0x2e, 11, 4, 4), 21);
append_case(false, EncodeSop2(0x2f, 12, 1, 5), 22);
append_case(true, EncodeSop2(0x2f, 13, 4, 4), 23);
append_case(false, EncodeSop2(0x30, 14, 2, 5), 24);
append_case(true, EncodeSop2(0x30, 15, 4, 4), 25);
append_case(false, EncodeSop2(0x31, 16, 3, 5), 26);
append_case(true, EncodeSop2(0x31, 17, 4, 4), 27);
for (u32 i = 0; i < 8u; i++) {
AppendStoreSgpr(&code, 20u + i, i);
}
AppendEnd(&code);
return {"ScalarShiftLeftAddSccCarryEdges",
code,
{},
{1, 0, 1, 0, 1, 0, 1, 0},
{O::S_MOV_B32, O::S_LSHL1_ADD_U32, O::S_LSHL2_ADD_U32,
O::S_LSHL3_ADD_U32, O::S_LSHL4_ADD_U32, O::S_CMP_EQ_U32,
O::S_CSELECT_B32, O::V_MOV_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}
TestCase ScalarCompareOps() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendSMovLiteral(&code, 0, 0xfffffffeu);
code.push_back(EncodeSMovB32(1, InlineU32(1)));
code.push_back(EncodeSMovB32(2, InlineU32(5)));
code.push_back(EncodeSMovB32(3, InlineU32(2)));
code.push_back(EncodeSMovB32(4, InlineU32(3)));
AppendSMovLiteral(&code, 10, 0x00000001u);
code.push_back(EncodeSMovB32(11, InlineU32(0)));
code.push_back(EncodeSMovB32(12, InlineU32(2)));
code.push_back(EncodeSMovB32(13, InlineU32(0)));
code.push_back(EncodeSMovB32(14, InlineU32(1)));
code.push_back(EncodeSMovB32(15, InlineU32(1)));
u32 dst = 20;
auto append_compare = [&](u32 opcode, u32 src0, u32 src1) {
code.push_back(EncodeSopc(opcode, src0, src1));
code.push_back(EncodeSop2(0x0a, dst++, InlineU32(1), InlineU32(0)));
};
append_compare(0x00, 2, 2);
append_compare(0x01, 0, 1);
append_compare(0x02, 1, 0);
append_compare(0x03, 1, 1);
append_compare(0x04, 0, 1);
append_compare(0x05, 0, 0);
append_compare(0x07, 3, 4);
append_compare(0x08, 4, 3);
append_compare(0x09, 4, 4);
append_compare(0x0b, 3, 4);
append_compare(0x12, 10, 10);
append_compare(0x12, 10, 12);
append_compare(0x12, 10, 14);
append_compare(0x13, 10, 12);
append_compare(0x13, 10, 14);
for (u32 i = 0; i < 15u; i++) {
AppendStoreSgpr(&code, 20u + i, i);
}
AppendEnd(&code);
return {"ScalarCompareOps",
code,
{},
{1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 1},
{O::S_MOV_B32, O::S_CMP_EQ_I32, O::S_CMP_LG_I32, O::S_CMP_GT_I32,
O::S_CMP_GE_I32, O::S_CMP_LT_I32, O::S_CMP_LE_I32, O::S_CMP_LG_U32,
O::S_CMP_GT_U32, O::S_CMP_GE_U32, O::S_CMP_LE_U32, O::S_CMP_EQ_U64,
O::S_CMP_LG_U64, O::S_CSELECT_B32, O::V_MOV_B32,
O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}
TestCase ScalarShiftAddAndMaskOps() {
using O = ShaderOpcode;
std::vector<u32> code;
code.push_back(EncodeSMovB32(0, InlineU32(2)));
code.push_back(EncodeSMovB32(1, InlineU32(3)));
code.push_back(EncodeSop2(0x2e, 2, 0, 1));
code.push_back(EncodeSop2(0x2f, 3, 0, 1));
code.push_back(EncodeSop2(0x30, 4, 0, 1));
code.push_back(EncodeSop2(0x31, 5, 0, 1));
AppendSMovLiteral(&code, 6, 0xfffffff8u);
code.push_back(EncodeSop2(0x22, 7, 6, InlineU32(2)));
AppendSMovLiteral(&code, 8, 0xffffffffu);
code.push_back(EncodeSMovB32(9, InlineU32(2)));
code.push_back(EncodeSop2(0x35, 10, 8, 9));
AppendSMovLiteral(&code, 12, 0x80000011u);
AppendSMovLiteral(&code, 13, 0x80000001u);
code.push_back(EncodeSop1(0x08, 14, 12));
code.push_back(EncodeSop1(0x0a, 16, 12));
AppendStoreSgpr(&code, 2, 0);
AppendStoreSgpr(&code, 3, 1);
AppendStoreSgpr(&code, 4, 2);
AppendStoreSgpr(&code, 5, 3);
AppendStoreSgpr(&code, 7, 4);
AppendStoreSgpr(&code, 10, 5);
AppendStoreSgprPair(&code, 14, 6);
AppendStoreSgprPair(&code, 16, 8);
AppendEnd(&code);
return {"ScalarShiftAddAndMaskOps",
code,
{},
{7, 11, 19, 35, 0xfffffffeu, 1, 0x7fffffeeu, 0x7ffffffeu, 0xf00000ffu,
0xf000000fu},
{O::S_MOV_B32, O::S_LSHL1_ADD_U32, O::S_LSHL2_ADD_U32,
O::S_LSHL3_ADD_U32, O::S_LSHL4_ADD_U32, O::S_ASHR_I32,
O::S_MUL_HI_U32, O::S_NOT_B64, O::S_WQM_B64, O::V_MOV_B32,
O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}
TestCase ScalarNotB64UpdatesScc() {
using O = ShaderOpcode;
std::vector<u32> code;
auto set_scc = [&](bool value) {
code.push_back(EncodeSopc(0x06, InlineU32(1), InlineU32(value ? 1u : 0u)));
};
auto capture_scc = [&](u32 dst_sgpr) {
code.push_back(EncodeSop2(0x0a, dst_sgpr, InlineU32(1), InlineU32(0)));
};
AppendSMovLiteral(&code, 0, 0xffffffffu);
AppendSMovLiteral(&code, 1, 0xffffffffu);
set_scc(true);
code.push_back(EncodeSop1(0x08, 2, 0));
capture_scc(4);
AppendSMovLiteral(&code, 6, 0xfffffffeu);
AppendSMovLiteral(&code, 7, 0xffffffffu);
set_scc(false);
code.push_back(EncodeSop1(0x08, 8, 6));
capture_scc(10);
code.push_back(EncodeVopc(0xc2, Vgpr(0), 0));
code.push_back(EncodeSop1(0x04, 12, 106));
code.push_back(EncodeSop1(0x08, 14, 12));
capture_scc(16);
AppendStoreSgpr(&code, 4, 0);
AppendStoreSgpr(&code, 10, 1);
AppendStoreSgprPair(&code, 2, 2);
AppendStoreSgprPair(&code, 8, 4);
AppendStoreSgpr(&code, 16, 6);
AppendStoreSgprPair(&code, 14, 7);
AppendEnd(&code);
return {"ScalarNotB64UpdatesScc",
code,
{},
{0, 1, 0, 0, 1, 0, 1, 0xfffffffeu, 0xffffffffu},
{O::S_MOV_B32, O::S_CMP_EQ_U32, O::S_NOT_B64, O::S_CSELECT_B32,
O::V_CMP_EQ_U32, O::S_MOV_B64, O::V_MOV_B32, O::BUFFER_STORE_DWORD,
O::S_ENDPGM}};
}
TestCase ScalarQuadmaskB64() {
using O = ShaderOpcode;
std::vector<u32> code;
auto capture_scc = [&](u32 dst_sgpr) {
code.push_back(EncodeSop2(0x0a, dst_sgpr, InlineU32(1), InlineU32(0)));
};
AppendSMovLiteral(&code, 106, 0x80000011u);
AppendSMovLiteral(&code, 107, 0x000000f0u);
code.push_back(EncodeSop1(0x2d, 106, 106));
capture_scc(0);
AppendStoreSgprPair(&code, 106, 0);
AppendStoreSgpr(&code, 0, 2);
code.push_back(EncodeSMovB32(106, InlineU32(0)));
code.push_back(EncodeSMovB32(107, InlineU32(0)));
code.push_back(EncodeSop1(0x2d, 106, 106));
capture_scc(0);
AppendStoreSgprPair(&code, 106, 3);
AppendStoreSgpr(&code, 0, 5);
AppendEnd(&code);
return {"ScalarQuadmaskB64",
code,
{},
{0x00000283u, 0, 1, 0, 0, 0},
{O::S_MOV_B32, O::S_QUADMASK_B64, O::S_CSELECT_B32, O::V_MOV_B32,
O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}
TestCase ScalarFlbitI32B64Gpu() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendSMovLiteral(&code, 0, 0x00000000u);
AppendSMovLiteral(&code, 1, 0x80000000u);
code.push_back(EncodeSop1(0x16, 20, 0));
AppendSMovLiteral(&code, 2, 0x00008000u);
AppendSMovLiteral(&code, 3, 0x00000000u);
code.push_back(EncodeSop1(0x16, 21, 2));
AppendSMovLiteral(&code, 4, 0x00000000u);
AppendSMovLiteral(&code, 5, 0x00000000u);
code.push_back(EncodeSop1(0x16, 22, 4));
AppendSMovLiteral(&code, 14, 0x00000008u);
AppendSMovLiteral(&code, 15, 0x00000000u);
code.push_back(EncodeSop1(0x16, 106, 14));
AppendStoreSgpr(&code, 20, 0);
AppendStoreSgpr(&code, 21, 1);
AppendStoreSgpr(&code, 22, 2);
AppendStoreSgpr(&code, 106, 3);
AppendEnd(&code);
return {"ScalarFlbitI32B64Gpu",
code,
{},
{0, 48, 0xffffffffu, 60},
{O::S_MOV_B32, O::S_FLBIT_I32_B64, O::V_MOV_B32,
O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}
TestCase ScalarFlbitI32B32Gpu() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendSMovLiteral(&code, 0, 0x00000000u);
code.push_back(EncodeSop1(0x15, 20, 0));
AppendSMovLiteral(&code, 0, 0x0000ccccu);
code.push_back(EncodeSop1(0x15, 21, 0));
AppendSMovLiteral(&code, 0, 0x80000000u);
code.push_back(EncodeSop1(0x15, 22, 0));
AppendSMovLiteral(&code, 0, 0x7fffffffu);
code.push_back(EncodeSop1(0x15, 23, 0));
AppendStoreSgpr(&code, 20, 0);
AppendStoreSgpr(&code, 21, 1);
AppendStoreSgpr(&code, 22, 2);
AppendStoreSgpr(&code, 23, 3);
AppendEnd(&code);
return {"ScalarFlbitI32B32Gpu",
code,
{},
{0xffffffffu, 16, 0, 1},
{O::S_MOV_B32, O::S_FLBIT_I32_B32, O::V_MOV_B32,
O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}
TestCase ScalarSaveExecOps() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendSMovLiteral(&code, 106, 0xffffffffu);
AppendSMovLiteral(&code, 107, 0xffffffffu);
code.push_back(0xbe80246au);
code.push_back(EncodeSop1(0x28, 2, 106));
code.push_back(EncodeSMovB32(106, InlineU32(0)));
code.push_back(EncodeSMovB32(107, InlineU32(0)));
code.push_back(EncodeSop1(0x37, 4, 106));
AppendSMovLiteral(&code, 106, 0x00000003u);
code.push_back(EncodeSop1(0x3c, 8, 106));
code.push_back(EncodeSop1(0x04, 10, 126));
code.push_back(EncodeSMovB32(106, InlineU32(1)));
code.push_back(EncodeSop1(0x44, 12, 106));
code.push_back(EncodeSop1(0x04, 14, 126));
AppendSMovLiteral(&code, 126, 0xffffffffu);
AppendSMovLiteral(&code, 127, 0xffffffffu);
AppendStoreSgprPair(&code, 0, 0);
AppendStoreSgprPair(&code, 2, 2);
AppendStoreSgprPair(&code, 4, 4);
AppendStoreSgpr(&code, 8, 6);
AppendStoreSgprPair(&code, 10, 7);
AppendStoreSgpr(&code, 12, 9);
AppendStoreSgprPair(&code, 14, 10);
AppendStoreSgpr(&code, 253, 12);
AppendEnd(&code);
TestCase test;
test.name = "ScalarSaveExecOps";
test.code = code;
// A full wave enters with EXEC set for all 64 lanes, so the two saved EXEC
// pairs are 0xffffffff/0xffffffff rather than the 1/0 a single-invocation
// dispatch would report.
test.expected = {0xffffffffu, 0xffffffffu, 0xffffffffu, 0xffffffffu,
0xffffffffu, 0xffffffffu, 0xffffffffu, 3, 0xffffffffu,
3, 2, 0xffffffffu, 1};
test.opcodes = {O::S_MOV_B32, O::S_AND_SAVEEXEC_B64, O::S_ORN2_SAVEEXEC_B64,
O::S_ANDN1_SAVEEXEC_B64, O::S_AND_SAVEEXEC_B32,
O::S_ANDN1_SAVEEXEC_B32, O::S_MOV_B64, O::V_MOV_B32,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.compute_info.threads_num[0] = 64;
test.has_compute_info = true;
return test;
}
TestCase ScalarOrn2SaveexecUsesSourceOrNotExec() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendSMovLiteral(&code, 126, 0x0000000cu);
AppendSMovLiteral(&code, 127, 0x80000000u);
AppendSMovLiteral(&code, 0, 0x00000001u);
AppendSMovLiteral(&code, 1, 0x00000001u);
code.push_back(EncodeSop1(0x28, 2, 0));
code.push_back(EncodeSop1(0x04, 4, 126));
code.push_back(EncodeSMovB32(126, InlineU32(1)));
code.push_back(EncodeSMovB32(127, InlineU32(0)));
AppendStoreSgprPair(&code, 2, 0);
AppendStoreSgprPair(&code, 4, 2);
AppendStoreSgpr(&code, 253, 4);
AppendEnd(&code);
TestCase test;
test.name = "ScalarOrn2SaveexecUsesSourceOrNotExec";
test.code = code;
test.expected = {0x0000000cu, 0x80000000u, 0xfffffff3u, 0x7fffffffu, 1};
test.opcodes = {O::S_MOV_B32, O::S_ORN2_SAVEEXEC_B64, O::S_MOV_B64, O::V_MOV_B32,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
// The case reads a whole 64-bit EXEC mask, so it needs a populated subgroup.
// A 1x1x1 dispatch leaves OpGroupNonUniformBallot with a single invocation
// and every lane above zero reads back as zero.
test.compute_info.threads_num[0] = 64;
test.has_compute_info = true;
return test;
}
TestCase ScalarOrn2SaveexecB32(u32 wave_size, u32 threads) {
using O = ShaderOpcode;
constexpr u32 exec_hi = 0xa5a55a5au;
constexpr u32 vcc_hi = 0x13579bdfu;
struct Case {
u32 exec_lo, source, destination, result;
};
constexpr Case cases[] = {
{0x8000000cu, 126, 106, 0xffffffffu}, // Captured vcc_lo, exec_lo.
{0x0000000fu, 6, 6, 0xfffffff3u}, // Aliased SGPR source/destination.
{0xffffffffu, 128, 8, 0}, // SCC ignores the nonzero high half.
{0xffffffffu, 107, 8, vcc_hi},
{0xffffffffu, 127, 8, exec_hi},
{0xffffffffu, 253, 8, 1}, // SCC is a scalar value, not a mask.
{0xffffffffu, 251, 8, 0}, // VCCZ summarizes the whole mask.
{0x00000001u, 252, 8, 0xfffffffeu}, // EXECZ is uniform across lanes.
};
TestCase test;
test.name = wave_size == 64 ? "ScalarOrn2SaveexecB32Wave64"
: threads == 32 ? "ScalarOrn2SaveexecB32Wave32"
: "ScalarOrn2SaveexecB32PartialWave";
auto &code = test.code;
// Zero flags ignore high bits only in wave32 and include inactive low bits.
constexpr u32 masks[][2] = {{0, 0}, {0x80000000u, 0}, {0, 0x80000000u}};
for (const auto &mask : masks) {
AppendSMovLiteral(&code, 106, mask[0]);
AppendSMovLiteral(&code, 107, mask[1]);
AppendSMovLiteral(&code, 126, mask[0]);
AppendSMovLiteral(&code, 127, mask[1]);
code.push_back(EncodeSMovB32(20, 251));
code.push_back(EncodeSMovB32(21, 252));
code.push_back(EncodeSop1(0x04, 126, 193u));
const u32 zero = mask[0] == 0 && (wave_size == 32 || mask[1] == 0);
for (u32 reg = 20; reg <= 21; ++reg) {
AppendStoreSgprAtLaneDwordOffset(&code, reg, 0,
static_cast<u32>(test.expected.size()));
test.expected.insert(test.expected.end(), threads, zero);
}
}
for (const auto &item : cases) {
AppendVMovU32(&code, 2, 0);
AppendSMovLiteral(&code, 126, item.exec_lo);
AppendSMovLiteral(&code, 127, exec_hi);
AppendSMovLiteral(&code, 106, 1);
AppendSMovLiteral(&code, 107, vcc_hi);
AppendSMovLiteral(&code, 6, 3);
code.push_back(EncodeSopc(0x06, InlineU32(0), InlineU32(0)));
code.push_back(item.source == 126 ? 0xbeea407eu
: EncodeSop1(0x40, item.destination, item.source));
const u32 snapshots[] = {item.destination, 126, 127, 107, 253};
for (u32 i = 0; i < std::size(snapshots); ++i) {
code.push_back(EncodeSMovB32(20 + i, snapshots[i]));
}
AppendVMovU32(&code, 2, 1);
code.push_back(EncodeSop1(0x04, 126, 193u)); // Restore EXEC for readback.
const u32 expected[] = {item.exec_lo, item.result, exec_hi, vcc_hi,
item.result != 0 ? 1u : 0u};
for (u32 i = 0; i < std::size(expected); ++i) {
AppendStoreSgprAtLaneDwordOffset(&code, 20 + i, 0,
static_cast<u32>(test.expected.size()));
test.expected.insert(test.expected.end(), threads, expected[i]);
}
AppendStoreVgprAtLaneDwordOffset(&code, 2, 0,
static_cast<u32>(test.expected.size()));
for (u32 lane = 0; lane < threads; ++lane) {
const u32 mask = lane < 32 ? item.result : exec_hi;
test.expected.push_back((mask >> (lane % 32)) & 1u);
}
}
AppendEnd(&code);
test.initial.resize(test.expected.size());
test.opcodes = {O::S_MOV_B32, O::S_MOV_B64, O::S_CMP_EQ_U32,
O::S_ORN2_SAVEEXEC_B32, O::V_MOV_B32, O::V_LSHLREV_B32,
O::V_ADD_NC_U32, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.decoded_counts = {{"S_ORN2_SAVEEXEC_B32 vcc_lo, exec_lo", 1}};
test.compute_info.threads_num[0] = threads;
test.compute_info.threads_num[1] = test.compute_info.threads_num[2] = 1;
test.compute_info.thread_ids_num = 1;
test.compute_info.wave_size = wave_size;
test.has_compute_info = true;
return test;
}
TestCase ScalarSubvectorLoops(u32 wave_size) {
using O = ShaderOpcode;
using namespace ShaderRecompiler::Decoder;
TestCase test;
test.name = wave_size == 64 ? "ScalarSubvectorLoopsWave64"
: "ScalarSubvectorLoopsWave32";
// The captured mesh shader's signed SOPK targets are relative to PC + 4.
std::array<u32, 47> captured{};
captured[0x30 / 4] = 0xbda60022u;
captured[0xb8 / 4] = 0xbe26ffdeu;
Instruction begin, end;
DecodeInstruction(captured, 0x30 / 4, begin);
DecodeInstruction(captured, 0xb8 / 4, end);
Require(test.name, "captured branch decode",
begin.opcode == O::S_SUBVECTOR_LOOP_BEGIN && begin.dst.reg == 38 &&
begin.dst.kind == OperandKind::Sgpr && begin.branch_target == 0xbc &&
end.opcode == O::S_SUBVECTOR_LOOP_END && end.dst.reg == 38 &&
end.dst.kind == OperandKind::Sgpr && end.branch_target == 0x34 &&
begin.word_count == 1 && end.word_count == 1,
"subvector SDST or signed PC-relative target was decoded incorrectly");
constexpr u32 sentinel = 0x5a13abcdu;
constexpr u32 low = 0x80000005u, high = 0x42000002u;
struct Case {
u32 lo, hi, body_mode, saved, final_lo, final_hi, passes;
};
constexpr Case cases[] = {
{0, 0, 0, sentinel, 0, 0, 0},
{low, 0, 0, 0, low, 0, 1},
{0, high, 0, 0, 0, high, 1},
{low, high, 0, low, low, high, 2},
{~0u, ~0u, 0, ~0u, ~0u, ~0u, 2},
// The game resets the full EXEC mask before END, selecting its HI != 0 arm.
{low, high, 1, high, high, ~0u, 1},
// END must restore the low mask saved after the body has changed it.
{low, high, 2, 0, 0, high, 2},
// BEGIN EXEC_LO first copies HI to its aliased destination, then clears HI.
{low, high, 3, 0, 0, high, 2},
};
auto &code = test.code;
for (u32 index = 0; index < std::size(cases); ++index) {
const auto &item = cases[index];
const u32 scc = index & 1u;
const u32 saved_reg = item.body_mode == 3 ? 126u : 38u;
AppendVMovU32(&code, 2, 0);
AppendVMovU32(&code, 3, 0);
AppendSMovLiteral(&code, 38, sentinel);
AppendSMovLiteral(&code, 32, 1); // Scalar 2^passes; MULK preserves SCC.
AppendSMovLiteral(&code, 33, 0xff);
AppendSMovLiteral(&code, 126, item.lo);
AppendSMovLiteral(&code, 127, item.hi);
code.push_back(EncodeSopc(0x06, InlineU32(0), InlineU32(scc == 0 ? 1 : 0)));
const auto begin_word = static_cast<u32>(code.size());
code.push_back(0);
const auto body_word = static_cast<u32>(code.size());
code.push_back(EncodeSMovB32(33, 253)); // Observe SCC immediately after BEGIN.
code.push_back(EncodeSopk(0x10, 32, 2));
code.push_back(EncodeVop2(0x25, 2, InlineU32(1), 2));
code.push_back(EncodeVop1(0x01, 3, 32));
if (item.body_mode == 1) {
code.push_back(EncodeSop1(0x04, 126, 193)); // S_MOV_B64 EXEC, -1.
} else if (item.body_mode == 2) {
code.push_back(EncodeSMovB32(126, InlineU32(0)));
}
const auto end_word = static_cast<u32>(code.size());
code.push_back(EncodeSopk(0x1c, saved_reg, body_word - end_word - 1u));
code[begin_word] = EncodeSopk(0x1b, saved_reg, end_word - begin_word);
const u32 sources[] = {saved_reg, 126, 127, 253, 32, 33};
for (u32 i = 0; i < std::size(sources); ++i) {
code.push_back(EncodeSMovB32(20 + i, sources[i]));
}
code.push_back(EncodeSop1(0x04, 126, 193)); // Restore all lanes for readback.
const u32 expected[] = {item.saved, item.final_lo, item.final_hi, scc,
1u << item.passes, item.passes == 0 ? 0xff : scc};
for (u32 i = 0; i < std::size(expected); ++i) {
AppendStoreSgprAtLaneDwordOffset(&code, 20 + i, 0,
static_cast<u32>(test.expected.size()));
test.expected.insert(test.expected.end(), wave_size, expected[i]);
}
for (u32 reg : {2u, 3u}) {
AppendStoreVgprAtLaneDwordOffset(&code, reg, 0,
static_cast<u32>(test.expected.size()));
for (u32 lane = 0; lane < wave_size; ++lane) {
const bool upper = lane >= 32;
const u32 mask = upper || item.body_mode == 3 ? item.hi : item.lo;
const bool active = ((mask >> (lane % 32)) & 1u) != 0 &&
!(upper && item.body_mode == 1);
const u32 pass_value = upper && item.lo != 0 ? 4u : 2u;
test.expected.push_back(active ? (reg == 2 ? 1u : pass_value) : 0u);
}
}
}
AppendEnd(&code);
test.initial.resize(test.expected.size());
test.opcodes = {O::S_SUBVECTOR_LOOP_BEGIN, O::S_SUBVECTOR_LOOP_END,
O::S_MOV_B32, O::S_MOV_B64, O::S_MULK_I32, O::S_CMP_EQ_U32,
O::V_MOV_B32, O::V_ADD_NC_U32, O::V_LSHLREV_B32,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.compute_info.threads_num[0] = wave_size;
test.compute_info.threads_num[1] = test.compute_info.threads_num[2] = 1;
test.compute_info.thread_ids_num = 1;
test.compute_info.wave_size = wave_size;
test.has_compute_info = true;
return test;
}
TestCase ScalarGetpcWritesNextInstructionPc() {
using O = ShaderOpcode;
std::vector<u32> code;
code.push_back(EncodeSop1(0x1f, 0, 0));
AppendStoreSgprPair(&code, 0, 0);
AppendEnd(&code);
return {"ScalarGetpcWritesNextInstructionPc",
code,
{},
{4, 0},
{O::S_GETPC_B64, O::V_MOV_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}
TestCase ScalarBitcmpB64DynamicOperands() {
using O = ShaderOpcode;
struct Input {
u32 low;
u32 high;
u32 index;
};
const Input inputs[] = {
{1u, 0u, 0u}, {0x80000000u, 0u, 31u}, {0u, 1u, 32u},
{0u, 0x80000000u, 63u}, {1u, 0u, 64u},
{0x80000000u, 0u, 95u}, {0u, 1u, 96u},
{0u, 0x80000000u, 127u}, {1u, 0u, 128u},
{0u, 0x80000000u, 0xffffffffu}, {0u, 0u, 17u},
{0xffffffffu, 0xffffffffu, 42u},
{0x89abcdefu, 0x76543210u, 4u},
{0x89abcdefu, 0x76543210u, 36u},
};
TestCase test;
test.name = "ScalarBitcmpB64DynamicOperands";
for (const auto &input : inputs) {
test.initial.insert(test.initial.end(), {input.low, input.high, input.index});
}
test.expected = test.initial;
for (u32 i = 0; i < std::size(inputs); ++i) {
// Load the value and index on the GPU so the compares cannot constant-fold.
for (u32 component = 0; component < 3; ++component) {
AppendVMovU32(&test.code, 30, (i * 3 + component) * 4);
AppendBufferLoadDword(&test.code, 0, 30);
test.code.push_back(EncodeVop1(0x02, 20 + component, Vgpr(0)));
}
const uint64_t value = uint64_t{inputs[i].low} | (uint64_t{inputs[i].high} << 32);
const u32 bit = static_cast<u32>((value >> (inputs[i].index & 63u)) & 1u);
for (u32 expected_bit = 0; expected_bit < 2; ++expected_bit) {
const bool scc = bit == expected_bit;
test.code.push_back(EncodeSopc(0x0e + expected_bit, 20, 22));
test.code.push_back(EncodeSop2(0x0a, 24, InlineU32(1), InlineU32(0)));
AppendStoreSgpr(&test.code, 24, static_cast<u32>(test.expected.size()));
test.expected.push_back(scc ? 1u : 0u);
// Exercise SCC-driven control flow as well as S_CSELECT_B32.
test.code.push_back(EncodeSopp(0x05, 2)); // s_cbranch_scc1 to the true arm.
test.code.push_back(EncodeSMovB32(24, InlineU32(3)));
test.code.push_back(EncodeSopp(0x02, 1)); // s_branch past the true arm.
test.code.push_back(EncodeSMovB32(24, InlineU32(7)));
AppendStoreSgpr(&test.code, 24, static_cast<u32>(test.expected.size()));
test.expected.push_back(scc ? 7u : 3u);
}
}
AppendEnd(&test.code);
test.initial.resize(test.expected.size());
test.opcodes = {O::V_MOV_B32, O::BUFFER_LOAD_DWORD, O::V_READFIRSTLANE_B32,
O::S_BITCMP0_B64, O::S_BITCMP1_B64, O::S_CSELECT_B32,
O::S_CBRANCH_SCC1, O::S_BRANCH, O::S_MOV_B32,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.required_spirv = {"OpBitFieldUExtract", "OpULessThan", "OpSelect"};
return test;
}
TestCase ScalarBitcmpB64IntegerConstants() {
using O = ShaderOpcode;
struct Source {
u32 encoding;
uint64_t bits;
};
// Negative integer inlines sign-extend; B64 literals zero-extend.
const Source sources[] = {
{128u, 0u}, {129u, 1u}, {192u, 64u},
{193u, 0xffffffffffffffffull}, {208u, 0xfffffffffffffff0ull},
{255u, 0xffffffffu}, {255u, 0x80000000u},
};
const u32 indices[] = {0u, 4u, 6u, 31u, 32u, 63u, 64u, 127u};
TestCase test;
test.name = "ScalarBitcmpB64IntegerConstants";
test.initial.assign(std::begin(indices), std::end(indices));
test.expected = test.initial;
for (u32 i = 0; i < std::size(indices); ++i) {
AppendVMovU32(&test.code, 30, i * 4);
AppendBufferLoadDword(&test.code, 0, 30);
test.code.push_back(EncodeVop1(0x02, 22, Vgpr(0)));
for (const auto &source : sources) {
const u32 bit = static_cast<u32>((source.bits >> (indices[i] & 63u)) & 1u);
for (u32 expected_bit = 0; expected_bit < 2; ++expected_bit) {
test.code.push_back(EncodeSopc(0x0e + expected_bit, source.encoding, 22));
if (source.encoding == 255u) {
test.code.push_back(static_cast<u32>(source.bits));
}
test.code.push_back(EncodeSop2(0x0a, 24, InlineU32(1), InlineU32(0)));
AppendStoreSgpr(&test.code, 24, static_cast<u32>(test.expected.size()));
test.expected.push_back(bit == expected_bit ? 1u : 0u);
}
}
}
// An instruction has one literal word even when both sources select it.
for (u32 expected_bit = 0; expected_bit < 2; ++expected_bit) {
test.code.push_back(EncodeSopc(0x0e + expected_bit, 255u, 255u));
test.code.push_back(0x8000001fu); // Select bit 31 of the same zero-extended literal.
test.code.push_back(EncodeSop2(0x0a, 24, InlineU32(1), InlineU32(0)));
AppendStoreSgpr(&test.code, 24, static_cast<u32>(test.expected.size()));
test.expected.push_back(expected_bit);
}
AppendEnd(&test.code);
test.initial.resize(test.expected.size());
test.opcodes = {O::V_MOV_B32, O::BUFFER_LOAD_DWORD, O::V_READFIRSTLANE_B32,
O::S_BITCMP0_B64, O::S_BITCMP1_B64, O::S_CSELECT_B32,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.required_spirv = {"OpBitFieldUExtract"};
return test;
}
TestCase ScalarBitfieldPack() {
using O = ShaderOpcode;
std::vector<u32> code;
code.push_back(EncodeSMovB32(0, InlineU32(15)));
code.push_back(EncodeSop1(0x0b, 1, 0));
AppendSMovLiteral(&code, 2, 0xf0000000u);
code.push_back(EncodeSop1(0x10, 3, 0));
AppendSMovLiteral(&code, 19, 0xf0f00001u);
code.push_back(EncodeSop1(0x0f, 20, 19));
code.push_back(EncodeSop2(0x0a, 21, InlineU32(1), InlineU32(0)));
code.push_back(EncodeSMovB32(22, InlineU32(0)));
code.push_back(EncodeSop1(0x0f, 23, 22));
code.push_back(EncodeSop2(0x0a, 24, InlineU32(1), InlineU32(0)));
code.push_back(EncodeSMovB32(4, InlineU32(3)));
code.push_back(EncodeSop1(0x3b, 5, 4));
code.push_back(EncodeSop2(0x24, 7, InlineU32(4), InlineU32(8)));
AppendSMovLiteral(&code, 8, 0x00f00000u);
AppendSMovLiteral(&code, 9, 0x00040014u);
code.push_back(EncodeSop2(0x27, 10, 8, 9));
AppendSMovLiteral(&code, 11, 0xaaaabbbbu);
AppendSMovLiteral(&code, 12, 0xccccddddu);
code.push_back(EncodeSop2(0x32, 13, 11, 12));
code.push_back(EncodeSop2(0x33, 14, 11, 12));
code.push_back(EncodeSop2(0x34, 15, 11, 12));
code.push_back(EncodeSopc(0x0d, InlineU32(4), InlineU32(2)));
code.push_back(EncodeSop2(0x0a, 17, InlineU32(1), InlineU32(0)));
code.push_back(EncodeSopc(0x0c, InlineU32(4), InlineU32(1)));
code.push_back(EncodeSop2(0x0a, 18, InlineU32(1), InlineU32(0)));
AppendSMovLiteral(&code, 25, 0u);
AppendSMovLiteral(&code, 26, 0x00000100u);
code.push_back(EncodeSopc(0x06, InlineU32(0), InlineU32(1)));
code.push_back(EncodeSop1(0x14, 31, 25));
code.push_back(EncodeSop2(0x0a, 34, InlineU32(1), InlineU32(0)));
AppendSMovLiteral(&code, 27, 0x00000020u);
AppendSMovLiteral(&code, 28, 0x00000100u);
code.push_back(EncodeSopc(0x06, InlineU32(1), InlineU32(1)));
code.push_back(EncodeSop1(0x14, 32, 27));
code.push_back(EncodeSop2(0x0a, 35, InlineU32(1), InlineU32(0)));
AppendSMovLiteral(&code, 29, 0u);
AppendSMovLiteral(&code, 30, 0u);
code.push_back(EncodeSop1(0x14, 33, 29));
AppendStoreSgpr(&code, 1, 0);
AppendStoreSgpr(&code, 3, 1);
AppendStoreSgpr(&code, 20, 2);
AppendStoreSgpr(&code, 21, 3);
AppendStoreSgpr(&code, 23, 4);
AppendStoreSgpr(&code, 24, 5);
AppendStoreSgprPair(&code, 5, 6);
AppendStoreSgpr(&code, 7, 8);
AppendStoreSgpr(&code, 10, 9);
AppendStoreSgpr(&code, 13, 10);
AppendStoreSgpr(&code, 14, 11);
AppendStoreSgpr(&code, 15, 12);
AppendStoreSgpr(&code, 17, 13);
AppendStoreSgpr(&code, 18, 14);
AppendStoreSgpr(&code, 31, 15);
AppendStoreSgpr(&code, 32, 16);
AppendStoreSgpr(&code, 33, 17);
AppendStoreSgpr(&code, 34, 18);
AppendStoreSgpr(&code, 35, 19);
AppendEnd(&code);
return {"ScalarBitfieldPack",
code,
{},
{0xf0000000u, 8, 9, 1, 0,
0, 0x0000000fu, 0, 0x00000f00u, 0x0000000fu,
0xddddbbbbu, 0xccccbbbbu, 0xccccaaaau, 1, 1,
40, 5, 0xffffffffu, 0, 1},
{O::S_MOV_B32, O::S_BREV_B32, O::S_BCNT1_I32_B32, O::S_BCNT1_I32_B64,
O::S_FF1_I32_B64, O::S_BITREPLICATE_B64_B32, O::S_BFM_B32,
O::S_BFE_U32, O::S_PACK_LL_B32_B16, O::S_PACK_LH_B32_B16,
O::S_PACK_HH_B32_B16, O::S_BITCMP0_B32, O::S_BITCMP1_B32,
O::S_CSELECT_B32, O::V_MOV_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}
TestCase ScalarBrevB32PreservesScc() {
using O = ShaderOpcode;
std::vector<u32> code;
auto set_scc = [&](bool value) {
code.push_back(EncodeSopc(0x06, InlineU32(1), InlineU32(value ? 1u : 0u)));
};
auto capture_scc = [&](u32 dst_sgpr) {
code.push_back(EncodeSop2(0x0a, dst_sgpr, InlineU32(1), InlineU32(0)));
};
code.push_back(EncodeSMovB32(0, InlineU32(1)));
set_scc(false);
code.push_back(EncodeSop1(0x0b, 1, 0));
capture_scc(2);
code.push_back(EncodeSMovB32(3, InlineU32(0)));
set_scc(true);
code.push_back(EncodeSop1(0x0b, 4, 3));
capture_scc(5);
AppendStoreSgpr(&code, 1, 0);
AppendStoreSgpr(&code, 2, 1);
AppendStoreSgpr(&code, 4, 2);
AppendStoreSgpr(&code, 5, 3);
AppendEnd(&code);
return {"ScalarBrevB32PreservesScc",
code,
{},
{0x80000000u, 0, 0, 1},
{O::S_MOV_B32, O::S_CMP_EQ_U32, O::S_BREV_B32, O::S_CSELECT_B32,
O::V_MOV_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}
TestCase ScalarZeroBitOps() {
using O = ShaderOpcode;
constexpr uint64_t inputs[] = {0, UINT64_MAX, 0xaaaaaaaa55555555ull,
0xffffffff7fffffffull, 0xfffffffeffffffffull, 0x7fffffffffffffffull,
0x00000000ffffffffull, 0xffffffff00000000ull};
TestCase test;
test.name = "ScalarZeroBitOps";
for (const auto input : inputs)
test.initial.insert(test.initial.end(), {u32(input), u32(input >> 32)});
test.expected = test.initial;
auto &code = test.code;
const auto check = [&](u32 opcode, u32 source, uint64_t input, bool scc) {
const u32 width = opcode == 0x0e || opcode == 0x12 ? 64 : 32;
u32 count = 0, first = UINT32_MAX;
for (u32 bit = 0; bit < width; ++bit) {
if (((input >> bit) & 1u) == 0) {
++count;
if (first == UINT32_MAX) first = bit;
}
}
const bool counting = opcode == 0x0d || opcode == 0x0e;
code.push_back(EncodeSopc(0x06, InlineU32(1), InlineU32(scc)));
code.push_back(EncodeSop1(opcode, 20, source));
if (source == 255) code.push_back(u32(input));
code.push_back(EncodeSMovB32(22, 253)); // Capture SCC before readback.
AppendStoreSgpr(&code, 20, u32(test.expected.size()));
AppendStoreSgpr(&code, 21, u32(test.expected.size() + 1));
AppendStoreSgpr(&code, 22, u32(test.expected.size() + 2));
test.expected.insert(test.expected.end(),
{counting ? count : first, u32(input >> 32), u32(counting ? count != 0 : scc)});
};
for (u32 index = 0; index < std::size(inputs); ++index) {
for (u32 word = 0; word < 2; ++word) {
AppendVMovU32(&code, 30, (index * 2 + word) * sizeof(u32));
AppendBufferLoadDword(&code, 1, 30);
code.push_back(EncodeVop1(0x02, 8 + word, Vgpr(1)));
}
for (const u32 opcode : {0x0du, 0x0eu, 0x11u, 0x12u}) {
for (const bool scc : {false, true}) {
code.push_back(EncodeSop1(0x04, 20, 8)); // Destination aliases the source.
check(opcode, 20, inputs[index], scc);
}
}
}
for (const u32 opcode : {0x0du, 0x0eu, 0x11u, 0x12u}) {
for (const u32 source : {128u, 193u, 255u}) {
const uint64_t input = source == 128 ? 0 : source == 193 ? UINT64_MAX : 0xffffffffull;
AppendSMovLiteral(&code, 21, u32(input >> 32));
check(opcode, source, input, true); // Only inline -1 sign extends to B64.
}
}
AppendEnd(&code);
test.initial.resize(test.expected.size());
test.opcodes = {O::S_BCNT0_I32_B32, O::S_BCNT0_I32_B64,
O::S_FF0_I32_B32, O::S_FF0_I32_B64, O::S_CMP_EQ_U32, O::S_MOV_B32,
O::S_MOV_B64, O::V_MOV_B32, O::V_READFIRSTLANE_B32,
O::BUFFER_LOAD_DWORD, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
return test;
}
TestCase ScalarSextMasks(u32 bits, u32 wave_size) {
using O = ShaderOpcode;
TestCase test;
test.name = bits == 8 ? (wave_size == 32 ? "ScalarSextI8MasksWave32"
: "ScalarSextI8MasksWave64")
: (wave_size == 32 ? "ScalarSextI16MasksWave32"
: "ScalarSextI16MasksWave64");
constexpr u32 high = 0x80000001u;
test.initial = bits == 8
? std::vector<u32>{0xdeadff00u, 0xabcdff7fu, 0x12340080u, 0xabcf00ffu}
: std::vector<u32>{0xdead0000u, 0xabcd7fffu, 0x12348000u, 0xabcfffffu};
const u32 extended_values[] = {0u, bits == 8 ? 0x7fu : 0x7fffu,
bits == 8 ? 0xffffff80u : 0xffff8000u, 0xffffffffu};
test.expected = test.initial;
auto &code = test.code;
for (u32 i = 0; i < 4u; ++i) {
const auto extended = extended_values[i];
for (const bool scc : {false, true}) {
AppendVMovU32(&code, 30, i * 4u);
AppendBufferLoadDword(&code, 3, 30);
code.push_back(EncodeVop1(0x02, 11, Vgpr(3)));
AppendSMovLiteral(&code, 107, high);
code.push_back(EncodeSopc(0x06, InlineU32(scc ? 1u : 0u), InlineU32(1)));
code.push_back(EncodeSop1(0x04, 126, 128)); // Scalar operations execute with empty EXEC.
// The I16 word is captured s_sext_i32_i16 vcc_lo, s11.
code.push_back(bits == 8 ? 0xbeea190bu : 0xbeea1a0bu);
code.push_back(EncodeSMovB32(20, 106));
code.push_back(EncodeSMovB32(21, 107));
code.push_back(EncodeSMovB32(22, 253));
code.push_back(EncodeSop1(0x04, 126, 193));
AppendVMovU32(&code, 1, 1);
code.push_back(EncodeVop2(0x01, 2, InlineU32(0), 1));
const u32 expected[] = {extended, high, scc ? 1u : 0u};
for (u32 word = 0; word < 3u; ++word) {
AppendStoreSgprAtLaneDwordOffset(&code, 20u + word, 0, test.expected.size());
test.expected.insert(test.expected.end(), wave_size, expected[word]);
}
AppendStoreVgprAtLaneDwordOffset(&code, 2, 0, test.expected.size());
const uint64_t mask = (uint64_t{high} << 32u) | extended;
for (u32 lane = 0; lane < wave_size; ++lane)
test.expected.push_back(static_cast<u32>((mask >> lane) & 1u));
}
}
AppendEnd(&code);
test.initial.resize(test.expected.size());
test.opcodes = {O::BUFFER_LOAD_DWORD, O::V_READFIRSTLANE_B32, O::S_MOV_B32,
O::S_MOV_B64, O::S_CMP_EQ_U32,
bits == 8 ? O::S_SEXT_I32_I8 : O::S_SEXT_I32_I16,
O::V_CNDMASK_B32, O::V_MOV_B32, O::V_ADD_NC_U32,
O::V_LSHLREV_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.decoded_counts = {{bits == 8 ? "S_SEXT_I32_I8 vcc_lo, s11"
: "S_SEXT_I32_I16 vcc_lo, s11", 8u}};
test.required_spirv = {"OpBitFieldSExtract"};
test.compute_info.threads_num[0] = wave_size;
test.compute_info.threads_num[1] = test.compute_info.threads_num[2] = 1;
test.compute_info.thread_ids_num = 1;
test.compute_info.wave_size = wave_size;
test.has_compute_info = true;
return test;
}
TestCase ScalarSextConstantsAndAliases() {
using O = ShaderOpcode;
TestCase test;
test.name = "ScalarSextConstantsAndAliases";
auto &code = test.code;
for (const u32 opcode : {0x19u, 0x1au}) {
const u32 literal = opcode == 0x19u ? 0x12345680u : 0x12348000u;
const u32 extended = opcode == 0x19u ? 0xffffff80u : 0xffff8000u;
for (const bool scc : {false, true}) {
code.push_back(EncodeSopc(0x06, InlineU32(scc ? 1u : 0u), InlineU32(1)));
code.push_back(EncodeSop1(opcode, 10, 255));
code.push_back(literal);
code.push_back(EncodeSop1(opcode, 10, 10)); // In-place source/destination.
code.push_back(EncodeSop1(opcode, 11, InlineU32(0)));
code.push_back(EncodeSop1(opcode, 12, InlineU32(64)));
code.push_back(EncodeSop1(opcode, 13, 193)); // Inline -1.
code.push_back(EncodeSMovB32(14, 253));
const u32 expected[] = {extended, 0, 64, 0xffffffffu, scc ? 1u : 0u};
for (u32 word = 0; word < 5u; ++word) {
AppendStoreSgpr(&code, 10u + word, test.expected.size());
test.expected.push_back(expected[word]);
}
}
}
AppendEnd(&code);
test.initial.resize(test.expected.size());
test.opcodes = {O::S_CMP_EQ_U32, O::S_SEXT_I32_I8, O::S_SEXT_I32_I16,
O::S_MOV_B32, O::V_MOV_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.decoded_counts = {{"S_SEXT_I32_I8", 10u}, {"S_SEXT_I32_I16", 10u}};
return test;
}
TestCase ScalarBrevB64OperandsAndMasks(u32 wave_size) {
using O = ShaderOpcode;
TestCase test;
test.name = wave_size == 64 ? "ScalarBrevB64OperandsAndMasksWave64"
: "ScalarBrevB64OperandsAndMasksWave32";
constexpr uint64_t values[] = {0, 1, 0x8000000000000000ull,
0x0123456789abcdefull};
for (const auto value : values) {
test.initial.push_back(static_cast<u32>(value));
test.initial.push_back(static_cast<u32>(value >> 32u));
}
test.expected = test.initial;
auto &code = test.code;
size_t instructions = 0;
const auto check = [&](u32 dst, u32 src, uint64_t value, bool scc,
std::optional<u32> literal = {}) {
uint64_t reversed = 0;
for (u32 bit = 0; bit < 64; ++bit) {
reversed |= ((value >> bit) & 1ull) << (63u - bit);
}
AppendVMovU32(&code, 1, 1);
AppendVMovU32(&code, 2, 0);
if (dst == 106u || dst == 126u) {
code.push_back(EncodeSop1(0x04, dst, src));
src = dst;
}
code.push_back(EncodeSopc(0x06, InlineU32(scc ? 1u : 0u), InlineU32(1)));
code.push_back(EncodeSop1(0x0c, dst, src));
if (literal) {
code.push_back(*literal);
}
code.push_back(EncodeSMovB32(24, dst));
code.push_back(EncodeSMovB32(25, dst + 1u));
code.push_back(EncodeSMovB32(26, 253));
if (dst == 106u) {
code.push_back(EncodeVop2(0x01, 2, InlineU32(0), 1));
} else if (dst == 126u) {
code.push_back(EncodeVop1(0x01, 2, InlineU32(1)));
}
code.push_back(EncodeSop1(0x04, 126, 193)); // Restore EXEC for readback.
const u32 expected[] = {static_cast<u32>(reversed),
static_cast<u32>(reversed >> 32u), scc ? 1u : 0u};
for (u32 i = 0; i < std::size(expected); ++i) {
AppendStoreSgprAtLaneDwordOffset(
&code, 24 + i, 0, static_cast<u32>(test.expected.size()));
test.expected.insert(test.expected.end(), wave_size, expected[i]);
}
if (dst == 106u || dst == 126u) {
AppendStoreVgprAtLaneDwordOffset(
&code, 2, 0, static_cast<u32>(test.expected.size()));
for (u32 lane = 0; lane < wave_size; ++lane) {
test.expected.push_back(static_cast<u32>((reversed >> lane) & 1u));
}
}
++instructions;
};
for (const u32 dst : {20u, 106u, 126u}) {
for (u32 i = 0; i < std::size(values); ++i) {
// Runtime inputs exercise the emitter, including in-place mask writes.
for (u32 word = 0; word < 2; ++word) {
AppendVMovU32(&code, 30, (2 * i + word) * 4u);
AppendBufferLoadDword(&code, 3, 30);
code.push_back(EncodeVop1(0x02, 20 + word, Vgpr(3)));
}
check(dst, 20, values[i], values[i] == 0);
}
}
check(20, 129, 1, false);
check(20, 193, ~uint64_t{0}, false);
check(20, 125, 0, true);
check(20, 255, 0x80000001u, true, 0x80000001u);
AppendEnd(&code);
test.initial.resize(test.expected.size());
test.opcodes = {O::BUFFER_LOAD_DWORD, O::V_READFIRSTLANE_B32, O::S_MOV_B32,
O::S_MOV_B64, O::S_BREV_B64, O::S_CMP_EQ_U32, O::V_MOV_B32,
O::V_CNDMASK_B32, O::V_LSHLREV_B32, O::V_ADD_NC_U32,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.decoded_counts = {{"S_BREV_B64", instructions}};
test.required_spirv = {"OpBitReverse"};
test.forbidden_spirv = {"OpTypeInt 64"};
test.compute_info.threads_num[0] = wave_size;
test.compute_info.threads_num[1] = test.compute_info.threads_num[2] = 1;
test.compute_info.thread_ids_num = 1;
test.compute_info.wave_size = wave_size;
test.has_compute_info = true;
return test;
}
TestCase ScalarBfeI32CapturedRawSignExtends() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendBufferLoadDword(&code, 0, 30);
code.push_back(EncodeVop1(0x02, 20, Vgpr(0)));
code.push_back(0x946aff14u); // s_bfe_i32 vcc_lo, s20, literal
code.push_back(0x0004001cu); // offset 28, width 4
code.push_back(EncodeSop2(0x0a, 21, InlineU32(1), InlineU32(0)));
code.push_back(EncodeSop2(0x28, 22, 20, 255u));
code.push_back(0u); // zero width produces zero and clears SCC
code.push_back(EncodeSop2(0x0a, 23, InlineU32(1), InlineU32(0)));
AppendStoreSgpr(&code, 106, 1);
AppendStoreSgpr(&code, 21, 2);
AppendStoreSgpr(&code, 22, 3);
AppendStoreSgpr(&code, 23, 4);
AppendEnd(&code);
TestCase test;
test.name = "ScalarBfeI32CapturedRawSignExtends";
test.code = std::move(code);
test.initial = {0xf0000000u, 0, 0, 0, 0};
test.expected = {0xf0000000u, 0xffffffffu, 1, 0, 0};
test.opcodes = {O::BUFFER_LOAD_DWORD, O::V_READFIRSTLANE_B32,
O::S_BFE_I32, O::S_CSELECT_B32,
O::V_MOV_B32, O::BUFFER_STORE_DWORD,
O::S_ENDPGM};
test.required_spirv = {"OpBitFieldSExtract"};
return test;
}
TestCase BitfieldExtractWidthPastEndEdges() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendSMovLiteral(&code, 0, 0xf0000000u);
AppendSMovLiteral(&code, 1, (20u << 16u) | 20u);
code.push_back(EncodeSop2(0x27, 2, 0, 1));
AppendSMovLiteral(&code, 3, 0x80000000u);
AppendSMovLiteral(&code, 4, (7u << 16u) | 31u);
code.push_back(EncodeSop2(0x27, 5, 3, 4));
AppendVMovLiteral(&code, 6, 0xf0000000u);
AppendVMovU32(&code, 7, 28);
AppendVMovU32(&code, 8, 8);
AppendVop3(&code, 0x148, 9, Vgpr(6), Vgpr(7), Vgpr(8));
AppendStoreSgpr(&code, 2, 0);
AppendStoreSgpr(&code, 5, 1);
AppendStoreVgpr(&code, 9, 2);
AppendEnd(&code);
return {"BitfieldExtractWidthPastEndEdges",
code,
{},
{0x00000f00u, 1, 0x0000000fu},
{O::S_MOV_B32, O::S_BFE_U32, O::V_MOV_B32, O::V_BFE_U32,
O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}
TestCase Scalar64BitOps() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendSMovLiteral(&code, 0, 0x0f0f0f0fu);
AppendSMovLiteral(&code, 1, 0x00ff00ffu);
AppendSMovLiteral(&code, 2, 0x33333333u);
AppendSMovLiteral(&code, 3, 0x0f0f0f0fu);
code.push_back(EncodeSop2(0x0f, 4, 0, 2));
code.push_back(EncodeSop2(0x15, 6, 0, 2));
code.push_back(EncodeSop2(0x11, 8, 0, 2));
code.push_back(EncodeSop2(0x17, 10, 0, 2));
code.push_back(EncodeSop2(0x13, 12, 0, 2));
code.push_back(EncodeSop2(0x19, 14, 0, 2));
code.push_back(EncodeSop2(0x1b, 16, 0, 2));
code.push_back(EncodeSop2(0x1d, 18, 0, 2));
code.push_back(EncodeSop1(0x04, 20, 0));
code.push_back(EncodeSop2(0x1f, 22, 0, InlineU32(4)));
code.push_back(EncodeSop2(0x21, 24, 0, InlineU32(8)));
code.push_back(EncodeSop2(0x25, 26, InlineU32(36), InlineU32(4)));
code.push_back(EncodeSop2(0x21, 34, 193u, InlineU32(1)));
AppendSMovLiteral(&code, 28, 0x000c0004u);
code.push_back(EncodeSop2(0x29, 30, 0, 28));
code.push_back(EncodeSopc(0x06, 0, 0));
code.push_back(EncodeSop2(0x0b, 32, 0, 2));
const u32 result_pairs[] = {4, 6, 8, 10, 12, 14, 16, 18,
20, 22, 24, 26, 34, 30, 32};
u32 out = 0;
for (auto sgpr : result_pairs) {
AppendStoreSgprPair(&code, sgpr, out);
out += 2;
}
AppendEnd(&code);
return {"Scalar64BitOps",
code,
{},
{0x03030303u, 0x000f000fu, 0x0c0c0c0cu, 0x00f000f0u, 0x3f3f3f3fu,
0x0fff0fffu, 0xcfcfcfcfu, 0xf0fff0ffu, 0x3c3c3c3cu, 0x0ff00ff0u,
0xfcfcfcfcu, 0xfff0fff0u, 0xc0c0c0c0u, 0xf000f000u, 0xc3c3c3c3u,
0xf00ff00fu, 0x0f0f0f0fu, 0x00ff00ffu, 0xf0f0f0f0u, 0x0ff00ff0u,
0xff0f0f0fu, 0x0000ff00u, 0xfffffff0u, 0x000000ffu, 0xffffffffu,
0x7fffffffu, 0x000000f0u, 0, 0x0f0f0f0fu, 0x00ff00ffu},
{O::S_MOV_B32, O::S_MOV_B64, O::S_AND_B64, O::S_ANDN2_B64,
O::S_OR_B64, O::S_ORN2_B64, O::S_XOR_B64, O::S_NAND_B64,
O::S_NOR_B64, O::S_XNOR_B64, O::S_LSHL_B64, O::S_LSHR_B64,
O::S_BFM_B64, O::S_BFE_U64, O::S_CMP_EQ_U32, O::S_CSELECT_B64,
O::V_MOV_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}
TestCase ScalarDynamic64BitOps() {
using O = ShaderOpcode;
struct Input {
uint64_t lhs;
uint64_t rhs;
u32 count;
u32 offset;
};
constexpr Input inputs[] = {
{0, UINT64_MAX, 0, 0},
{UINT64_MAX, 0, 63, 0},
{0x8000000000000000ull, 1, 64, 0},
{0x0123456789abcdefull, 0xfedcba9876543210ull, 127, 0},
{UINT64_MAX, 0xffffffffull, 64, 1},
{0x0123456789abcdefull, 0x0123456789abcdefull, 63, 32},
{UINT64_MAX, 0x100000000ull, 127, 63},
{0, 0, 0, 127},
{0x8000000000000001ull, UINT64_MAX, UINT32_MAX, 64},
};
TestCase test;
test.name = "ScalarDynamic64BitOps";
for (const auto &input : inputs) {
test.initial.insert(test.initial.end(),
{u32(input.lhs), u32(input.lhs >> 32), u32(input.rhs),
u32(input.rhs >> 32), input.count, input.offset,
((input.count & 127u) << 16) | input.offset});
}
test.expected = test.initial;
auto &code = test.code;
const auto store = [&](uint64_t value, bool scc) {
const auto output = static_cast<u32>(test.expected.size());
code.push_back(EncodeSMovB32(18, 253));
AppendStoreSgprPair(&code, 16, output);
AppendStoreSgpr(&code, 18, output + 2);
test.expected.insert(test.expected.end(),
{u32(value), u32(value >> 32), u32(scc)});
};
for (u32 i = 0; i < std::size(inputs); ++i) {
const auto [lhs, rhs, count, offset] = inputs[i];
for (u32 word = 0; word < 7; ++word) {
AppendVMovU32(&code, 30, (i * 7 + word) * sizeof(u32));
AppendBufferLoadDword(&code, 1, 30);
code.push_back(EncodeVop1(0x02, 8 + word, Vgpr(1)));
}
for (const auto [opcode, value] :
std::array<std::pair<u32, uint64_t>, 8>{{{0x0f, lhs & rhs},
{0x11, lhs | rhs},
{0x13, lhs ^ rhs},
{0x15, lhs & ~rhs},
{0x17, lhs | ~rhs},
{0x19, ~(lhs & rhs)},
{0x1b, ~(lhs | rhs)},
{0x1d, ~(lhs ^ rhs)}}}) {
code.push_back(EncodeSopc(0x06, InlineU32(0), InlineU32(value != 0)));
code.push_back(EncodeSop2(opcode, 16, 8, 10));
store(value, value != 0);
}
for (const bool select_lhs : {false, true}) {
code.push_back(EncodeSopc(0x06, InlineU32(0), InlineU32(!select_lhs)));
code.push_back(EncodeSop2(0x0b, 16, 8, 10));
store(select_lhs ? lhs : rhs, select_lhs);
}
// BFM preserves SCC; BFE sets it from the entire extracted result.
code.push_back(EncodeSop2(0x25, 16, 12, 13));
store(((uint64_t{1} << (count & 63u)) - 1u) << (offset & 63u), true);
code.push_back(EncodeSop2(0x29, 16, 8, 14));
const auto width = std::min(count & 127u, 64u - (offset & 63u));
const auto mask = width == 64 ? UINT64_MAX : (uint64_t{1} << width) - 1u;
const auto extracted = (lhs >> (offset & 63u)) & mask;
store(extracted, extracted != 0);
for (const bool set : {false, true}) {
code.push_back(EncodeSop1(0x04, 16, 8));
code.push_back(EncodeSop1(set ? 0x1e : 0x1c, 16, 12));
const auto bit = uint64_t{1} << (count & 63u);
store(set ? lhs | bit : lhs & ~bit, extracted != 0);
}
code.push_back(EncodeSop1(0x2d, 16, 8));
uint64_t quads = 0;
for (u32 quad = 0; quad < 16; ++quad) {
quads |= uint64_t((lhs >> (quad * 4) & 15u) != 0) << quad;
}
store(quads, quads != 0);
code.push_back(EncodeSop1(0x3b, 16, 8));
uint64_t replicated = 0;
for (u32 bit = 0; bit < 32; ++bit) {
replicated |= ((lhs >> bit) & 1u) * (uint64_t{3} << (2 * bit));
}
store(replicated, quads != 0);
code.push_back(EncodeSMovB32(17, InlineU32(0)));
code.push_back(EncodeSop1(0x10, 16, 8));
store(std::popcount(lhs), lhs != 0);
}
AppendEnd(&code);
test.initial.resize(test.expected.size());
test.opcodes = {
O::V_MOV_B32, O::BUFFER_LOAD_DWORD, O::V_READFIRSTLANE_B32,
O::S_MOV_B32, O::S_CMP_EQ_U32, O::S_AND_B64,
O::S_OR_B64, O::S_XOR_B64, O::S_ANDN2_B64,
O::S_ORN2_B64, O::S_NAND_B64, O::S_NOR_B64,
O::S_XNOR_B64, O::S_CSELECT_B64, O::S_BFM_B64,
O::S_BFE_U64, O::S_MOV_B64, O::S_BITSET0_B64,
O::S_BITSET1_B64, O::S_QUADMASK_B64, O::S_BITREPLICATE_B64_B32,
O::S_BCNT1_I32_B64, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.required_spirv = {"OpTypeInt 64 0", "OpBitwiseAnd", "OpBitwiseOr",
"OpBitwiseXor"};
return test;
}
uint64_t ReferenceAshrI64(uint64_t bits, u32 count) {
const auto shift = count & 63u;
// Complementing negative values expresses sign fill using unsigned shifts.
return (bits & (uint64_t{1} << 63u)) != 0 ? ~(~bits >> shift) : bits >> shift;
}
TestCase ScalarAshrI64Edges(bool dynamic) {
using O = ShaderOpcode;
TestCase test;
test.name =
dynamic ? "ScalarAshrI64DynamicEdges" : "ScalarAshrI64ConstantEdges";
constexpr uint64_t sources[] = {0,
1,
0x80000000ull,
0x100000000ull,
0x7fffffffffffffffull,
0x8000000000000000ull,
0xffffffffffffffffull,
0x0123456789abcdefull,
0xfedcba9876543210ull};
constexpr u32 counts[] = {0, 1, 7, 31, 32, 33, 63,
64, 65, 95, 96, 127, 0xffffffffu};
test.initial.assign(std::begin(counts), std::end(counts));
for (const auto source : sources) {
test.initial.push_back(static_cast<u32>(source));
test.initial.push_back(static_cast<u32>(source >> 32u));
}
test.expected = test.initial;
auto &code = test.code;
const auto load_scalar = [&](u32 reg, u32 index) {
AppendVMovU32(&code, 30, index * sizeof(u32));
AppendBufferLoadDword(&code, 1, 30);
code.push_back(EncodeVop1(0x02, reg, Vgpr(1)));
};
for (u32 source_index = 0; source_index < std::size(sources);
++source_index) {
const auto source = sources[source_index];
// Runtime sources exercise both constant-count and dynamic-count emission.
load_scalar(20, static_cast<u32>(std::size(counts)) + source_index * 2u);
load_scalar(21,
static_cast<u32>(std::size(counts)) + source_index * 2u + 1u);
for (u32 index = 0; index < std::size(counts); ++index) {
const auto expected = ReferenceAshrI64(source, counts[index]);
if (dynamic) {
load_scalar(22, index);
}
// Start SCC at the opposite value so every result must update it.
code.push_back(
EncodeSopc(0x06, InlineU32(0), InlineU32(expected != 0 ? 1u : 0u)));
const auto count_src = dynamic ? 22u
: counts[index] <= 64u ? InlineU32(counts[index])
: 255u;
code.push_back(EncodeSop2(0x23, 24, 20, count_src));
if (!dynamic && count_src == 255u) {
code.push_back(counts[index]);
}
code.push_back(EncodeSMovB32(26, 253));
const auto output = static_cast<u32>(test.expected.size());
AppendStoreSgprPair(&code, 24, output);
AppendStoreSgpr(&code, 26, output + 2u);
test.expected.push_back(static_cast<u32>(expected));
test.expected.push_back(static_cast<u32>(expected >> 32u));
test.expected.push_back(expected != 0 ? 1u : 0u);
}
}
AppendEnd(&code);
test.initial.resize(test.expected.size());
test.opcodes = {O::BUFFER_LOAD_DWORD, O::V_READFIRSTLANE_B32,
O::S_MOV_B32, O::S_CMP_EQ_U32,
O::S_ASHR_I64, O::V_MOV_B32,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.decoded_counts = {
{"S_ASHR_I64", std::size(sources) * std::size(counts)}};
test.required_spirv = {"OpShiftRightArithmetic", "OpTypeInt 64 0"};
return test;
}
TestCase ScalarAshrI64OperandsAndAliases() {
using O = ShaderOpcode;
TestCase test;
test.name = "ScalarAshrI64OperandsAndAliases";
test.initial = {33}; // Runtime shift for signed literal source tests.
test.expected = test.initial;
auto &code = test.code;
AppendVMovU32(&code, 30, 0);
AppendBufferLoadDword(&code, 1, 30);
code.push_back(EncodeVop1(0x02, 22, Vgpr(1)));
size_t instructions = 0;
const auto check_result = [&](u32 dst, u32 src0, u32 src1, uint64_t value,
u32 count, std::optional<u32> literal = {}) {
code.push_back(EncodeSop2(0x23, dst, src0, src1));
if (literal) {
code.push_back(*literal);
}
code.push_back(EncodeSMovB32(26, 253));
const auto output = static_cast<u32>(test.expected.size());
AppendStoreSgprPair(&code, dst, output);
AppendStoreSgpr(&code, 26, output + 2u);
const auto expected = ReferenceAshrI64(value, count);
test.expected.push_back(static_cast<u32>(expected));
test.expected.push_back(static_cast<u32>(expected >> 32u));
test.expected.push_back(expected != 0 ? 1u : 0u);
++instructions;
};
for (const u32 literal : {0u, 0x7fffffffu, 0x80000000u, 0xffffffffu}) {
const uint64_t value =
literal | ((literal & 0x80000000u) != 0 ? 0xffffffff00000000ull : 0ull);
for (const u32 count : {0u, 1u, 31u, 32u, 63u, 64u}) {
check_result(24, 255, InlineU32(count), value, count, literal);
}
check_result(24, 255, 22, value, 33, literal);
}
for (const u32 source : {128u, 129u, 192u, 193u, 208u, 125u}) {
const uint64_t value = source == 125u ? 0ull
: source <= 192u ? source - 128u
: uint64_t{192} - source;
for (const u32 count : {0u, 1u, 32u, 63u}) {
check_result(24, source, InlineU32(count), value, count);
}
}
for (const u32 scc : {0u, 1u}) {
// SCC may be either source and must be read before it is overwritten.
code.push_back(EncodeSopc(0x06, InlineU32(scc), InlineU32(1)));
check_result(24, 253, InlineU32(1), scc, 1);
AppendSMovLiteral(&code, 20, 1);
AppendSMovLiteral(&code, 21, 0);
code.push_back(EncodeSopc(0x06, InlineU32(scc), InlineU32(1)));
check_result(24, 20, 253, 1, scc);
}
constexpr uint64_t value = 0xfedcba9889abcdefull;
for (const u32 count : {0u, 1u, 31u, 32u, 33u, 63u, 65u}) {
AppendSMovLiteral(&code, 20, static_cast<u32>(value));
AppendSMovLiteral(&code, 21, static_cast<u32>(value >> 32u));
AppendSMovLiteral(&code, 23, count);
check_result(20, 20, 23, value, count); // In-place source/destination pair.
for (const u32 count_reg : {24u, 25u}) {
AppendSMovLiteral(&code, 20, static_cast<u32>(value));
AppendSMovLiteral(&code, 21, static_cast<u32>(value >> 32u));
AppendSMovLiteral(&code, count_reg, count);
check_result(24, 20, count_reg, value,
count); // Count aliases destination.
}
}
for (const u32 count_reg : {20u, 21u}) {
const uint64_t alias_value =
count_reg == 20u ? 0x8000000000000021ull : 0x0000002100000001ull;
AppendSMovLiteral(&code, 20, static_cast<u32>(alias_value));
AppendSMovLiteral(&code, 21, static_cast<u32>(alias_value >> 32u));
check_result(20, 20, count_reg, alias_value, 33);
}
// Both operands may refer to the single literal word in the instruction.
check_result(24, 255, 255, 0xffffffffffffffffull, 0xffffffffu, 0xffffffffu);
for (const u32 source : {0u, 1u}) {
// Discarding the destination must still update SCC and leave M0 untouched.
AppendSMovLiteral(&code, 124, 0x5a13abcdu);
AppendSMovLiteral(&code, 20, 0);
AppendSMovLiteral(&code, 21, source);
code.push_back(EncodeSopc(0x06, InlineU32(0), InlineU32(source)));
code.push_back(EncodeSop2(0x23, 125, 20, InlineU32(0)));
code.push_back(EncodeSMovB32(26, 253));
const auto output = static_cast<u32>(test.expected.size());
AppendStoreSgpr(&code, 26, output);
AppendStoreSgpr(&code, 124, output + 1u);
AppendStoreSgprPair(&code, 20, output + 2u);
test.expected.insert(test.expected.end(), {source, 0x5a13abcdu, 0, source});
++instructions;
}
AppendEnd(&code);
test.initial.resize(test.expected.size());
test.opcodes = {O::BUFFER_LOAD_DWORD, O::V_READFIRSTLANE_B32,
O::S_MOV_B32, O::S_CMP_EQ_U32,
O::S_ASHR_I64, O::V_MOV_B32,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.decoded_counts = {{"S_ASHR_I64", instructions}};
test.required_spirv.push_back("OpTypeInt 64 0");
return test;
}
TestCase ScalarAshrI64Masks(u32 wave_size) {
using O = ShaderOpcode;
TestCase test;
test.name =
wave_size == 64 ? "ScalarAshrI64MasksWave64" : "ScalarAshrI64MasksWave32";
auto &code = test.code;
struct Case {
uint64_t source;
u32 count;
};
constexpr Case cases[] = {
{0x8000000100000001ull, 1}, {0x8000000100000001ull, 32},
{0x8000000100000001ull, 63}, {0x0000000100000000ull, 0},
{0x0000000100000000ull, 32}, {1, 1}};
for (const u32 mask_reg : {106u, 126u}) {
for (const auto &item : cases) {
const auto result = ReferenceAshrI64(item.source, item.count);
AppendVMovU32(&code, 1, 1);
AppendVMovU32(&code, 2, 0);
AppendSMovLiteral(&code, mask_reg, static_cast<u32>(item.source));
AppendSMovLiteral(&code, mask_reg + 1u,
static_cast<u32>(item.source >> 32u));
code.push_back(
EncodeSopc(0x06, InlineU32(0), InlineU32(result != 0 ? 1u : 0u)));
code.push_back(
EncodeSop2(0x23, mask_reg, mask_reg, InlineU32(item.count)));
code.push_back(EncodeSMovB32(20, mask_reg));
code.push_back(EncodeSMovB32(21, mask_reg + 1u));
code.push_back(EncodeSMovB32(22, 253));
if (mask_reg == 106u) {
code.push_back(EncodeVop2(0x01, 2, InlineU32(0), 1));
} else {
code.push_back(EncodeVop1(0x01, 2, InlineU32(1)));
}
code.push_back(EncodeSop1(0x04, 126, 193)); // Restore EXEC for readback.
const u32 expected[] = {static_cast<u32>(result),
static_cast<u32>(result >> 32u),
result != 0 ? 1u : 0u};
for (u32 i = 0; i < std::size(expected); ++i) {
AppendStoreSgprAtLaneDwordOffset(
&code, 20 + i, 0, static_cast<u32>(test.expected.size()));
test.expected.insert(test.expected.end(), wave_size, expected[i]);
}
AppendStoreVgprAtLaneDwordOffset(&code, 2, 0,
static_cast<u32>(test.expected.size()));
for (u32 lane = 0; lane < wave_size; ++lane) {
test.expected.push_back(static_cast<u32>((result >> lane) & 1u));
}
}
}
AppendEnd(&code);
test.initial.resize(test.expected.size());
test.opcodes = {O::S_MOV_B32, O::S_MOV_B64, O::S_CMP_EQ_U32,
O::S_ASHR_I64, O::V_MOV_B32, O::V_CNDMASK_B32,
O::V_LSHLREV_B32, O::V_ADD_NC_U32, O::BUFFER_STORE_DWORD,
O::S_ENDPGM};
test.decoded_counts = {{"S_ASHR_I64 vcc_lo, vcc_lo", std::size(cases)},
{"S_ASHR_I64 exec_lo, exec_lo", std::size(cases)}};
test.compute_info.threads_num[0] = wave_size;
test.compute_info.threads_num[1] = test.compute_info.threads_num[2] = 1;
test.compute_info.thread_ids_num = 1;
test.compute_info.wave_size = wave_size;
test.has_compute_info = true;
return test;
}
TestCase ScalarConditionalMoveB64() {
using O = ShaderOpcode;
TestCase test;
test.name = "ScalarConditionalMoveB64";
auto &code = test.code;
for (u32 scc : {0u, 1u}) {
AppendSMovLiteral(&code, 0, 0x13579bdfu);
AppendSMovLiteral(&code, 1, 0x2468ace0u);
AppendSMovLiteral(&code, 2, 0xabcdef01u);
AppendSMovLiteral(&code, 3, 0xfedcba98u);
code.push_back(EncodeSopc(0x06, InlineU32(scc), InlineU32(1)));
code.push_back(EncodeSop1(0x06, 0, 2));
code.push_back(EncodeSop2(0x0a, 4, InlineU32(1), InlineU32(0)));
AppendStoreSgprPair(&code, 0, scc * 3);
AppendStoreSgpr(&code, 4, scc * 3 + 2);
}
AppendEnd(&code);
test.expected = {0x13579bdfu, 0x2468ace0u, 0,
0xabcdef01u, 0xfedcba98u, 1};
test.opcodes = {O::S_MOV_B32, O::S_CMP_EQ_U32, O::S_CMOV_B64,
O::S_CSELECT_B32, O::V_MOV_B32, O::BUFFER_STORE_DWORD,
O::S_ENDPGM};
return test;
}
TestCase ScalarConditionalMoveB64PreservesMasks() {
using O = ShaderOpcode;
TestCase test;
test.name = "ScalarConditionalMoveB64PreservesMasks";
auto &code = test.code;
code.push_back(EncodeSop1(0x04, 16, 126)); // Save full EXEC.
AppendVMovU32(&code, 1, 3);
AppendVMovU32(&code, 2, 9);
u32 offset = 0;
for (u32 destination : {126u, 106u, 8u}) {
for (u32 scc : {0u, 1u}) {
code.push_back(EncodeVopc(0xc2, Vgpr(0), 1)); // Only lane 3.
code.push_back(EncodeSop1(0x04, destination, 106));
code.push_back(EncodeSopc(0x06, InlineU32(scc), InlineU32(1)));
// EXEC encoding is the captured Pathless instruction, 0xbefe06c1.
code.push_back(EncodeSop1(0x06, destination, 193u));
code.push_back(EncodeSop2(0x0a, 4, InlineU32(1), InlineU32(0)));
code.push_back(EncodeSop1(0x04, 126, destination));
AppendStoreVgprAtLaneDwordOffset(&code, 2, 0, offset);
code.push_back(EncodeSop1(0x04, 126, 16));
AppendStoreSgprAtLaneDwordOffset(&code, 4, 0, offset + 64);
for (u32 lane = 0; lane < 64; ++lane) {
test.expected.push_back(scc || lane == 3 ? 9u : 0xdeadbeefu);
}
test.expected.insert(test.expected.end(), 64, scc);
offset += 128;
}
}
AppendEnd(&code);
test.initial.assign(test.expected.size(), 0xdeadbeefu);
test.opcodes = {O::S_MOV_B64, O::V_MOV_B32, O::V_CMP_EQ_U32,
O::S_CMP_EQ_U32, O::S_CMOV_B64, O::S_CSELECT_B32,
O::V_LSHLREV_B32, O::V_ADD_NC_U32, O::BUFFER_STORE_DWORD,
O::S_ENDPGM};
test.compute_info.threads_num[0] = 64;
test.compute_info.threads_num[1] = test.compute_info.threads_num[2] = 1;
test.compute_info.thread_ids_num = 1;
test.compute_info.wave_size = 64;
test.has_compute_info = true;
return test;
}
TestCase ScalarAndn2B64SccUsesMaskShadow() {
using O = ShaderOpcode;
std::vector<u32> code = {
EncodeVopc(0xc2, InlineU32(0), 0),
EncodeSop1(0x04, 0, 106),
EncodeSop2(0x15, 4, 0, InlineU32(0)),
EncodeVop1(0x01, 0, InlineU32(1)),
EncodeSopp(0x04, 1),
EncodeVop1(0x01, 0, InlineU32(7)),
};
AppendStoreVgpr(&code, 0, 0);
AppendEnd(&code);
return {"ScalarAndn2B64SccUsesMaskShadow",
code,
{},
{7},
{O::V_CMP_EQ_U32, O::S_MOV_B64, O::S_ANDN2_B64, O::V_MOV_B32,
O::S_CBRANCH_SCC0, O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}
TestCase ScalarMaskHighWriteInvalidatesProvenance() {
using O = ShaderOpcode;
std::vector<u32> code = {
EncodeVopc(0xc2, Vgpr(0), 0), EncodeSop1(0x04, 0, 106),
EncodeSMovB32(1, InlineU32(1)), EncodeSop1(0x08, 2, 0),
EncodeVop1(0x01, 0, InlineU32(1)), EncodeSopp(0x04, 1),
EncodeVop1(0x01, 0, InlineU32(7)),
};
AppendStoreVgpr(&code, 0, 0);
AppendEnd(&code);
return {"ScalarMaskHighWriteInvalidatesProvenance",
code,
{},
{7},
{O::V_CMP_EQ_U32, O::S_MOV_B64, O::S_MOV_B32, O::S_NOT_B64,
O::V_MOV_B32, O::S_CBRANCH_SCC0, O::BUFFER_STORE_DWORD,
O::S_ENDPGM}};
}
TestCase ScalarSelectB64PreservesMaskProvenance(u32 wave_size) {
using O = ShaderOpcode;
TestCase test;
test.name = wave_size == 32 ? "ScalarSelectB64PreservesMaskProvenanceWave32"
: "ScalarSelectB64PreservesMaskProvenanceWave64";
auto &code = test.code;
const auto set_scc = [&] {
code.push_back(EncodeSopc(0x06, InlineU32(1), InlineU32(1)));
};
const auto capture_scc = [&](u32 dst) {
code.push_back(EncodeSop2(0x0a, dst, InlineU32(1), InlineU32(0)));
};
AppendSMovLiteral(&code, 0, 0xfffffffeu);
AppendSMovLiteral(&code, 1, 0xffffffffu);
code.push_back(EncodeSMovB32(2, InlineU32(0)));
code.push_back(EncodeSMovB32(3, InlineU32(0)));
set_scc();
code.push_back(EncodeSop2(0x0b, 4, 0, 2));
code.push_back(EncodeSop1(0x08, 6, 4));
capture_scc(8);
code.push_back(EncodeVopc(0xc2, InlineU32(0), 0)); // Only lane zero is set.
code.push_back(EncodeSop1(0x04, 10, 106));
set_scc();
code.push_back(EncodeSop2(0x0b, 12, 10, 2));
code.push_back(EncodeSop1(0x08, 14, 12));
capture_scc(16);
AppendStoreSgprAtLaneDwordOffset(&code, 8, 0, 0);
AppendStoreSgprAtLaneDwordOffset(&code, 16, 0, wave_size);
test.expected.assign(2u * wave_size, 1u);
code.push_back(EncodeSop1(0x04, 16, 126)); // Save full EXEC.
AppendVMovU32(&code, 2, 9);
constexpr std::array<uint32_t, 4> words{
0x5aa55aa5u, 0xc39ac39au, 0xa55aa55au, 0x3c653c65u};
for (u32 i = 0; i < words.size(); ++i) AppendSMovLiteral(&code, i, words[i]);
for (const u32 destination : {106u, 126u}) {
for (const u32 scc : {0u, 1u}) {
code.push_back(EncodeSopc(0x06, InlineU32(scc), InlineU32(1)));
code.push_back(EncodeSop2(0x0b, destination, 0, 2));
code.push_back(EncodeSop1(0x04, 8, destination)); // Preserve both raw words.
code.push_back(EncodeSop1(0x04, 126, destination));
AppendStoreVgprAtLaneDwordOffset(&code, 2, 0,
static_cast<u32>(test.expected.size()));
const u32 selected = scc ? 0u : 2u;
for (u32 lane = 0; lane < wave_size; ++lane)
test.expected.push_back((words[selected + lane / 32u] >> (lane & 31u)) & 1u
? 9u : 0xdeadbeefu);
code.push_back(EncodeSop1(0x04, 126, 16));
for (u32 component = 0; component < 2; ++component) {
AppendStoreSgprAtLaneDwordOffset(&code, 8 + component, 0,
static_cast<u32>(test.expected.size()));
test.expected.insert(test.expected.end(), wave_size, words[selected + component]);
}
}
}
AppendEnd(&code);
test.initial.assign(test.expected.size(), 0xdeadbeefu);
test.opcodes = {O::S_MOV_B32, O::S_CMP_EQ_U32, O::S_CSELECT_B64, O::S_NOT_B64,
O::S_CSELECT_B32, O::V_CMP_EQ_U32, O::S_MOV_B64, O::V_MOV_B32,
O::V_LSHLREV_B32, O::V_ADD_NC_U32, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.compute_info.threads_num[0] = wave_size;
test.compute_info.threads_num[1] = test.compute_info.threads_num[2] = 1;
test.compute_info.thread_ids_num = 1;
test.compute_info.wave_size = wave_size;
test.has_compute_info = true;
return test;
}
TestCase ScalarWqmB32Masks(u32 wave_size) {
using O = ShaderOpcode;
TestCase test;
test.name = wave_size == 64 ? "ScalarWqmB32MasksWave64" : "ScalarWqmB32MasksWave32";
auto &code = test.code;
const auto store_scalar = [&](u32 reg, u32 expected) {
AppendStoreSgprAtLaneDwordOffset(&code, reg, 0,
static_cast<u32>(test.expected.size()));
test.expected.insert(test.expected.end(), wave_size, expected);
};
const auto store_marker = [&](u32 lo, u32 hi) {
AppendStoreVgprAtLaneDwordOffset(&code, 2, 0,
static_cast<u32>(test.expected.size()));
for (u32 lane = 0; lane < wave_size; ++lane) {
const u32 mask = lane < 32 ? lo : hi;
test.expected.push_back(((mask >> (lane % 32)) & 1u) != 0 ? 9u : 0u);
}
};
constexpr u32 other_half = 0x2468ace0u;
constexpr u32 masks[] = {0, 1, 8, 0x10, 0x100, 0x80000000u, 0x12481248u, ~0u};
for (u32 mask : masks) {
u32 expanded = 0;
for (u32 bit = 0; bit < 32; bit += 4) {
if ((mask & (0xfu << bit)) != 0) expanded |= 0xfu << bit;
}
AppendVMovU32(&code, 2, 0);
AppendSMovLiteral(&code, 126, mask);
AppendSMovLiteral(&code, 127, other_half);
code.push_back(EncodeSopc(0x06, InlineU32(0), InlineU32(mask == 0 ? 0 : 1)));
code.push_back(0xbefe097eu); // Captured S_WQM_B32 EXEC_LO, EXEC_LO.
code.push_back(EncodeSMovB32(20, 126));
code.push_back(EncodeSMovB32(21, 127));
code.push_back(EncodeSMovB32(22, 253));
AppendVMovU32(&code, 2, 9);
code.push_back(EncodeSop1(0x04, 126, 193));
store_scalar(20, expanded);
store_scalar(21, other_half);
store_scalar(22, mask != 0 ? 1u : 0u);
store_marker(expanded, other_half);
}
// VALU-produced masks keep WQM dynamic and carry provenance through S_MOV_B64.
// Updating either word must invalidate an old SGPR-pair predicate and preserve
// the other word. Wave32 still reads/writes the scalar high half independently.
for (u32 base : {8u, 106u, 126u}) {
for (u32 half : {0u, 1u}) {
AppendVMovU32(&code, 2, 0);
code.push_back(EncodeVop2(0x1b, 1, InlineU32(31), 0));
code.push_back(EncodeVopc(0xc2, InlineU32(3), 1)); // Lanes 3 and 35.
code.push_back(EncodeSop1(0x04, base, 106));
code.push_back(EncodeSop1(0x09, base + half, base + half));
code.push_back(EncodeSMovB32(20, base));
code.push_back(EncodeSMovB32(21, base + 1));
code.push_back(EncodeSMovB32(22, 253));
code.push_back(EncodeSop1(0x04, 126, base));
AppendVMovU32(&code, 2, 9);
code.push_back(EncodeSop1(0x04, 126, 193));
const u32 lo = half == 0 ? 0xfu : 8u;
const u32 hi = wave_size == 32 ? 0u : half == 1 ? 0xfu : 8u;
store_scalar(20, lo);
store_scalar(21, hi);
store_scalar(22, wave_size == 32 && half == 1 ? 0u : 1u);
store_marker(lo, hi);
}
}
for (u32 base : {106u, 126u}) {
for (u32 half : {0u, 1u}) {
AppendVMovU32(&code, 2, 0);
code.push_back(EncodeSMovB32(107, InlineU32(0)));
code.push_back(EncodeVopc(0xc2, InlineU32(3), 0)); // Only lane 3.
code.push_back(EncodeSop1(0x04, base, 106));
code.push_back(EncodeSop1(0x09, base + half, base + (half ^ 1u)));
code.push_back(EncodeSMovB32(20, base));
code.push_back(EncodeSMovB32(21, base + 1));
code.push_back(EncodeSop1(0x04, 126, base));
AppendVMovU32(&code, 2, 9);
code.push_back(EncodeSop1(0x04, 126, 193));
const u32 lo = half == 0 ? 0u : 8u;
const u32 hi = half == 0 ? 0u : 0xfu;
store_scalar(20, lo);
store_scalar(21, hi);
store_marker(lo, hi);
}
}
AppendEnd(&code);
test.initial.resize(test.expected.size());
test.opcodes = {O::S_WQM_B32, O::S_MOV_B32, O::S_MOV_B64, O::S_CMP_EQ_U32,
O::V_CMP_EQ_U32, O::V_MOV_B32, O::V_AND_B32, O::V_ADD_NC_U32,
O::V_LSHLREV_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.decoded_counts = {{"S_WQM_B32 exec_lo, exec_lo", 9}};
test.compute_info.threads_num[0] = wave_size;
test.compute_info.threads_num[1] = test.compute_info.threads_num[2] = 1;
test.compute_info.thread_ids_num = 1;
test.compute_info.wave_size = wave_size;
test.has_compute_info = true;
return test;
}
TestCase ScalarWqmB64SelectsSccDomain() {
using O = ShaderOpcode;
std::vector<u32> code;
const auto capture_scc = [&](u32 dst) {
code.push_back(EncodeSop2(0x0a, dst, InlineU32(1), InlineU32(0)));
};
code.push_back(EncodeSMovB32(0, InlineU32(0)));
code.push_back(EncodeSMovB32(1, InlineU32(1)));
code.push_back(EncodeSop1(0x0a, 2, 0));
capture_scc(4);
code.push_back(EncodeVopc(0xc2, Vgpr(0), 0));
code.push_back(EncodeSop1(0x04, 6, 106));
code.push_back(EncodeSop1(0x08, 8, 6));
code.push_back(EncodeSop1(0x0a, 10, 8));
capture_scc(12);
AppendStoreSgpr(&code, 4, 0);
AppendStoreSgpr(&code, 12, 1);
AppendEnd(&code);
return {"ScalarWqmB64SelectsSccDomain",
code,
{},
{1, 1},
{O::S_MOV_B32, O::S_WQM_B64, O::S_CSELECT_B32, O::V_CMP_EQ_U32,
O::S_MOV_B64, O::S_NOT_B64, O::V_MOV_B32, O::BUFFER_STORE_DWORD,
O::S_ENDPGM}};
}
TestCase ScalarWqmB64PreservesPartialMasks() {
using O = ShaderOpcode;
TestCase test;
test.name = "ScalarWqmB64PreservesPartialMasks";
auto &code = test.code;
code.push_back(EncodeSop1(0x04, 16, 126)); // Save full EXEC.
AppendVMovU32(&code, 1, 3);
AppendVMovU32(&code, 2, 9);
u32 offset = 0;
for (const u32 destination : {126u, 106u, 8u}) {
code.push_back(EncodeVopc(0xc2, Vgpr(0), 1)); // Only lane 3.
code.push_back(EncodeSop1(0x04, destination, 106));
code.push_back(EncodeSop1(0x0a, destination, destination));
code.push_back(EncodeSop1(0x04, 126, destination));
AppendStoreVgprAtLaneDwordOffset(&code, 2, 0, offset);
code.push_back(EncodeSop1(0x04, 126, 16));
offset += 64;
}
code.push_back(EncodeSMovB32(8, InlineU32(32))); // Raw bits 5 and 33.
code.push_back(EncodeSMovB32(9, InlineU32(2)));
code.push_back(EncodeSop1(0x0a, 10, 8));
code.push_back(EncodeSop1(0x04, 126, 10));
AppendStoreVgprAtLaneDwordOffset(&code, 2, 0, offset);
code.push_back(EncodeSop1(0x04, 126, 16));
for (u32 source : {253u, 127u, 107u}) {
if (source == 253u) {
code.push_back(EncodeSopc(0x06, InlineU32(1), InlineU32(1))); // SCC=1.
} else {
AppendSMovLiteral(&code, source - 1, ~0u);
code.push_back(EncodeSMovB32(source, InlineU32(1)));
}
code.push_back(EncodeSop1(0x0a, 126, source));
offset += 64;
AppendStoreVgprAtLaneDwordOffset(&code, 2, 0, offset);
code.push_back(EncodeSop1(0x04, 126, 16));
}
AppendEnd(&code);
test.initial.assign(448, 0xdeadbeef);
test.expected = test.initial;
for (u32 lane = 0; lane < 64; ++lane) {
for (u32 region = 0; region < 3; ++region) {
if (lane < 4) test.expected[region * 64 + lane] = 9;
}
if ((lane >= 4 && lane < 8) || (lane >= 32 && lane < 36)) {
test.expected[192 + lane] = 9;
}
for (u32 region = 4; region < 7; ++region) {
if (lane < 4) test.expected[region * 64 + lane] = 9;
}
}
test.opcodes = {O::S_MOV_B64, O::S_MOV_B32, O::S_WQM_B64, O::V_MOV_B32,
O::V_CMP_EQ_U32, O::S_CMP_EQ_U32, O::V_LSHLREV_B32, O::BUFFER_STORE_DWORD,
O::S_ENDPGM};
test.compute_info.threads_num[0] = 64;
test.compute_info.threads_num[1] = test.compute_info.threads_num[2] = 1;
test.compute_info.thread_ids_num = 1;
test.has_compute_info = true;
return test;
}
TestCase ScalarMaskProvenanceOverlapAndMixedBinary() {
using O = ShaderOpcode;
std::vector<u32> code;
const auto capture_scc = [&](u32 dst) {
code.push_back(EncodeSop2(0x0a, dst, InlineU32(1), InlineU32(0)));
};
code.push_back(EncodeVopc(0xc2, Vgpr(0), 0));
code.push_back(EncodeSop1(0x04, 0, 106));
code.push_back(EncodeSop1(0x04, 1, 0));
code.push_back(EncodeSop1(0x08, 3, 1));
capture_scc(5);
code.push_back(EncodeVopc(0xc2, Vgpr(0), 0));
code.push_back(EncodeSop1(0x04, 8, 106));
code.push_back(EncodeSMovB32(10, InlineU32(1)));
code.push_back(EncodeSMovB32(11, InlineU32(0)));
code.push_back(EncodeSop2(0x19, 12, 8, 10));
capture_scc(14);
AppendStoreSgpr(&code, 5, 0);
AppendStoreSgpr(&code, 14, 1);
AppendEnd(&code);
return {"ScalarMaskProvenanceOverlapAndMixedBinary",
code,
{},
{1, 1},
{O::V_CMP_EQ_U32, O::S_MOV_B64, O::S_NOT_B64, O::S_CSELECT_B32,
O::S_MOV_B32, O::S_NAND_B64, O::V_MOV_B32, O::BUFFER_STORE_DWORD,
O::S_ENDPGM}};
}
TestCase ScalarB32MaskDataflow(u32 wave_size) {
using O = ShaderOpcode;
constexpr u32 mask = 0xffff0000u, rhs = 0x0f0f0f0fu;
constexpr u32 sentinel = 0xdeadbeefu, marker = 0x12345678u;
struct Operation {
u32 encoding;
O opcode;
u32 expected;
u32 source = rhs;
};
const std::array operations{
Operation{0x0e, O::S_AND_B32, mask & rhs},
Operation{0x10, O::S_OR_B32, mask | rhs},
Operation{0x12, O::S_XOR_B32, mask ^ rhs},
Operation{0x14, O::S_ANDN2_B32, mask & ~rhs},
Operation{0x16, O::S_ORN2_B32, mask | ~rhs},
Operation{0x18, O::S_NAND_B32, ~(mask & rhs)},
Operation{0x1a, O::S_NOR_B32, ~(mask | rhs)},
Operation{0x1c, O::S_XNOR_B32, ~(mask ^ rhs)},
Operation{0x07, O::S_NOT_B32, ~mask},
Operation{0x12, O::S_XOR_B32, 0, mask}};
TestCase test;
test.name = wave_size == 32 ? "ScalarB32MaskDataflowWave32" : "ScalarB32MaskDataflowWave64";
auto &code = test.code;
code.push_back(EncodeSop1(0x04, 16, 126)); // Preserve both EXEC halves.
AppendVMovU32(&code, 2, marker);
AppendVMovU32(&code, 1, 4);
for (const bool predicate_source : {false, true}) {
for (const auto &operation : operations) {
code.push_back(EncodeVopc(0xc1, InlineU32(15), 0)); // Lanes greater than 15.
code.push_back(EncodeSopc(0x06, InlineU32(1), InlineU32(1)));
code.push_back(EncodeSop2(0x0a, 107, 106, InlineU32(0))); // VCC_HI is a wave32 mask temporary.
code.push_back(EncodeSMovB32(8, 107));
if (wave_size == 32) {
code.push_back(EncodeSMovB32(127, 8));
code.push_back(EncodeSMovB32(8, 127)); // EXEC_HI must preserve its own scalar word.
code.push_back(EncodeSMovB32(127, InlineU32(0)));
}
AppendSMovLiteral(&code, 9, sentinel); // Adjacent write must preserve a wave32 predicate.
if (predicate_source) {
if (operation.source == mask) {
code.push_back(EncodeVopc(0xc1, InlineU32(15), 0));
} else {
code.push_back(EncodeVop2(0x1b, 3, InlineU32(7), 0));
code.push_back(EncodeVopc(0xc1, Vgpr(3), 1)); // (lane & 7) < 4.
}
code.push_back(EncodeSMovB32(10, 106));
} else {
code.push_back(EncodeSMovB32(10, operation.source == rhs ? 60u : 61u));
}
code.push_back(operation.opcode == O::S_NOT_B32
? EncodeSop1(operation.encoding, 8, 8)
: EncodeSop2(operation.encoding, 8, 8, 10));
code.push_back(EncodeSop2(0x0a, 12, InlineU32(1), InlineU32(0))); // Capture scalar SCC.
code.push_back(EncodeSop2(0x0a, 8, 8, 10)); // Aliased scalar select, including SCC0.
code.push_back(EncodeSMovB32(126, 8));
const auto offset = static_cast<u32>(test.expected.size());
AppendStoreVgprAtLaneDwordOffset(&code, 2, 0, offset);
code.push_back(EncodeSop1(0x04, 126, 16));
const auto selected = operation.expected != 0 ? operation.expected : operation.source;
for (u32 lane = 0; lane < wave_size; ++lane) {
test.expected.push_back(lane >= 32 || (selected & (1u << lane)) != 0
? marker : sentinel);
}
for (const auto [reg, value] : std::array<std::pair<u32, u32>, 3>{{
{8, selected}, {9, sentinel}, {12, operation.expected != 0}}}) {
AppendStoreSgprAtLaneDwordOffset(&code, reg, 0, static_cast<u32>(test.expected.size()));
test.expected.insert(test.expected.end(), wave_size, value);
}
test.opcodes.push_back(operation.opcode);
}
}
if (wave_size == 32) {
for (u32 low : {106u, 126u}) {
code.push_back(EncodeVopc(0xc1, InlineU32(15), 0));
code.push_back(EncodeSMovB32(low + 1, 106)); // A previously known high-word predicate.
code.push_back(EncodeSop1(0x04, low, 193u)); // Full scalar -1 overwrites both words.
code.push_back(EncodeSMovB32(8, low + 1));
code.push_back(EncodeSop1(0x04, 126, 16));
code.push_back(EncodeSMovB32(126, 8));
AppendStoreVgprAtLaneDwordOffset(&code, 2, 0, static_cast<u32>(test.expected.size()));
test.expected.insert(test.expected.end(), wave_size, marker);
code.push_back(EncodeSop1(0x04, 126, 16));
AppendStoreSgprAtLaneDwordOffset(&code, 8, 0, static_cast<u32>(test.expected.size()));
test.expected.insert(test.expected.end(), wave_size, UINT32_MAX);
}
}
AppendEnd(&code);
test.initial.assign(test.expected.size(), sentinel);
test.user_data = MakeNativeUserData(nullptr);
test.user_data[50] = static_cast<u32>(test.expected.size() * sizeof(u32));
test.user_data[60] = rhs; // Raw masks initialized at shader entry.
test.user_data[61] = mask;
test.has_user_data = true;
test.compute_info.threads_num[0] = wave_size;
test.compute_info.threads_num[1] = test.compute_info.threads_num[2] = 1;
test.compute_info.thread_ids_num = 1;
test.compute_info.wave_size = wave_size;
test.has_compute_info = true;
test.opcodes.insert(test.opcodes.end(), {O::S_MOV_B32, O::S_MOV_B64, O::S_CSELECT_B32, O::S_CMP_EQ_U32,
O::V_CMP_LT_U32, O::V_MOV_B32, O::V_AND_B32, O::V_LSHLREV_B32, O::V_ADD_NC_U32,
O::BUFFER_STORE_DWORD, O::S_ENDPGM});
return test;
}
TestCase ScalarLiteral() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendSMovLiteral(&code, 0, 0x12345678u);
code.push_back(EncodeVop1(0x01, 0, 0));
AppendBufferStoreDword(&code, 0, 30);
AppendEnd(&code);
return {"ScalarLiteral",
code,
{},
{0x12345678u},
{O::S_MOV_B32, O::V_MOV_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}
TestCase VectorMoves() {
using O = ShaderOpcode;
std::vector<u32> code = {
EncodeVop1(0x00, 0, 0),
EncodeVop1(0x01, 0, InlineU32(5)),
EncodeVop1(0x01, 2, Vgpr(0)),
};
AppendBufferStoreDword(&code, 2, 30);
AppendEnd(&code);
return {"VectorRegisterMoves",
code,
{},
{5},
{O::V_NOP, O::V_MOV_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}
TestCase VectorVop3MoveAppliesFloatSourceModifiers() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendSMovLiteral(&code, 24, 0x40000000u);
AppendVop3(&code, 0x181, 1, 24, 0, 0, 0, 0, false, 0, 0x1);
AppendStoreVgpr(&code, 1, 0);
AppendEnd(&code);
TestCase test;
test.name = "VectorVop3MoveAppliesFloatSourceModifiers";
test.code = code;
test.expected = {0xc0000000u};
test.opcodes = {O::S_MOV_B32, O::V_MOV_B32, O::BUFFER_STORE_DWORD,
O::S_ENDPGM};
test.required_spirv = {"OpFNegate"};
return test;
}
TestCase VectorIntegerOps() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovLiteral(&code, 0, 0xfffffff8u);
code.push_back(EncodeVop1(0x01, 1, InlineU32(5)));
code.push_back(EncodeVop1(0x01, 4, InlineU32(7)));
AppendVMovLiteral(&code, 10, 0x01000001u);
code.push_back(EncodeVop1(0x01, 11, InlineU32(2)));
AppendVMovLiteral(&code, 12, 0x0f0f0f0fu);
AppendVMovLiteral(&code, 13, 0x33333333u);
AppendVMovLiteral(&code, 14, 0xfffffff0u);
code.push_back(EncodeVop1(0x01, 15, InlineU32(4)));
code.push_back(EncodeVop1(0x01, 29, InlineU32(15)));
code.push_back(EncodeVop2(0x11, 2, Vgpr(0), 1));
code.push_back(EncodeVop2(0x12, 3, Vgpr(0), 1));
code.push_back(EncodeVop2(0x13, 5, Vgpr(0), 1));
code.push_back(EncodeVop2(0x14, 6, Vgpr(0), 1));
code.push_back(EncodeVop2(0x25, 7, Vgpr(4), 1));
code.push_back(EncodeVop2(0x26, 8, Vgpr(4), 1));
code.push_back(EncodeVop2(0x27, 9, InlineU32(3), 4));
code.push_back(EncodeVop2(0x26, 35, 249, 11));
code.push_back(EncodeVop2Sdwa(12, 6, 0, 0, 6));
code.push_back(EncodeVop2(0x27, 36, 249, 11));
code.push_back(EncodeVop2Sdwa(12, 6, 0, 0, 6));
code.push_back(EncodeVop2(0x0b, 16, Vgpr(10), 11));
code.push_back(EncodeVop2(0x09, 37, Vgpr(0), 1));
code.push_back(EncodeVop2(0x1b, 17, Vgpr(12), 13));
code.push_back(EncodeVop2(0x1c, 18, Vgpr(12), 13));
code.push_back(EncodeVop2(0x1d, 19, Vgpr(12), 13));
code.push_back(EncodeVop2(0x1e, 20, Vgpr(12), 13));
code.push_back(EncodeVop2(0x19, 21, Vgpr(11), 15));
code.push_back(EncodeVop2(0x1a, 22, InlineU32(4), 11));
code.push_back(EncodeVop2(0x15, 23, Vgpr(14), 15));
code.push_back(EncodeVop2(0x16, 24, InlineU32(4), 14));
code.push_back(EncodeVop2(0x17, 25, Vgpr(14), 15));
code.push_back(EncodeVop2(0x18, 26, InlineU32(4), 14));
code.push_back(EncodeVop1(0x37, 27, Vgpr(12)));
code.push_back(EncodeVop1(0x38, 28, Vgpr(29)));
code.push_back(EncodeVop1(0x3a, 30, Vgpr(11)));
code.push_back(EncodeVop1(0x39, 32, Vgpr(11)));
code.push_back(EncodeVopc(0xc2, Vgpr(4), 4));
code.push_back(EncodeVop2(0x01, 33, InlineU32(3), 4));
code.push_back(EncodeVop2(0x11, 34, 249, 4));
code.push_back(EncodeVop2Sdwa(0));
const u32 results[] = {2, 3, 5, 6, 7, 8, 9, 35, 36, 16, 37, 17, 18, 19,
20, 21, 22, 23, 24, 25, 26, 27, 28, 30, 32, 33, 34};
for (u32 i = 0; i < static_cast<u32>(std::size(results)); i++) {
AppendStoreVgpr(&code, results[i], i);
}
AppendEnd(&code);
return {"VectorIntegerOps",
code,
{},
{0xfffffff8u, 5, 5, 0xfffffff8u, 12,
2, 4, 13, 0xfffffff3u, 2,
0xffffffd8u, 0x03030303u, 0x3f3f3f3fu, 0x3c3c3c3cu, 0xc3c3c3c3u,
32, 32, 0x0fffffffu, 0x0fffffffu, 0xffffffffu,
0xffffffffu, 0xf0f0f0f0u, 0xf0000000u, 1, 30,
7, 0xfffffff8u},
{O::V_MOV_B32, O::V_MIN_I32, O::V_MAX_I32,
O::V_MIN_U32, O::V_MAX_U32, O::V_ADD_NC_U32,
O::V_SUB_NC_U32, O::V_SUBREV_NC_U32, O::V_MUL_U32_U24,
O::V_MUL_I32_I24, O::V_AND_B32, O::V_OR_B32,
O::V_XOR_B32, O::V_XNOR_B32, O::V_LSHL_B32,
O::V_LSHLREV_B32, O::V_LSHR_B32, O::V_LSHRREV_B32,
O::V_ASHR_I32, O::V_ASHRREV_I32, O::V_NOT_B32,
O::V_BFREV_B32, O::V_FFBL_B32, O::V_FFBH_U32,
O::V_CMP_EQ_U32, O::V_CNDMASK_B32, O::BUFFER_STORE_DWORD,
O::S_ENDPGM}};
}
TestCase VectorIntegerExtremaRuntimeEdges() {
using O = ShaderOpcode;
constexpr std::array<std::array<u32, 3>, 8> inputs{{
{0x80000000u, 0u, 0x7fffffffu},
{0xffffffffu, 1u, 0x80000000u},
{1u, 0x80000000u, 0xffffffffu},
{1u, 0xffffffffu, 0x80000000u},
{0x7fffffffu, 0x7fffffffu, 0x80000000u},
{0x80000000u, 0x80000000u, 0xffffffffu},
{0xffffffffu, 0xffffffffu, 0xffffffffu},
{0u, 0u, 0u},
}};
TestCase test;
test.name = "VectorIntegerExtremaRuntimeEdges";
for (const auto &values : inputs)
test.initial.insert(test.initial.end(), values.begin(), values.end());
test.expected = test.initial;
for (u32 i = 0; i < inputs.size(); ++i) {
for (u32 reg = 0; reg < 3; ++reg) {
AppendVMovU32(&test.code, 30, (i * 3u + reg) * 4u);
AppendBufferLoadDword(&test.code, reg, 30);
}
for (u32 op = 0; op < 4; ++op)
test.code.push_back(EncodeVop2(0x11u + op, 4u + op, Vgpr(0), 1));
constexpr std::array opcodes{0x152u, 0x155u, 0x158u, 0x153u, 0x156u, 0x159u};
for (u32 op = 0; op < opcodes.size(); ++op)
AppendVop3(&test.code, opcodes[op], 8u + op, Vgpr(0), Vgpr(1), Vgpr(2));
const auto output = static_cast<u32>(test.expected.size());
for (u32 result = 0; result < 10; ++result)
AppendStoreVgpr(&test.code, 4u + result, output + result);
auto unsigned_values = inputs[i];
std::array<int32_t, 3> signed_values;
for (u32 j = 0; j < 3; ++j)
signed_values[j] = std::bit_cast<int32_t>(unsigned_values[j]);
test.expected.insert(test.expected.end(), {
static_cast<u32>(std::min(signed_values[0], signed_values[1])),
static_cast<u32>(std::max(signed_values[0], signed_values[1])),
std::min(unsigned_values[0], unsigned_values[1]),
std::max(unsigned_values[0], unsigned_values[1])});
std::sort(signed_values.begin(), signed_values.end());
std::sort(unsigned_values.begin(), unsigned_values.end());
test.expected.insert(test.expected.end(), {
static_cast<u32>(signed_values[0]), static_cast<u32>(signed_values[2]),
static_cast<u32>(signed_values[1]), unsigned_values[0],
unsigned_values[2], unsigned_values[1]});
}
AppendEnd(&test.code);
test.opcodes = {O::V_MOV_B32, O::BUFFER_LOAD_DWORD, O::V_MIN_I32, O::V_MAX_I32,
O::V_MIN_U32, O::V_MAX_U32, O::V_MIN3_I32, O::V_MAX3_I32,
O::V_MED3_I32, O::V_MIN3_U32, O::V_MAX3_U32, O::V_MED3_U32,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.required_spirv = {" SMin ", " SMax ", " UMin ", " UMax "};
return test;
}
TestCase VectorFfbhI32NativeAndVop3OnGpu() {
using O = ShaderOpcode;
constexpr std::array inputs = {0x00000000u, 0x40000000u, 0x80000000u,
0x0fffffffu, 0xffff0000u, 0xfffffffeu,
0xffffffffu, 0x80000000u};
std::vector<u32> code;
for (u32 i = 0; i < inputs.size() - 1u; i++) {
AppendVMovU32(&code, 30, i * 4u);
AppendBufferLoadDword(&code, 14, 30);
code.push_back(0x7e1c770eu); // v_ffbh_i32 v14, v14
AppendStoreVgpr(&code, 14, i);
}
AppendVMovU32(&code, 30, (inputs.size() - 1u) * 4u);
AppendBufferLoadDword(&code, 14, 30);
AppendVop3(&code, 0x1bbu, 15, Vgpr(14), 0);
AppendStoreVgpr(&code, 15, inputs.size() - 1u);
AppendEnd(&code);
TestCase test;
test.name = "VectorFfbhI32NativeAndVop3OnGpu";
test.code = std::move(code);
test.initial.assign(inputs.begin(), inputs.end());
test.expected = {0xffffffffu, 1u, 1u, 4u, 16u, 31u, 0xffffffffu, 1u};
test.opcodes = {O::V_MOV_B32, O::BUFFER_LOAD_DWORD, O::V_FFBH_I32,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.decoded_counts = {{"V_FFBH_I32 v14, v14", inputs.size() - 1u},
{"V_FFBH_I32 v15, v14", 1u}};
return test;
}
TestCase Vop1SdwaBfrevSourceSelectors() {
using O = ShaderOpcode;
constexpr u32 expected[2][7] = {
{0x8f000000u, 0xd5000000u, 0x21000000u, 0x49000000u,
0x8fd50000u, 0x21490000u, 0x8fd52149u},
{0x8fffffffu, 0xd5ffffffu, 0x21ffffffu, 0x49ffffffu,
0x8fd5ffffu, 0x2149ffffu, 0x8fd52149u}};
TestCase test;
test.name = "Vop1SdwaBfrevSourceSelectors";
for (u32 sext = 0; sext < 2; sext++) {
for (u32 selector = 0; selector < 7; selector++) {
const u32 index = static_cast<u32>(test.initial.size());
AppendVMovU32(&test.code, 30, index * 4u);
AppendBufferLoadDword(&test.code, 0, 30);
// selector=4, sext=0 reproduces captured words 7e0070f9, 00040600.
test.code.insert(test.code.end(),
{0x7e0070f9u, EncodeVop1Sdwa(0, 6, 0, selector, sext)});
AppendStoreVgpr(&test.code, 0, index);
test.initial.push_back(0x9284abf1u);
test.expected.push_back(expected[sext][selector]);
}
}
AppendEnd(&test.code);
test.opcodes = {O::V_MOV_B32, O::BUFFER_LOAD_DWORD, O::V_BFREV_B32,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.decoded_counts = {{"V_BFREV_B32 v0, v0.sdwa(sel=4,sext=0)", 1}};
test.required_spirv = {"OpBitFieldUExtract", "OpBitFieldSExtract", "OpBitReverse"};
return test;
}
TestCase Vop1SdwaFfbhCapturedScalarLowWordSource() {
using O = ShaderOpcode;
constexpr std::array inputs = {0xdead0000u, 0xffff0001u, 0x00008000u,
0x00010008u, 0xffff4000u, 0x0000ffffu};
std::vector<u32> code;
for (u32 i = 0; i < inputs.size(); i++) {
AppendSmemLoadOpcode(&code, 0x08, 39, i * 4u);
code.insert(code.end(), {0x7e0072f9u, 0x00840627u}); // v_ffbh_u32 v0, s39.word0
AppendStoreVgpr(&code, 0, i);
}
AppendEnd(&code);
TestCase test;
test.name = "Vop1SdwaFfbhCapturedScalarLowWordSource";
test.code = std::move(code);
test.initial.assign(inputs.begin(), inputs.end());
test.expected = {0xffffffffu, 31u, 16u, 28u, 17u, 16u};
test.opcodes = {O::S_BUFFER_LOAD_DWORD, O::V_FFBH_U32, O::V_MOV_B32,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.decoded_counts = {{"V_FFBH_U32 v0, s39.sdwa(sel=4,sext=0)", inputs.size()}};
test.required_spirv = {"OpBitFieldUExtract", "FindUMsb"};
return test;
}
TestCase Vop1SdwaFfblCapturedHighWordSource() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendBufferLoadDword(&code, 2, 30);
code.push_back(0x7e2074f9u);
code.push_back(0x00050602u); // v_ffbl_b32 v16, v2.word1
AppendStoreVgpr(&code, 16, 0);
AppendEnd(&code);
TestCase test;
test.name = "Vop1SdwaFfblCapturedHighWordSource";
test.code = std::move(code);
test.initial = {0x00400000u};
test.expected = {6u};
test.opcodes = {O::BUFFER_LOAD_DWORD, O::V_FFBL_B32, O::BUFFER_STORE_DWORD,
O::S_ENDPGM};
test.decoded_counts = {{"V_FFBL_B32 v16, v2.sdwa(sel=5,sext=0)", 1}};
test.ir_counts = {{" = BitFieldUExtract ", 1}, {" = FindILsb32 ", 1}};
test.required_spirv = {"OpBitFieldUExtract"};
return test;
}
TestCase Vop1SdwaNotCapturedByte0Source() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendBufferLoadDword(&code, 2, 30);
code.push_back(0x7e046ef9u);
code.push_back(0x00000602u); // v_not_b32 v2, v2.byte0
AppendStoreVgpr(&code, 2, 0);
AppendEnd(&code);
TestCase test;
test.name = "Vop1SdwaNotCapturedByte0Source";
test.code = std::move(code);
test.initial = {0x12345678u};
test.expected = {0xffffff87u};
test.opcodes = {O::BUFFER_LOAD_DWORD, O::V_NOT_B32, O::BUFFER_STORE_DWORD,
O::S_ENDPGM};
test.decoded_counts = {{"V_NOT_B32 v2, v2.sdwa(sel=0,sext=0)", 1}};
test.ir_counts = {{" = BitFieldUExtract ", 1}, {" = BitwiseNot32 ", 1}};
test.required_spirv = {"OpBitFieldUExtract", "OpNot"};
return test;
}
TestCase Vop1SdwaNotPreservesHighWordDestination() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendBufferLoadDword(&code, 0, 30);
AppendVMovLiteral(&code, 3, 0xabcd5555u);
code.push_back(0x7e066ef9u);
code.push_back(0x00061400u);
AppendStoreVgpr(&code, 3, 0);
AppendEnd(&code);
TestCase test;
test.name = "Vop1SdwaNotPreservesHighWordDestination";
test.code = std::move(code);
test.initial = {0x12345678u};
test.expected = {0xabcda987u};
test.opcodes = {O::BUFFER_LOAD_DWORD, O::V_MOV_B32, O::V_NOT_B32,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.ir_counts = {{" = BitwiseNot32 ", 1}};
test.required_spirv = {"OpNot"};
return test;
}
TestCase Vop1SdwaNotPartialSourcesAndDestinations() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendBufferLoadDword(&code, 8, 30);
AppendVMovLiteral(&code, 28, 0xabcd5555u);
code.push_back(EncodeVop1(0x37, 28, 249));
code.push_back(EncodeVop1Sdwa(8, 4, 2, 5)); // v28.word0 = ~v8.word1, preserve word1
AppendStoreVgpr(&code, 28, 0);
AppendVMovLiteral(&code, 3, 0xa1b2c3d4u);
code.push_back(EncodeVop1(0x37, 3, 249));
code.push_back(EncodeVop1Sdwa(8, 2, 2, 0)); // v3.byte2 = ~v8.byte0, preserve others
AppendStoreVgpr(&code, 3, 1);
code.push_back(EncodeVop1(0x37, 4, 249));
code.push_back(EncodeVop1Sdwa(8, 0, 0, 0)); // v4.byte0 = ~v8.byte0, zero others
AppendStoreVgpr(&code, 4, 2);
AppendEnd(&code);
TestCase test;
test.name = "Vop1SdwaNotPartialSourcesAndDestinations";
test.code = std::move(code);
test.initial = {0x12345678u, 0u, 0u};
test.expected = {0xabcdedcbu, 0xa187c3d4u, 0x00000087u};
test.opcodes = {O::BUFFER_LOAD_DWORD, O::V_MOV_B32, O::V_NOT_B32,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.decoded_counts = {{"V_NOT_B32", 3u}};
test.required_spirv = {"OpBitFieldUExtract", "OpNot"};
return test;
}
TestCase Vop2SdwaBitwiseByteDestinations(u32 wave_size) {
using O = ShaderOpcode;
// Low result bytes of AND, OR, XOR, XNOR for the two runtime inputs.
constexpr std::array<u32, 4> result_bytes{0xa4u, 0xbeu, 0x1au, 0xe5u};
constexpr std::array<u32, 4> alias_expected{
0xc350e5a6u, 0xc3fce5a6u, 0xc3ace5a6u, 0xc353e5a6u};
constexpr u32 sentinel = 0xa1b2c3d4u;
TestCase test;
test.name = wave_size == 64 ? "Vop2SdwaBitwiseByteDestinationsWave64"
: "Vop2SdwaBitwiseByteDestinationsWave32";
test.initial = {0xc3d4e5a6u, 0x9b785abcu};
test.expected = test.initial;
auto &code = test.code;
const auto store = [&](u32 reg, u32 expected) {
AppendStoreVgpr(&code, reg, static_cast<u32>(test.expected.size()));
test.expected.push_back(expected);
};
for (u32 op = 0; op < 4; ++op) {
AppendVMovU32(&code, 30, 0);
AppendBufferLoadDword(&code, 12, 30);
AppendVMovU32(&code, 30, 4);
AppendBufferLoadDword(&code, 13, 30);
for (u32 selector = 0; selector < 4; ++selector) {
for (u32 unused = 0; unused < 3; ++unused) {
AppendVMovLiteral(&code, 26, sentinel);
code.push_back(EncodeVop2(0x1b + op, 26, 249, 13));
code.push_back(EncodeVop2Sdwa(12, selector, unused));
// RDNA2 table 88: pad, sign extend above/zero below, or preserve.
const u32 shift = selector * 8u;
u32 expected = result_bytes[op] << shift;
if (unused == 1 && (result_bytes[op] & 0x80u) != 0) {
expected = (0xffffff00u | result_bytes[op]) << shift;
} else if (unused == 2) {
expected |= sentinel & ~(0xffu << shift);
}
store(26, expected);
}
}
// Read source byte2 and word1 before overwriting source0's byte2.
code.push_back(EncodeVop2(0x1b + op, 12, 249, 13));
code.push_back(EncodeVop2Sdwa(12, 2, 2, 2, 5));
store(12, alias_expected[op]);
// Source1 may also alias the destination.
AppendVMovU32(&code, 30, 0);
AppendBufferLoadDword(&code, 12, 30);
code.push_back(EncodeVop2(0x1b + op, 13, 249, 13));
code.push_back(EncodeVop2Sdwa(12, 1, 2));
store(13, 0x9b7800bcu | (result_bytes[op] << 8u));
for (u32 unused = 0; unused < 3; ++unused) {
AppendVMovLiteral(&code, 26, sentinel);
code.push_back(EncodeSMovB32(126, InlineU32(0)));
code.push_back(EncodeVop2(0x1b + op, 26, 249, 13));
code.push_back(EncodeVop2Sdwa(12, 2, unused));
code.push_back(EncodeSMovB32(126, InlineU32(1)));
store(26, sentinel);
}
}
AppendEnd(&code);
test.initial.resize(test.expected.size());
test.opcodes = {O::V_MOV_B32, O::S_MOV_B32, O::BUFFER_LOAD_DWORD,
O::V_AND_B32, O::V_OR_B32, O::V_XOR_B32, O::V_XNOR_B32,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.required_spirv = {"OpBitwiseAnd", "OpBitwiseOr", "OpBitwiseXor", "OpNot",
"OpBitFieldInsert", "OpBitFieldSExtract"};
test.compute_info.wave_size = wave_size;
test.has_compute_info = true;
return test;
}
TestCase Vop2SdwaLshlrevByteDestinations(u32 wave_size) {
using O = ShaderOpcode;
constexpr u32 sentinel = 0xa1b2c3d4u;
TestCase test;
test.name = wave_size == 64 ? "Vop2SdwaLshlrevByteDestinations64" : "Vop2SdwaLshlrevByteDestinations32";
test.initial = {0x44668122u};
test.expected = test.initial;
auto &code = test.code;
const auto store = [&](u32 reg, u32 expected) {
AppendStoreVgpr(&code, reg, static_cast<u32>(test.expected.size()));
test.expected.push_back(expected);
};
AppendBufferLoadDword(&code, 4, 30);
for (const u32 count : {6u, 7u, 39u}) {
const u32 byte = count == 6u ? 0x40u : 0x80u;
for (u32 selector = 0; selector < 4u; ++selector) {
for (u32 unused = 0; unused < 3u; ++unused) {
AppendVMovLiteral(&code, 5, sentinel);
code.push_back(EncodeVop2(0x1a, 5, 249, 4));
// count7/byte1/preserve is captured 340a08f9 01861187 at PC4e98.
code.push_back(EncodeVop2Sdwa(InlineU32(count), selector, unused, 6, 1,
0, 0, 0, 0, 0, 0, 1));
const u32 shift = selector * 8u;
const u32 fill = unused == 1u && byte == 0x80u ? 0xffffff00u : 0u;
store(5, ((byte | fill) << shift) |
(unused == 2u ? sentinel & ~(0xffu << shift) : 0u));
}
}
}
code.insert(code.end(), {EncodeVop2(0x1a, 4, 249, 4), 0x01861187u});
store(4, 0x44668022u); // Source1 aliases the preserved destination.
AppendVMovLiteral(&code, 5, sentinel);
code.push_back(EncodeSMovB32(126, InlineU32(0)));
code.insert(code.end(), {0x340a08f9u, 0x01861187u});
code.push_back(EncodeSMovB32(126, InlineU32(1)));
store(5, sentinel);
AppendEnd(&code);
test.initial.resize(test.expected.size());
test.opcodes = {O::BUFFER_LOAD_DWORD, O::V_MOV_B32, O::S_MOV_B32,
O::V_LSHLREV_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.required_spirv = {"OpShiftLeftLogical", "OpBitFieldInsert", "OpBitFieldSExtract"};
test.compute_info.wave_size = wave_size;
test.has_compute_info = true;
return test;
}
TestCase Vop1SdwaMovByteDestinations() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendBufferLoadDword(&code, 3, 30);
AppendVMovU32(&code, 30, 4);
AppendBufferLoadDword(&code, 2, 30);
code.push_back(EncodeVop1(0x01, 1, Vgpr(3)));
code.push_back(0x7e0202f9u);
code.push_back(0x00861280u); // Captured v_mov_b32 v1.byte2, 0, preserve.
AppendStoreVgpr(&code, 1, 2);
// RDNA2 table 88: select any byte, then pad, sign extend, or preserve.
for (u32 dst_u = 0; dst_u < 3; dst_u++) {
for (u32 dst_sel = 0; dst_sel < 4; dst_sel++) {
code.push_back(EncodeVop1(0x01, 1, Vgpr(3)));
code.push_back(EncodeVop1(0x01, 1, 249));
code.push_back(EncodeVop1Sdwa(2, dst_sel, dst_u));
AppendStoreVgpr(&code, 1, 3 + dst_u * 4 + dst_sel);
}
}
AppendEnd(&code);
TestCase test;
test.name = "Vop1SdwaMovByteDestinations";
test.code = std::move(code);
test.initial = {0xa1b2c3d4u, 0x12345680u};
test.expected = {0xa1b2c3d4u, 0x12345680u, 0xa100c3d4u,
0x00000080u, 0x00008000u, 0x00800000u, 0x80000000u,
0xffffff80u, 0xffff8000u, 0xff800000u, 0x80000000u,
0xa1b2c380u, 0xa1b280d4u, 0xa180c3d4u, 0x80b2c3d4u};
test.initial.resize(test.expected.size());
test.opcodes = {O::BUFFER_LOAD_DWORD, O::V_MOV_B32, O::BUFFER_STORE_DWORD,
O::S_ENDPGM};
test.decoded_counts = {{"V_MOV_B32 v1.sdwa(sel=2,sext=0), 0", 1}};
test.required_spirv = {"OpBitFieldInsert", "OpBitFieldSExtract", "OpBitwiseOr"};
return test;
}
TestCase Vop1SdwaMovByteSources(u32 wave_size) {
using O = ShaderOpcode;
// Select 0x80 from each byte position, with nonzero neighboring bytes.
constexpr std::array<u32, 4> sources{
0x12345680u, 0x12348056u, 0x12803456u, 0x80341256u};
// RDNA2 table 88 / PS5 SDWA-related options: PAD, SEXT, PRESERVE.
constexpr std::array<u32, 18> partial_expected{
0x00000080u, 0xffffff80u, 0xa1b2c380u,
0x00008000u, 0xffff8000u, 0xa1b280d4u,
0x00800000u, 0xff800000u, 0xa180c3d4u,
0x80000000u, 0x80000000u, 0x80b2c3d4u,
0x00000080u, 0x00000080u, 0xa1b20080u,
0x00800000u, 0x00800000u, 0x0080c3d4u};
constexpr std::array<u32, 6> sext_word_expected{
0x0000ff80u, 0xffffff80u, 0xa1b2ff80u,
0xff800000u, 0xff800000u, 0xff80c3d4u};
TestCase test;
test.name = wave_size == 64 ? "Vop1SdwaMovByteSourcesWave64"
: "Vop1SdwaMovByteSourcesWave32";
test.initial.assign(sources.begin(), sources.end());
test.expected = test.initial;
auto &code = test.code;
const auto store = [&](u32 expected) {
AppendStoreVgpr(&code, 1, static_cast<u32>(test.expected.size()));
test.expected.push_back(expected);
};
for (u32 src_sel = 0; src_sel < sources.size(); ++src_sel) {
AppendVMovU32(&code, 30, src_sel * sizeof(u32));
AppendBufferLoadDword(&code, 2, 30);
for (u32 sext = 0; sext < 2; ++sext) {
for (u32 dst_sel = 0; dst_sel < 6; ++dst_sel) {
for (u32 unused = 0; unused < 3; ++unused) {
AppendVMovLiteral(&code, 1, 0xa1b2c3d4u);
code.push_back(EncodeVop1(0x01, 1, 249));
code.push_back(EncodeVop1Sdwa(2, dst_sel, unused, src_sel, sext));
store(sext && dst_sel >= 4 ? sext_word_expected[(dst_sel - 4u) * 3u + unused]
: partial_expected[dst_sel * 3u + unused]);
}
}
}
}
// Aliasing must read the source byte before replacing the destination byte.
AppendVMovU32(&code, 30, 0);
AppendBufferLoadDword(&code, 1, 30);
code.push_back(EncodeVop1(0x01, 1, 249));
code.push_back(EncodeVop1Sdwa(1, 3, 2, 1));
store(0x56345680u);
// Scalar and inline sources use the same independent selectors.
AppendSMovLiteral(&code, 18, 0x12348056u);
code.push_back(EncodeVop1(0x01, 1, 249));
code.push_back(EncodeVop1Sdwa(18, 5, 0, 1, 1, 0, 0, 1));
store(0xff800000u);
AppendVMovLiteral(&code, 1, 0xa1b2c3d4u);
code.push_back(EncodeVop1(0x01, 1, 249));
code.push_back(EncodeVop1Sdwa(193, 0, 2, 3, 0, 0, 0, 1)); // Inline -1.
store(0xa1b2c3ffu);
for (u32 unused = 0; unused < 3; ++unused) {
AppendVMovLiteral(&code, 1, 0xa1b2c3d4u);
code.push_back(EncodeSMovB32(126, InlineU32(0)));
code.push_back(EncodeVop1(0x01, 1, 249));
code.push_back(EncodeVop1Sdwa(2, 4, unused, 0));
code.push_back(EncodeSMovB32(126, InlineU32(1)));
store(0xa1b2c3d4u);
}
AppendEnd(&code);
test.initial.resize(test.expected.size());
test.opcodes = {O::V_MOV_B32, O::S_MOV_B32, O::BUFFER_LOAD_DWORD,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.required_spirv = {"OpBitFieldUExtract", "OpBitFieldSExtract", "OpBitFieldInsert"};
test.compute_info.wave_size = wave_size;
test.has_compute_info = true;
return test;
}
TestCase Vop2SdwaSubNcExactByte2Destination() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 11, 2);
code.push_back(0x4c1616f9u);
code.push_back(0x0686128du);
AppendStoreVgpr(&code, 11, 0);
AppendEnd(&code);
TestCase test;
test.name = "Vop2SdwaSubNcExactByte2Destination";
test.code = code;
test.expected = {0x000b0002u};
test.opcodes = {O::V_MOV_B32, O::V_SUB_NC_U32, O::BUFFER_STORE_DWORD,
O::S_ENDPGM};
return test;
}
TestCase Vop2SdwaAddNcCapturedHighWordDestination() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendBufferLoadDword(&code, 0, 30);
code.push_back(EncodeVop1(0x02, 106, Vgpr(0)));
AppendVMovLiteral(&code, 97, 0x00030004u);
AppendVMovLiteral(&code, 4, 0xa1b2c3d4u);
code.push_back(0x4a08c2f9u);
code.push_back(0x0686156au);
AppendStoreVgpr(&code, 4, 1);
AppendEnd(&code);
TestCase test;
test.name = "Vop2SdwaAddNcCapturedHighWordDestination";
test.code = std::move(code);
test.initial = {0x00010002u, 0};
test.expected = {0x00010002u, 0x0006c3d4u};
test.opcodes = {O::BUFFER_LOAD_DWORD, O::V_READFIRSTLANE_B32, O::V_MOV_B32,
O::V_ADD_NC_U32, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.required_spirv = {"OpIAdd", "OpShiftLeftLogical", "OpBitwiseOr"};
return test;
}
TestCase Vop2SdwaAshrrevCapturedWord0SignExtends() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendBufferLoadDword(&code, 1, 30);
code.push_back(0x303202f9u);
code.push_back(0x0c860686u); // v_ashrrev_i32 v25, 6, sign_extend(v1.word0)
AppendStoreVgpr(&code, 25, 1);
AppendEnd(&code);
TestCase test;
test.name = "Vop2SdwaAshrrevCapturedWord0SignExtends";
test.code = std::move(code);
test.initial = {0xabcd8000u, 0};
test.expected = {0xabcd8000u, 0xfffffe00u};
test.opcodes = {O::BUFFER_LOAD_DWORD, O::V_ASHRREV_I32, O::V_MOV_B32,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.decoded_counts = {{"V_ASHRREV_I32 v25, 6, v1.sdwa(sel=4,sext=1)", 1}};
test.ir_counts = {{" = BitFieldSExtract ", 1},
{" = ShiftRightArithmetic32 ", 1}};
test.required_spirv = {"OpBitFieldSExtract", "OpShiftRightArithmetic"};
return test;
}
TestCase Vop2SdwaMaxI32CapturedHighWord(u32 wave_size) {
using O = ShaderOpcode;
struct MaxCase { u32 lhs, packed_rhs, expected, exec = 1; };
constexpr std::array<MaxCase, 9> cases{{
{0xfffffffbu, 0xfffe1234u, 0xfffffffeu}, // -5 versus sign-extended -2.
{0xffffffffu, 0x80007fffu, 0xffffffffu},
{3, 0x0007ffffu, 7}, // Ignore the low word.
{0xffffffffu, 0x0000ffffu, 0},
{0x80000000u, 0x80000000u, 0xffff8000u},
{0x7fffffffu, 0x8000ffffu, 0x7fffffffu},
{0x0000ffffu, 0x7fffffffu, 0x0000ffffu}, // Source 0 stays 32-bit.
{0x00007fffu, 0x7fff8000u, 0x00007fffu},
{0x80000001u, 0x7fff0123u, 0x80000001u, 0}, // Inactive destination.
}};
TestCase test;
test.name = wave_size == 64 ? "Vop2SdwaMaxI32HighWordWave64"
: "Vop2SdwaMaxI32HighWordWave32";
for (const auto &entry : cases) {
test.initial.insert(test.initial.end(), {entry.lhs, entry.packed_rhs});
}
test.initial.resize(cases.size() * 3u, 0xdeadbeefu);
test.expected = test.initial;
auto &code = test.code;
for (u32 i = 0; i < cases.size(); ++i) {
AppendVMovU32(&code, 30, i * 8u);
AppendBufferLoadDword(&code, 11, 30);
AppendVMovU32(&code, 30, i * 8u + 4u);
AppendBufferLoadDword(&code, 22, 30);
code.push_back(EncodeSMovB32(126, InlineU32(cases[i].exec)));
// Exact game encoding: v11 = max(v11, sign_extend(v22.word1)).
code.insert(code.end(), {0x24162cf9u, 0x0d06060bu});
code.push_back(EncodeSMovB32(126, InlineU32(1)));
const u32 out = cases.size() * 2u + i;
AppendStoreVgpr(&code, 11, out);
test.expected[out] = cases[i].expected;
}
AppendEnd(&code);
test.opcodes = {O::V_MOV_B32, O::S_MOV_B32, O::BUFFER_LOAD_DWORD,
O::V_MAX_I32, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.decoded_counts = {{"V_MAX_I32 v11, v11, v22.sdwa(sel=5,sext=1)", cases.size()}};
test.required_spirv = {"OpBitFieldSExtract"};
test.compute_info.wave_size = wave_size;
test.has_compute_info = true;
return test;
}
TestCase Vop2SdwaMinI32CapturedByte0(u32 wave_size) {
using O = ShaderOpcode;
struct MinCase { u32 lhs, packed_rhs, expected, exec = 1; };
constexpr std::array<MinCase, 9> cases{{
{0xfffffffbu, 0x12345678u, 0xfffffffbu},
{0x00000064u, 0xffffff80u, 0x00000064u},
{0x000000c8u, 0xffffff80u, 0x00000080u},
{0x7fffffffu, 0x000000ffu, 0x000000ffu},
{0x80000000u, 0x7fffffffu, 0x80000000u},
{0x00000100u, 0x00000105u, 0x00000005u},
{0xffffffffu, 0x00000000u, 0xffffffffu},
{0x00000010u, 0xdeadbe0fu, 0x0000000fu},
{0x80000001u, 0x7fff0123u, 0x7fff0123u, 0},
}};
TestCase test;
test.name = wave_size == 64 ? "Vop2SdwaMinI32Byte0Wave64"
: "Vop2SdwaMinI32Byte0Wave32";
for (const auto &entry : cases) {
test.initial.insert(test.initial.end(), {entry.lhs, entry.packed_rhs});
}
test.initial.resize(cases.size() * 3u, 0xdeadbeefu);
test.expected = test.initial;
auto &code = test.code;
for (u32 i = 0; i < cases.size(); ++i) {
AppendVMovU32(&code, 30, i * 8u);
AppendBufferLoadDword(&code, 5, 30);
AppendVMovU32(&code, 30, i * 8u + 4u);
AppendBufferLoadDword(&code, 2, 30);
code.push_back(EncodeSMovB32(126, InlineU32(cases[i].exec)));
code.insert(code.end(), {0x220404f9u, 0x00060605u});
code.push_back(EncodeSMovB32(126, InlineU32(1)));
const u32 out = cases.size() * 2u + i;
AppendStoreVgpr(&code, 2, out);
test.expected[out] = cases[i].expected;
}
AppendEnd(&code);
test.opcodes = {O::V_MOV_B32, O::S_MOV_B32, O::BUFFER_LOAD_DWORD,
O::V_MIN_I32, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.decoded_counts = {{"V_MIN_I32 v2, v5, v2.sdwa(sel=0,sext=0)", cases.size()}};
test.required_spirv = {"OpBitFieldUExtract"};
test.compute_info.wave_size = wave_size;
test.has_compute_info = true;
return test;
}
TestCase Vop2SdwaMulI24Destinations(u32 wave_size) {
using O = ShaderOpcode;
constexpr std::array<u32, 5> sources{
0xdead1234u, 0xabcdffffu, 0x98768001u, 0x34560000u, 0x56789abcu};
constexpr std::array<u32, 10> captured_expected{
0xdead91a0u, 0xdead3400u, 0xabcdfff8u, 0xabcdff00u,
0x98760008u, 0x98760100u, 0x34560000u, 0x34560000u,
0x56789abcu, 0x56789abcu};
constexpr std::array<u32, 18> partial_expected{
0x00000080u, 0xffffff80u, 0xa1b2c380u,
0x00008000u, 0xffff8000u, 0xa1b280d4u,
0x00800000u, 0xff800000u, 0xa180c3d4u,
0x80000000u, 0x80000000u, 0x80b2c3d4u,
0x0000a180u, 0xffffa180u, 0xa1b2a180u,
0xa1800000u, 0xa1800000u, 0xa180c3d4u};
TestCase test;
test.name = wave_size == 64 ? "Vop2SdwaMulI24DestinationsWave64"
: "Vop2SdwaMulI24DestinationsWave32";
test.initial.assign(sources.begin(), sources.end());
test.initial.push_back(0xabcde080u);
test.expected = test.initial;
auto &code = test.code;
for (u32 index = 0; index < sources.size(); ++index) {
AppendVMovU32(&code, 30, index * sizeof(u32));
AppendBufferLoadDword(&code, 12, 30);
AppendBufferLoadDword(&code, 26, 30);
AppendSMovLiteral(&code, 106, 256);
code.push_back(EncodeSMovB32(126, InlineU32(index + 1u == sources.size() ? 0 : 1)));
// Captured WORD0/PRESERVE encodings alias their source and destination.
code.insert(code.end(), {0x121818f9u, 0x04861488u,
0x123434f9u, 0x0486146au});
code.push_back(EncodeSMovB32(126, InlineU32(1)));
AppendStoreVgpr(&code, 12, static_cast<u32>(test.expected.size()));
AppendStoreVgpr(&code, 26, static_cast<u32>(test.expected.size()) + 1u);
test.expected.push_back(captured_expected[index * 2u]);
test.expected.push_back(captured_expected[index * 2u + 1u]);
}
AppendVMovU32(&code, 30, sources.size() * sizeof(u32));
AppendBufferLoadDword(&code, 12, 30);
for (u32 selector = 0; selector < 6; ++selector) {
for (u32 unused = 0; unused < 3; ++unused) {
AppendVMovLiteral(&code, 26, 0xa1b2c3d4u);
code.push_back(EncodeVop2(0x09, 26, 249, 12));
code.push_back(EncodeVop2Sdwa(InlineU32(3), selector, unused, 6, 4,
0, 0, 0, 0, 0, 0, 1));
AppendStoreVgpr(&code, 26, static_cast<u32>(test.expected.size()));
test.expected.push_back(partial_expected[selector * 3u + unused]);
}
}
AppendEnd(&code);
test.initial.resize(test.expected.size());
test.opcodes = {O::V_MOV_B32, O::S_MOV_B32, O::BUFFER_LOAD_DWORD,
O::V_MUL_I32_I24, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.required_spirv = {"OpIMul", "OpBitFieldUExtract"};
test.compute_info.wave_size = wave_size;
test.has_compute_info = true;
return test;
}
TestCase Vop2SdwaMulU24Destinations(u32 wave_size) {
using O = ShaderOpcode;
// The high source bytes are discarded: 3 * 0xe080 = 0x2a180.
constexpr std::array<u32, 18> partial_expected{
0x00000080u, 0xffffff80u, 0xa1b2c380u,
0x00008000u, 0xffff8000u, 0xa1b280d4u,
0x00800000u, 0xff800000u, 0xa180c3d4u,
0x80000000u, 0x80000000u, 0x80b2c3d4u,
0x0000a180u, 0xffffa180u, 0xa1b2a180u,
0xa1800000u, 0xa1800000u, 0xa180c3d4u};
TestCase test;
test.name = wave_size == 64 ? "Vop2SdwaMulU24DestinationsWave64"
: "Vop2SdwaMulU24DestinationsWave32";
test.initial = {0xab000003u, 0xcd00e080u, 0xffffffffu};
test.expected = test.initial;
auto &code = test.code;
const auto store = [&](u32 reg, u32 expected) {
AppendStoreVgpr(&code, reg, static_cast<u32>(test.expected.size()));
test.expected.push_back(expected);
};
for (u32 i = 0; i < test.initial.size(); ++i) {
AppendVMovU32(&code, 30, i * sizeof(u32));
AppendBufferLoadDword(&code, 12u + i, 30);
}
for (u32 selector = 0; selector < 6; ++selector) {
for (u32 unused = 0; unused < 3; ++unused) {
AppendVMovLiteral(&code, 26, 0xa1b2c3d4u);
code.push_back(EncodeVop2(0x0b, 26, 249, 13));
code.push_back(EncodeVop2Sdwa(12, selector, unused));
store(26, partial_expected[selector * 3u + unused]);
}
}
// Full result still truncates the 48-bit unsigned product to 32 bits.
code.push_back(EncodeVop2(0x0b, 26, 249, 14));
code.push_back(EncodeVop2Sdwa(14));
store(26, 0xfe000001u);
// Either multiplicand may alias the partial destination.
code.push_back(EncodeVop2(0x0b, 12, 249, 13));
code.push_back(EncodeVop2Sdwa(12, 5, 2));
store(12, 0xa1800003u);
code.push_back(EncodeVop2(0x0b, 13, 249, 13));
code.push_back(EncodeVop2Sdwa(InlineU32(3), 2, 2, 6, 4,
0, 0, 0, 0, 0, 0, 1));
store(13, 0xcd80e080u);
// Source-byte sign extension happens before the unsigned 24-bit multiply.
AppendSMovLiteral(&code, 18, 0x12348056u);
code.push_back(EncodeVop2(0x0b, 26, 249, InlineU32(3)));
code.push_back(EncodeVop2Sdwa(18, 5, 0, 1, 6, 1, 0, 0, 0, 0, 0, 1, 1));
store(26, 0xfe800000u);
for (u32 unused = 0; unused < 3; ++unused) {
AppendVMovLiteral(&code, 26, 0xa1b2c3d4u);
code.push_back(EncodeSMovB32(126, InlineU32(0)));
code.push_back(EncodeVop2(0x0b, 26, 249, 13));
code.push_back(EncodeVop2Sdwa(12, 4, unused));
code.push_back(EncodeSMovB32(126, InlineU32(1)));
store(26, 0xa1b2c3d4u);
}
AppendEnd(&code);
test.initial.resize(test.expected.size());
test.opcodes = {O::V_MOV_B32, O::S_MOV_B32, O::BUFFER_LOAD_DWORD,
O::V_MUL_U32_U24, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.decoded_counts = {{"V_MUL_U32_U24", 25u}};
test.required_spirv = {"OpIMul", "OpBitFieldUExtract", "OpBitFieldSExtract"};
test.compute_info.wave_size = wave_size;
test.has_compute_info = true;
return test;
}
TestCase Vop2SdwaLshrrevCapturedByte1Source() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendSMovLiteral(&code, 106, 4);
AppendBufferLoadDword(&code, 8, 30);
code.push_back(0x2c1210f9u);
code.push_back(0x0186066au); // v_lshrrev_b32 v9, vcc_lo, v8.byte1
AppendStoreVgpr(&code, 9, 1);
AppendEnd(&code);
TestCase test;
test.name = "Vop2SdwaLshrrevCapturedByte1Source";
test.code = std::move(code);
test.initial = {0x12348678u, 0};
test.expected = {0x12348678u, 8};
test.opcodes = {O::S_MOV_B32, O::BUFFER_LOAD_DWORD, O::V_LSHRREV_B32,
O::V_MOV_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.decoded_counts = {
{"V_LSHRREV_B32 v9, vcc_lo, v8.sdwa(sel=1,sext=0)", 1}};
test.ir_counts = {{" = BitFieldUExtract ", 1},
{" = ShiftRightLogical32 ", 1}};
test.required_spirv = {"OpBitFieldUExtract", "OpShiftRightLogical"};
return test;
}
TestCase Vop2SdwaSubNcPreservesByteAndWordDestinations() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 0, 3);
for (u32 dst = 10; dst <= 15; dst++) {
AppendVMovLiteral(&code, dst, 0xa1b2c3d4u);
}
for (u32 sel = 0; sel <= 5; sel++) {
code.push_back(EncodeVop2(0x26, 10 + sel, 249, 0));
code.push_back(
EncodeVop2Sdwa(InlineU32(20), sel, 2, 6, 6, 0, 0, 0, 0, 0, 0, 1));
AppendStoreVgpr(&code, 10 + sel, sel);
}
AppendEnd(&code);
return {"Vop2SdwaSubNcPreservesByteAndWordDestinations",
code,
{},
{0xa1b2c311u, 0xa1b211d4u, 0xa111c3d4u, 0x11b2c3d4u, 0xa1b20011u,
0x0011c3d4u},
{O::V_MOV_B32, O::V_SUB_NC_U32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}
TestCase Vop3CvtPkI16I32Captured() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendBufferLoadDword(&code, 7, 30);
AppendVMovU32(&code, 30, 4);
AppendBufferLoadDword(&code, 16, 30);
code.push_back(0xd76b0005u);
code.push_back(0x00022107u); // v_cvt_pk_i16_i32 v5, v7, v16
AppendStoreVgpr(&code, 5, 2);
AppendEnd(&code);
TestCase test;
test.name = "Vop3CvtPkI16I32Captured";
test.code = std::move(code);
test.initial = {0x00018001u, 0xfffe7ffeu, 0};
test.expected = {0x00018001u, 0xfffe7ffeu, 0x7ffe8001u};
test.opcodes = {O::BUFFER_LOAD_DWORD, O::V_MOV_B32, O::V_CVT_PK_I16_I32,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.decoded_counts = {{"0x00000014: V_CVT_PK_I16_I32 v5, v7, v16\n", 1}};
test.ir_counts = {{" = BitwiseAnd32 ", 2},
{" = ShiftLeftLogical32 ", 1},
{" = BitwiseOr32 ", 1}};
test.required_spirv = {"OpBitwiseAnd", "OpShiftLeftLogical", "OpBitwiseOr"};
return test;
}
TestCase Vop3MulLoU16CapturedAndSelectors() {
using O = ShaderOpcode;
TestCase test;
test.name = "Vop3MulLoU16CapturedAndSelectors";
// Distinct runtime halves expose source selection and prevent constant folding.
test.initial = {0x00138001u, 0x0005ffffu, 0xffffffffu};
test.expected = {0x00138001u, 0x0005ffffu, 0xffffffffu,
0xa5a57fffu, 0xa5a5ffedu, 0xa5a58005u, 0xa5a5005fu,
0x7fff1234u, 0xffed1234u, 0x80051234u, 0x005f1234u,
0xffff0001u, 0x00231234u};
auto &code = test.code;
AppendBufferLoadDword(&code, 12, 30);
AppendVMovU32(&code, 30, 4);
AppendBufferLoadDword(&code, 24, 30);
u32 output = 3;
for (u32 selectors : {0u, 1u, 2u, 3u, 8u, 9u, 10u, 11u}) {
AppendVMovLiteral(&code, 2, 0xa5a51234u);
if (selectors == 8u) {
code.push_back(0xd7054002u);
code.push_back(0x0002310cu); // Captured v_mul_lo_u16 v2.hi, v12.lo, v24.lo.
} else {
AppendVop3(&code, 0x305, 2, Vgpr(12), Vgpr(24), 0, 0, selectors);
}
AppendStoreVgpr(&code, 2, output++);
}
// 0xffff * 0xffff wraps to 1; aliasing preserves the original high half.
AppendVMovU32(&code, 30, 8);
AppendBufferLoadDword(&code, 12, 30);
AppendVop3(&code, 0x305, 12, Vgpr(12), Vgpr(12), 0, 0, 1);
AppendStoreVgpr(&code, 12, output++);
AppendVMovLiteral(&code, 2, 0xa5a51234u);
AppendVop3(&code, 0x305, 2, 255, Vgpr(24), 0, 0, 11);
code.push_back(0x0007deadu); // Select literal high half: 7 * 5 = 35.
AppendStoreVgpr(&code, 2, output);
AppendEnd(&code);
test.opcodes = {O::V_MOV_B32, O::BUFFER_LOAD_DWORD, O::V_MUL_LO_U16,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.decoded_counts = {{"V_MUL_LO_U16", 10}};
test.required_spirv = {"OpIMul"};
return test;
}
TestCase Vop3F32ClampFlushesNaN() {
using O = ShaderOpcode;
struct ClampCase {
u32 opcode, input, expected;
u32 abs = 0;
};
constexpr u32 inf = 0x7f800000u, one = 0x3f800000u;
// GoW Ragnarok PS 3d52c7172f95fd37 0x2564: v_mul_f32 v18 clamp, inf, |v2|; and
// fma(x, inf, inf) clamp. The clamp modifier (DX10 clamp mode) turns their NaN into 0.
const std::array<ClampCase, 10> cases{{
{0x108, 0x00000000u, 0, 2}, // inf * |0| = NaN.
{0x108, 0x80000000u, 0, 2}, // inf * |-0| = NaN.
{0x108, 0x3f000000u, one, 2},
{0x108, 0xc0400000u, one, 2},
{0x14b, 0xbf800000u, 0}, // -1 * inf + inf = NaN.
{0x14b, 0x40000000u, one},
{0x103, 0x7fc00000u, 0}, // Quiet NaN input.
{0x103, 0xffc00001u, 0}, // Negative NaN input.
{0x103, 0x3e800000u, 0x3e800000u},
{0x103, 0x3fc00000u, one},
}};
TestCase test;
test.name = "Vop3F32ClampFlushesNaN";
for (const auto &entry : cases) test.initial.push_back(entry.input);
test.expected = test.initial;
auto &code = test.code;
AppendVMovLiteral(&code, 20, inf);
AppendVMovU32(&code, 21, 0);
for (u32 i = 0; i < cases.size(); ++i) {
const auto &entry = cases[i];
AppendVMovU32(&code, 30, i * 4u);
AppendBufferLoadDword(&code, 2, 30); // Runtime inputs prevent folding.
AppendVMovLiteral(&code, 18, 0xa5a5beefu);
if (entry.opcode == 0x108) {
AppendVop3(&code, 0x108, 18, Vgpr(20), Vgpr(2), 0, entry.abs, 0, true);
} else if (entry.opcode == 0x14b) {
AppendVop3(&code, 0x14b, 18, Vgpr(2), Vgpr(20), Vgpr(20), 0, 0, true);
} else {
AppendVop3(&code, 0x103, 18, Vgpr(2), Vgpr(21), 0, 0, 0, true);
}
AppendStoreVgpr(&code, 18, static_cast<u32>(cases.size()) + i);
test.expected.push_back(entry.expected);
}
AppendEnd(&code);
test.opcodes = {O::V_MOV_B32, O::BUFFER_LOAD_DWORD, O::V_MUL_F32, O::V_FMA_F32,
O::V_ADD_F32, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.decoded_counts = {{"V_MUL_F32", 4}, {"V_FMA_F32", 2}, {"V_ADD_F32", 4}};
return test;
}
TestCase Vop3MadI16CapturedSelectorsAndSaturation() {
using O = ShaderOpcode;
struct MadCase {
u32 lhs, rhs, addend, expected;
u32 selectors = 0;
bool clamp = false;
u32 dst = 1;
};
constexpr u32 a = 0x8000fffdu, b = 0xfffe0005u, c = 0xfff90007u;
const std::array<MadCase, 14> cases{{
{a, b, c, 0xa5a5fff8u}, // Captured: -3 * 5 + 7 = -8.
{a, b, c, 0x8007beefu, 9}, // Source 0 high; destination high.
{a, b, c, 0xa5a5000du, 2}, // Source 1 high: -3 * -2 + 7.
{a, b, c, 0xa5a5ffeau, 4}, // Source 2 high: -3 * 5 - 7.
{a, b, c, 0xfff9beefu, 15}, // All high: -32768 * -2 - 7 wraps.
{a, b, c, 0xfff8fffdu, 8, false, 9}, // Destination aliases source 0.
{0x7fffu, 1, 1, 0xa5a58000u},
{0x7fffu, 1, 1, 0x7fffbeefu, 8, true}, // Positive saturation.
{0x8000u, 1, 0xffffu, 0xa5a57fffu},
{0x8000u, 1, 0xffffu, 0x8000beefu, 8, true}, // Negative saturation.
{0x8000u, 0x8000u, 0xffffu, 0xa5a5ffffu},
{0x8000u, 0x8000u, 0xffffu, 0xa5a57fffu, 0, true},
{200, 200, 0xd8f0u, 0xa5a57530u, 0, true}, // 40000 - 10000 = 30000.
{200, 0xff38u, 10000, 0xa5a58ad0u, 0, true}, // -40000 + 10000 = -30000.
}};
TestCase test;
test.name = "Vop3MadI16CapturedSelectorsAndSaturation";
for (const auto &entry : cases) {
test.initial.insert(test.initial.end(), {entry.lhs, entry.rhs, entry.addend});
}
test.expected = test.initial;
auto &code = test.code;
for (u32 i = 0; i < cases.size(); ++i) {
const auto &entry = cases[i];
u32 offset = i * 12u;
for (u32 reg : {9u, 6u, 15u}) {
AppendVMovU32(&code, 30, offset);
AppendBufferLoadDword(&code, reg, 30); // Runtime inputs prevent folding.
offset += 4u;
}
AppendVMovLiteral(&code, 1, 0xa5a5beefu);
if (i == 0u) {
code.insert(code.end(), {0xd75e0001u, 0x043e0d09u});
} else {
AppendVop3(&code, 0x35e, entry.dst, Vgpr(9), Vgpr(6), Vgpr(15), 0,
entry.selectors, entry.clamp);
}
AppendStoreVgpr(&code, entry.dst, static_cast<u32>(test.initial.size()) + i);
test.expected.push_back(entry.expected);
}
AppendEnd(&code);
test.opcodes = {O::V_MOV_B32, O::BUFFER_LOAD_DWORD, O::V_MAD_I16,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.decoded_counts = {{"V_MAD_I16", cases.size()}};
test.required_spirv = {"OpIMul", "OpIAdd"};
return test;
}
TestCase Vop3Min3U16CapturedAndSelectors() {
using O = ShaderOpcode;
struct MinCase {
u32 lhs, rhs, third, expected;
u32 selectors = 0;
u32 dst = 17;
bool active = true;
};
constexpr u32 a = 0x0009ea60u, b = 0x0007c350u, c = 0x00059c40u;
const std::array<MinCase, 10> cases{{
{a, b, c, 0x0005beefu, 14}, // Captured: min(60000, 7, 5) into v17.hi.
{a, b, c, 0xa5a59c40u}, // Unsigned values above INT16_MAX.
{a, b, c, 0xa5a50009u, 1},
{a, b, c, 0xa5a50007u, 2},
{a, b, c, 0xa5a50005u, 4},
{a, b, c, 0x0005beefu, 15},
{a, b, c, 0x9c40ea60u, 8, 4}, // Destination aliases source 0.
{0xabcdffffu, 0xffff0000u, 0x98768000u, 0xa5a50000u},
{0x0000ffffu, 0x7fffffffu, 0x8000ffffu, 0xa5a5ffffu},
{a, b, c, 0xa5a5beefu, 14, 17, false},
}};
TestCase test;
test.name = "Vop3Min3U16CapturedAndSelectors";
for (const auto &entry : cases) {
test.initial.insert(test.initial.end(), {entry.lhs, entry.rhs, entry.third});
}
test.expected = test.initial;
auto &code = test.code;
for (u32 i = 0; i < cases.size(); ++i) {
const auto &entry = cases[i];
u32 offset = i * 12u;
for (u32 reg : {4u, 0u, 7u}) {
AppendVMovU32(&code, 30, offset);
AppendBufferLoadDword(&code, reg, 30); // Runtime inputs prevent folding.
offset += 4u;
}
AppendVMovLiteral(&code, 17, 0xa5a5beefu);
if (!entry.active) {
code.push_back(EncodeSop1(0x04, 24, 126)); // Preserve the initial active lanes.
code.push_back(EncodeSop1(0x04, 126, 128));
}
if (i == 0u) {
code.insert(code.end(), {0xd7537011u, 0x041e0104u});
} else {
AppendVop3(&code, 0x353, entry.dst, Vgpr(4), Vgpr(0), Vgpr(7), 0,
entry.selectors);
}
if (!entry.active) code.push_back(EncodeSop1(0x04, 126, 24));
AppendStoreVgpr(&code, entry.dst, static_cast<u32>(test.initial.size()) + i);
test.expected.push_back(entry.expected);
}
AppendEnd(&code);
test.opcodes = {O::V_MOV_B32, O::BUFFER_LOAD_DWORD, O::V_MIN3_U16,
O::S_MOV_B64, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.decoded_counts = {{"V_MIN3_U16", cases.size()}};
test.required_spirv = {" UMin "};
return test;
}
TestCase Vop3Med3I16Captured() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovLiteral(&code, 5, 0x8ad003e8u); // high=-30000, low=1000
AppendSMovLiteral(&code, 25, 0xb1e04e20u); // high=-20000, low=20000
AppendSMovLiteral(&code, 27, 0x7530fc18u); // high=30000, low=-1000
AppendVMovLiteral(&code, 3, 0x1234cafeu);
code.push_back(0xd7584803u);
code.push_back(0x006c3305u); // v_med3_i16 v3.hi, v5.hi, s25.lo, s27.lo
AppendStoreVgpr(&code, 3, 0);
AppendEnd(&code);
TestCase test;
test.name = "Vop3Med3I16Captured";
test.code = std::move(code);
test.expected = {0xfc18cafeu};
test.opcodes = {O::V_MOV_B32, O::S_MOV_B32, O::V_MED3_I16,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.decoded_counts = {{"0x00000020: V_MED3_I16 v3.sdwa(sel=5,sext=0), "
"v5.opsel(lo=1,hi=0,neghi=0), s25, s27\n",
1}};
test.required_spirv = {" SMin ", " SMax "};
return test;
}
TestCase Vop2SdwaMinU32PreservesWordDestination() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovLiteral(&code, 10, 0xa1b2c3d4u);
AppendVMovU32(&code, 12, 7);
code.push_back(0x261418f9u);
code.push_back(0x0686149fu);
AppendStoreVgpr(&code, 10, 0);
AppendVMovLiteral(&code, 10, 0xa1b2c3d4u);
AppendVMovU32(&code, 12, 64);
code.push_back(0x261418f9u);
code.push_back(0x0686149fu);
AppendStoreVgpr(&code, 10, 1);
AppendEnd(&code);
TestCase test;
test.name = "Vop2SdwaMinU32PreservesWordDestination";
test.code = code;
test.expected = {0xa1b20007u, 0xa1b2001fu};
test.opcodes = {O::V_MOV_B32, O::V_MIN_U32, O::BUFFER_STORE_DWORD,
O::S_ENDPGM};
test.required_spirv = {" UMin ", "OpSelect"};
return test;
}
TestCase VectorShiftCountsMaskLowBits() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 0, 1);
AppendVMovU32(&code, 1, 32);
AppendVMovU32(&code, 2, 33);
AppendVMovLiteral(&code, 3, 0x80000000u);
code.push_back(EncodeVop2(0x19, 10, Vgpr(0), 1));
code.push_back(EncodeVop2(0x1a, 11, Vgpr(2), 0));
code.push_back(EncodeVop2(0x15, 12, Vgpr(3), 1));
code.push_back(EncodeVop2(0x16, 13, Vgpr(2), 3));
code.push_back(EncodeVop2(0x17, 14, Vgpr(3), 1));
code.push_back(EncodeVop2(0x18, 15, Vgpr(2), 3));
for (u32 i = 0; i < 6u; i++) {
AppendStoreVgpr(&code, 10u + i, i);
}
AppendEnd(&code);
return {"VectorShiftCountsMaskLowBits",
code,
{},
{1, 2, 0x80000000u, 0x40000000u, 0x80000000u, 0xc0000000u},
{O::V_MOV_B32, O::V_LSHL_B32, O::V_LSHLREV_B32, O::V_LSHR_B32,
O::V_LSHRREV_B32, O::V_ASHR_I32, O::V_ASHRREV_I32,
O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}
TestCase VectorVop3LshrrevB64Captured() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendSMovLiteral(&code, 106, 0x89abcdefu);
AppendSMovLiteral(&code, 107, 0x01234567u);
AppendVMovU32(&code, 12, 68);
code.push_back(0xd7000001u);
code.push_back(0x0000d50cu); // v_lshrrev_b64 v[1:2], v12, vcc
AppendStoreVgpr(&code, 1, 0);
AppendStoreVgpr(&code, 2, 1);
AppendEnd(&code);
TestCase test;
test.name = "VectorVop3LshrrevB64Captured";
test.code = std::move(code);
test.expected = {0x789abcdeu, 0x00123456u};
test.opcodes = {O::S_MOV_B32, O::V_MOV_B32, O::V_LSHRREV_B64,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.required_spirv = {"OpShiftRightLogical"};
return test;
}
TestCase VectorVop3LshlrevB64Captured() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendBufferLoadDword(&code, 3, 30);
code.push_back(0xd6ff0021u);
code.push_back(0x00010303u); // v_lshlrev_b64 v[33:34], v3, 1
AppendStoreVgpr(&code, 33, 1);
AppendStoreVgpr(&code, 34, 2);
code.push_back(EncodeVop2(0x03, 3, InlineU32(31), 3));
code.push_back(0xd6ff0021u);
code.push_back(0x00010303u);
AppendStoreVgpr(&code, 33, 3);
AppendStoreVgpr(&code, 34, 4);
AppendEnd(&code);
TestCase test;
test.name = "VectorVop3LshlrevB64Captured";
test.code = std::move(code);
test.initial = {33u, 0u, 0u, 0u, 0u};
test.expected = {33u, 0u, 2u, 1u, 0u};
test.opcodes = {O::BUFFER_LOAD_DWORD, O::V_LSHLREV_B64, O::V_ADD_NC_U32,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.required_spirv = {"OpShiftLeftLogical", "OpTypeInt 64 0"};
return test;
}
TestCase VectorDynamicU64ShiftEdges() {
using O = ShaderOpcode;
TestCase test;
test.name = "VectorDynamicU64ShiftEdges";
test.initial = {0u, 1u, 31u, 32u, 33u, 63u, 64u, 65u, 127u, UINT32_MAX};
test.expected = test.initial;
auto& code = test.code;
for (const uint64_t source : {uint64_t{0}, uint64_t{1},
uint64_t{0x8000000000000000ull},
uint64_t{0xffffffffffffffffull},
uint64_t{0x0123456789abcdefull}}) {
AppendVMovLiteral(&code, 4, static_cast<u32>(source));
AppendVMovLiteral(&code, 5, static_cast<u32>(source >> 32u));
for (u32 index = 0; index < test.initial.size(); ++index) {
// A buffer load prevents the constant-shift specialization.
AppendVMovU32(&code, 30, index * sizeof(u32));
AppendBufferLoadDword(&code, 3, 30);
for (const bool left : {true, false}) {
AppendVop3(&code, left ? 0x2ff : 0x300, 33, Vgpr(3), Vgpr(4));
const auto amount = test.initial[index] & 63u;
const auto expected = left ? source << amount : source >> amount;
const auto output = static_cast<u32>(test.expected.size());
AppendStoreVgpr(&code, 33, output);
AppendStoreVgpr(&code, 34, output + 1u);
test.expected.push_back(static_cast<u32>(expected));
test.expected.push_back(static_cast<u32>(expected >> 32u));
}
}
}
AppendEnd(&code);
test.initial.resize(test.expected.size());
test.opcodes = {O::V_MOV_B32, O::BUFFER_LOAD_DWORD, O::V_LSHLREV_B64,
O::V_LSHRREV_B64, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.required_spirv = {"OpShiftLeftLogical", "OpShiftRightLogical",
"OpTypeInt 64 0"};
return test;
}
TestCase VectorVop3IntegerOps() {
using O = ShaderOpcode;
std::vector<u32> code;
code.push_back(EncodeVop1(0x01, 0, InlineU32(2)));
code.push_back(EncodeVop1(0x01, 1, InlineU32(3)));
code.push_back(EncodeVop1(0x01, 2, InlineU32(4)));
AppendVMovLiteral(&code, 3, 0xfffffff8u);
code.push_back(EncodeVop1(0x01, 4, InlineU32(5)));
AppendVMovLiteral(&code, 5, 0x11223344u);
AppendVMovLiteral(&code, 6, 0x55667788u);
AppendVMovLiteral(&code, 7, 0x0f0f0f0fu);
AppendVMovLiteral(&code, 8, 0x33333333u);
AppendVMovLiteral(&code, 9, 0xaaaaaaaau);
AppendVMovLiteral(&code, 14, 0x00f00000u);
code.push_back(EncodeVop1(0x01, 15, InlineU32(8)));
AppendVMovLiteral(&code, 30, 0x00010000u);
AppendVMovLiteral(&code, 31, 0x00010000u);
AppendVop3(&code, 0x36d, 10, Vgpr(0), Vgpr(1), Vgpr(2));
AppendVop3(&code, 0x152, 11, Vgpr(3), Vgpr(4), Vgpr(0));
AppendVop3(&code, 0x155, 12, Vgpr(3), Vgpr(4), Vgpr(0));
AppendVop3(&code, 0x158, 13, Vgpr(3), Vgpr(4), Vgpr(0));
AppendVop3(&code, 0x153, 16, Vgpr(3), Vgpr(4), Vgpr(0));
AppendVop3(&code, 0x156, 17, Vgpr(3), Vgpr(4), Vgpr(0));
AppendVop3(&code, 0x159, 18, Vgpr(3), Vgpr(4), Vgpr(0));
AppendVop3(&code, 0x15d, 19, Vgpr(0), Vgpr(4), Vgpr(2));
AppendVop3(&code, 0x346, 20, Vgpr(1), Vgpr(0), Vgpr(2));
AppendVop3(&code, 0x347, 21, Vgpr(0), Vgpr(1), Vgpr(0));
AppendVop3(&code, 0x345, 22, Vgpr(7), Vgpr(8), Vgpr(0));
AppendVop3(&code, 0x36f, 23, Vgpr(1), Vgpr(2), Vgpr(0));
AppendVop3(&code, 0x371, 24, Vgpr(7), Vgpr(8), Vgpr(2));
AppendVop3(&code, 0x372, 25, Vgpr(7), Vgpr(8), Vgpr(2));
AppendVop3(&code, 0x178, 26, Vgpr(7), Vgpr(8), Vgpr(2));
AppendVop3(&code, 0x148, 27, Vgpr(14), InlineU32(20), Vgpr(2));
AppendVop3(&code, 0x149, 28, Vgpr(3), Vgpr(2), Vgpr(2));
AppendVop3(&code, 0x14a, 29, Vgpr(8), Vgpr(7), Vgpr(9));
AppendVop3(&code, 0x14e, 32, Vgpr(5), Vgpr(6), Vgpr(15));
AppendVop3(&code, 0x363, 33, Vgpr(2), Vgpr(15));
AppendVop3(&code, 0x169, 34, Vgpr(30), Vgpr(31));
AppendVop3(&code, 0x16a, 35, Vgpr(30), Vgpr(31));
AppendVop3(&code, 0x16b, 36, Vgpr(30), Vgpr(31));
AppendVop3(&code, 0x16c, 44, Vgpr(3), Vgpr(30));
AppendVop3(&code, 0x142, 37, Vgpr(0), Vgpr(1), Vgpr(2));
AppendVop3(&code, 0x143, 38, Vgpr(0), Vgpr(1), Vgpr(2));
AppendVop3(&code, 0x30f, 39, Vgpr(0), Vgpr(1));
AppendVop3(&code, 0x310, 40, Vgpr(0), Vgpr(1));
AppendVop3(&code, 0x319, 41, Vgpr(0), Vgpr(1));
const u32 results[] = {10, 11, 12, 13, 16, 17, 18, 19, 20, 21,
22, 23, 24, 25, 26, 27, 28, 29, 32, 33,
34, 35, 36, 44, 37, 38, 39, 40, 41};
for (u32 i = 0; i < static_cast<u32>(std::size(results)); i++) {
AppendStoreVgpr(&code, results[i], i);
}
AppendEnd(&code);
return {"VectorVop3IntegerOps",
code,
{},
{9,
0xfffffff8u,
5,
2,
2,
0xfffffff8u,
5,
7,
16,
20,
0x3c3c3c3eu,
0x32,
0x03030307u,
0x3f3f3f3fu,
0x3c3c3c38u,
0x0fu,
0xffffffffu,
0x8b8b8b8bu,
0x44556677u,
0x00000f00u,
0,
1,
0,
0xffffffffu,
10,
10,
5,
0xffffffffu,
1},
{O::V_MOV_B32, O::V_ADD3_U32, O::V_MIN3_I32,
O::V_MAX3_I32, O::V_MED3_I32, O::V_MIN3_U32,
O::V_MAX3_U32, O::V_MED3_U32, O::V_SAD_U32,
O::V_LSHL_ADD_U32, O::V_ADD_LSHL_U32, O::V_XAD_U32,
O::V_LSHL_OR_B32, O::V_AND_OR_B32, O::V_OR3_B32,
O::V_XOR3_B32, O::V_BFE_U32, O::V_BFE_I32,
O::V_BFI_B32, O::V_ALIGNBIT_B32, O::V_BFM_B32,
O::V_MUL_LO_U32, O::V_MUL_HI_U32, O::V_MUL_LO_I32,
O::V_MUL_HI_I32, O::V_MAD_I32_I24, O::V_MAD_U32_U24,
O::V_ADD_I32, O::V_SUB_I32, O::V_SUBREV_I32,
O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}
TestCase VectorBfeI32SignExtendsField() {
using O = ShaderOpcode;
std::vector<u32> code;
for (u32 vgpr = 0; vgpr < 6; vgpr++) {
AppendVMovU32(&code, 30, vgpr * sizeof(u32));
AppendBufferLoadDword(&code, vgpr, 30);
}
AppendVop3(&code, 0x149, 10, Vgpr(0), Vgpr(2), Vgpr(3));
AppendVop3(&code, 0x149, 11, Vgpr(1), Vgpr(4), Vgpr(5));
AppendStoreVgpr(&code, 10, 0);
AppendStoreVgpr(&code, 11, 1);
AppendEnd(&code);
TestCase test;
test.name = "VectorBfeI32SignExtendsField";
test.code = std::move(code);
test.initial = {0x80000000u, 0xfffffff8u, 31, 1, 3, 4};
test.expected = {0xffffffffu, 0xffffffffu};
test.opcodes = {O::V_MOV_B32, O::BUFFER_LOAD_DWORD, O::V_BFE_I32,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.required_spirv = {"OpBitFieldSExtract"};
return test;
}
TestCase VectorAlignByteUsesTwoBitByteOffset() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovLiteral(&code, 0, 0x11223344u);
AppendVMovLiteral(&code, 1, 0x55667788u);
const std::vector<u32> offsets = {0, 1, 2, 3, 4, 5, 7, 8, 31, 32, 0xffffffffu};
for (u32 i = 0; i < static_cast<u32>(std::size(offsets)); i++) {
AppendVMovU32(&code, 2, i * sizeof(u32));
AppendBufferLoadDword(&code, 2, 2);
AppendVop3(&code, 0x14f, 10u + i, Vgpr(0), Vgpr(1), Vgpr(2));
AppendStoreVgpr(&code, 10u + i, i);
}
AppendEnd(&code);
TestCase test{
"VectorAlignByteUsesTwoBitByteOffset",
code,
offsets,
{0x55667788u, 0x44556677u, 0x33445566u, 0x22334455u, 0x55667788u,
0x44556677u, 0x22334455u, 0x55667788u, 0x22334455u, 0x55667788u,
0x22334455u},
{O::V_MOV_B32, O::BUFFER_LOAD_DWORD, O::V_ALIGNBYTE_B32,
O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
test.forbidden_spirv = {"OpTypeInt 64"};
return test;
}
TestCase VectorCarryAndBitCountOps() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovLiteral(&code, 0, 0xffffffffu);
code.push_back(EncodeVop1(0x01, 1, InlineU32(0)));
AppendVMovLiteral(&code, 3, 0x0000f0f0u);
code.push_back(EncodeVop1(0x01, 4, InlineU32(5)));
code.push_back(EncodeVopc(0xc7, Vgpr(1), 1));
code.push_back(EncodeVop2(0x28, 2, Vgpr(0), 1));
code.push_back(EncodeVop2(0x22, 5, Vgpr(3), 4));
AppendVMovLiteral(&code, 6, 0xffffffffu);
code.push_back(EncodeVop2(0x23, 7, Vgpr(6), 4));
code.push_back(EncodeVop2(0x24, 8, Vgpr(6), 4));
AppendStoreVgpr(&code, 2, 0);
AppendStoreSgpr(&code, 106, 1);
AppendStoreVgpr(&code, 5, 2);
AppendStoreVgpr(&code, 7, 3);
AppendStoreVgpr(&code, 8, 4);
AppendEnd(&code);
return {"VectorCarryAndBitCountOps",
code,
{},
{0, 1, 13, 5, 5},
{O::V_MOV_B32, O::V_CMP_T_U32, O::V_ADDC_U32, O::V_BCNT_U32_B32,
O::V_MBCNT_LO_U32_B32, O::V_MBCNT_HI_U32_B32, O::BUFFER_STORE_DWORD,
O::S_ENDPGM}};
}
TestCase VectorMbcntUsesThreadMask() {
using O = ShaderOpcode;
std::vector<u32> code = {
EncodeVop1(0x01, 1, 0),
EncodeVop2(0x1a, 1, InlineU32(2), 1),
EncodeVop2(0x25, 1, Vgpr(0), 1),
EncodeVop2(0x1a, 3, InlineU32(2), 1),
};
AppendVMovLiteral(&code, 2, 0xffffffffu);
code.push_back(EncodeVop1(0x01, 4, InlineU32(0)));
code.push_back(EncodeVop2(0x23, 5, Vgpr(2), 4));
code.push_back(EncodeVop2(0x24, 6, Vgpr(2), 5));
AppendBufferStoreDword(&code, 6, 3);
AppendEnd(&code);
TestCase test;
test.name = "VectorMbcntUsesThreadMask";
test.code = code;
test.expected = {0, 1, 2, 3, 0, 1, 2, 3};
test.opcodes = {O::V_MOV_B32,
O::V_LSHLREV_B32,
O::V_ADD_NC_U32,
O::V_MBCNT_LO_U32_B32,
O::V_MBCNT_HI_U32_B32,
O::BUFFER_STORE_DWORD,
O::S_ENDPGM};
test.compute_info.threads_num[0] = 4;
test.compute_info.threads_num[1] = 1;
test.compute_info.threads_num[2] = 1;
test.compute_info.group_id[0] = true;
test.compute_info.thread_ids_num = 1;
test.compute_info.workgroup_register = 0;
test.has_compute_info = true;
test.dispatch_x = 2;
return test;
}
TestCase VectorAddcUsesPerLaneCarryIn() {
using O = ShaderOpcode;
std::vector<u32> code = {
EncodeVop1(0x01, 1, 0),
EncodeVop2(0x1a, 1, InlineU32(2), 1),
EncodeVop2(0x25, 1, Vgpr(0), 1),
EncodeVopc(0xc7, Vgpr(0), 0),
EncodeVop1(0x01, 2, InlineU32(0)),
EncodeVop2(0x28, 3, Vgpr(2), 2),
EncodeVop2(0x1a, 4, InlineU32(2), 1),
};
AppendBufferStoreDword(&code, 3, 4);
AppendEnd(&code);
TestCase test;
test.name = "VectorAddcUsesPerLaneCarryIn";
test.code = code;
test.expected = std::vector<u32>(8, 1);
test.opcodes = {O::V_MOV_B32, O::V_LSHLREV_B32, O::V_ADD_NC_U32,
O::V_CMP_T_U32, O::V_ADDC_U32, O::BUFFER_STORE_DWORD,
O::S_ENDPGM};
test.compute_info.threads_num[0] = 4;
test.compute_info.threads_num[1] = 1;
test.compute_info.threads_num[2] = 1;
test.compute_info.group_id[0] = true;
test.compute_info.thread_ids_num = 1;
test.compute_info.workgroup_register = 0;
test.has_compute_info = true;
test.dispatch_x = 2;
return test;
}
TestCase VectorAddcWritesPerLaneCarryOut() {
using O = ShaderOpcode;
std::vector<u32> code = {EncodeVop1(0x01, 1, Vgpr(0)),
EncodeVop2(0x1a, 4, InlineU32(2), 1),
EncodeVopc(0xc7, Vgpr(0), 0)};
AppendBufferLoadDword(&code, 26, 4);
code.push_back(0x500434f9u);
code.push_back(0x0c860680u);
code.push_back(EncodeVop1(0x01, 6, 106));
AppendBufferStoreDword(&code, 6, 4);
AppendEnd(&code);
TestCase test;
test.name = "VectorAddcWritesPerLaneCarryOut";
test.code = code;
test.initial = {0x0000ffffu, 0u, 0x0000ffffu, 0u};
test.expected = {5u, 5u, 5u, 5u};
test.opcodes = {O::V_MOV_B32, O::V_LSHLREV_B32, O::V_CMP_T_U32,
O::BUFFER_LOAD_DWORD, O::V_ADDC_U32, O::BUFFER_STORE_DWORD,
O::S_ENDPGM};
test.decoded_counts = {
{"V_ADDC_U32 v2, vcc_lo, 0, v26.sdwa(sel=4,sext=1), vcc_lo", 1}};
test.ir_counts = {{" = BitFieldSExtract ", 1}, {" = IAddCarry32 ", 2}};
test.required_spirv = {"OpBitFieldSExtract"};
test.compute_info.threads_num[0] = 4;
test.compute_info.threads_num[1] = 1;
test.compute_info.threads_num[2] = 1;
test.compute_info.thread_ids_num = 1;
test.has_compute_info = true;
return test;
}
TestCase VectorSubCoCiU32CompactAndVop3() {
using O = ShaderOpcode;
std::vector<u32> code;
code.push_back(EncodeVop2(0x1a, 4, InlineU32(2), 0));
AppendBufferLoadDword(&code, 1, 4);
code.push_back(EncodeVop2(0x25, 5, InlineU32(16), 4));
AppendBufferLoadDword(&code, 2, 5);
code.push_back(EncodeVop2(0x1b, 3, InlineU32(1), 0));
code.push_back(EncodeVopc(0xc5, InlineU32(0), 3));
code.push_back(EncodeSop1(0x04, 20, 106));
code.push_back(EncodeVop2(0x29, 10, Vgpr(1), 2));
AppendVMovU32(&code, 6, 1);
code.push_back(EncodeVop2(0x01, 11, InlineU32(0), 6));
AppendStoreVgprAtLaneDwordOffset(&code, 10, 0, 0);
AppendStoreVgprAtLaneDwordOffset(&code, 11, 0, 4);
code.push_back(EncodeVopc(0xc2, InlineU32(0), 6));
AppendVop3B(&code, 0x129u, 12, 22, Vgpr(1), Vgpr(2), 20);
AppendVop3(&code, 0x101u, 13, InlineU32(0), Vgpr(6), 22);
code.push_back(EncodeVop2(0x01, 14, InlineU32(0), 6));
AppendStoreVgprAtLaneDwordOffset(&code, 12, 0, 8);
AppendStoreVgprAtLaneDwordOffset(&code, 13, 0, 12);
AppendStoreVgprAtLaneDwordOffset(&code, 14, 0, 16);
AppendEnd(&code);
TestCase test;
test.name = "VectorSubCoCiU32CompactAndVop3";
test.code = std::move(code);
test.initial = {5u, 5u, 3u, 3u, 3u, 3u, 5u, 3u, 0u, 0u,
0u, 0u, 0u, 0u, 0u, 0u, 0u, 0u, 0u, 0u};
test.expected = {2u, 1u, 0xfffffffeu, 0xffffffffu, 0u, 0u, 1u,
1u, 2u, 1u, 0xfffffffeu, 0xffffffffu, 0u, 0u,
1u, 1u, 0u, 0u, 0u, 0u};
test.opcodes = {O::V_MOV_B32, O::V_LSHLREV_B32, O::V_ADD_NC_U32,
O::V_AND_B32, O::V_CMP_NE_U32, O::V_CMP_EQ_U32,
O::V_SUB_CO_CI_U32, O::V_CNDMASK_B32, O::S_MOV_B64,
O::BUFFER_LOAD_DWORD, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.decoded_counts = {{"V_SUB_CO_CI_U32", 2}};
test.required_spirv = {"OpISub", "OpUGreaterThan", "OpLogicalOr"};
test.compute_info.threads_num[0] = 4;
test.compute_info.threads_num[1] = 1;
test.compute_info.threads_num[2] = 1;
test.compute_info.thread_ids_num = 1;
test.compute_info.wave_size = 32;
test.has_compute_info = true;
return test;
}
TestCase VectorSubrevCoCiU32ExactRawOnGpu() {
using O = ShaderOpcode;
std::vector<u32> code;
code.push_back(EncodeVop1(0x01, 24, Vgpr(0)));
code.push_back(EncodeSop1(0x04, 20, 126)); // s_mov_b64 s[20:21], exec
code.push_back(EncodeVop1(0x01, 1, InlineU32(2)));
code.push_back(EncodeVop1(0x01, 2, InlineU32(1)));
code.push_back(EncodeVop2(0x1a, 4, InlineU32(2), 0));
code.push_back(EncodeVop2(0x25, 5, InlineU32(16), 4));
code.push_back(EncodeVopc(0xc1, Vgpr(0), 1));
code.push_back(EncodeSop1(0x04, 22, 106)); // s_mov_b64 s[22:23], vcc
code.push_back(EncodeVopc(0xc5, InlineU32(0), 0));
AppendVMovLiteral(&code, 0, 0xdeadbeefu);
code.push_back(EncodeSop1(0x04, 126, 106)); // s_mov_b64 exec, vcc
code.push_back(EncodeSop1(0x04, 106, 22)); // s_mov_b64 vcc, s[22:23]
code.push_back(0x54003080u); // v_subrev_co_ci_u32 v0, vcc, 0, v24, vcc
code.push_back(EncodeSop1(0x04, 126, 20)); // s_mov_b64 exec, s[20:21]
code.push_back(EncodeVop2(0x01, 3, InlineU32(0), 2));
AppendBufferStoreDword(&code, 0, 4);
AppendBufferStoreDword(&code, 3, 5);
AppendEnd(&code);
TestCase test;
test.name = "VectorSubrevCoCiU32ExactRawOnGpu";
test.code = code;
test.expected = {0xdeadbeefu, 0, 2, 3, 0, 0, 0, 0};
test.opcodes = {O::V_MOV_B32, O::V_LSHLREV_B32,
O::V_ADD_NC_U32, O::V_CMP_LT_U32,
O::S_MOV_B64, O::V_CMP_NE_U32,
O::V_SUBREV_CO_CI_U32, O::V_CNDMASK_B32,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.required_spirv = {"OpISub", "OpUGreaterThan"};
test.compute_info.threads_num[0] = 4;
test.compute_info.threads_num[1] = 1;
test.compute_info.threads_num[2] = 1;
test.compute_info.group_id[0] = true;
test.compute_info.wave_size = 32;
test.compute_info.thread_ids_num = 1;
test.compute_info.workgroup_register = 0;
test.has_compute_info = true;
return test;
}
TestCase VectorVop3BSubrevCoCiUsesEncodedMasks() {
using O = ShaderOpcode;
std::vector<u32> code;
code.push_back(EncodeVop1(0x01, 24, Vgpr(0)));
code.push_back(EncodeVop2(0x25, 24, InlineU32(3), 24));
code.push_back(EncodeVop1(0x01, 1, InlineU32(2)));
code.push_back(EncodeVop1(0x01, 2, InlineU32(1)));
code.push_back(EncodeVop2(0x1a, 4, InlineU32(2), 0));
code.push_back(EncodeVop2(0x25, 5, InlineU32(16), 4));
code.push_back(EncodeVopc(0xc2, InlineU32(1), 0));
code.push_back(EncodeSop1(0x04, 20, 106)); // s_mov_b64 s[20:21], vcc
AppendSMovLiteral(&code, 106, 0);
AppendSMovLiteral(&code, 107, 0);
AppendVop3B(&code, 0x12au, 10, 22, InlineU32(5), Vgpr(24), 20);
AppendVop3(&code, 0x101u, 3, InlineU32(0), Vgpr(2), 22);
AppendBufferStoreDword(&code, 10, 4);
AppendBufferStoreDword(&code, 3, 5);
AppendEnd(&code);
TestCase test;
test.name = "VectorVop3BSubrevCoCiUsesEncodedMasks";
test.code = code;
test.expected = {0xfffffffeu, 0xfffffffeu, 0, 1, 1, 1, 0, 0};
test.opcodes = {O::V_MOV_B32, O::V_LSHLREV_B32,
O::V_ADD_NC_U32, O::V_CMP_EQ_U32,
O::S_MOV_B64, O::S_MOV_B32,
O::V_SUBREV_CO_CI_U32, O::V_CNDMASK_B32,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.compute_info.threads_num[0] = 4;
test.compute_info.threads_num[1] = 1;
test.compute_info.threads_num[2] = 1;
test.compute_info.group_id[0] = true;
test.compute_info.wave_size = 32;
test.compute_info.thread_ids_num = 1;
test.compute_info.workgroup_register = 0;
test.has_compute_info = true;
return test;
}
TestCase VectorVop3BCarryOutWritesSgprMask() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovLiteral(&code, 1, 0xffffffffu);
AppendVMovU32(&code, 2, 1);
AppendVMovU32(&code, 3, 0);
AppendVop3B(&code, 0x30fu, 10, 0, Vgpr(1), Vgpr(2));
AppendStoreSgprAtLaneDwordOffset(&code, 0, 0, 0);
AppendVop3B(&code, 0x310u, 11, 0, Vgpr(3), Vgpr(2));
AppendStoreSgprAtLaneDwordOffset(&code, 0, 0, 4);
AppendVop3B(&code, 0x319u, 12, 0, Vgpr(2), Vgpr(3));
AppendStoreSgprAtLaneDwordOffset(&code, 0, 0, 8);
AppendEnd(&code);
TestCase test;
test.name = "VectorVop3BCarryOutWritesSgprMask";
test.code = code;
test.expected = std::vector<u32>(12, 15u);
test.opcodes = {O::V_MOV_B32, O::V_ADD_I32, O::V_SUB_I32,
O::V_SUBREV_I32, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.compute_info.threads_num[0] = 4;
test.compute_info.threads_num[1] = 1;
test.compute_info.threads_num[2] = 1;
test.compute_info.group_id[0] = true;
test.compute_info.thread_ids_num = 1;
test.compute_info.workgroup_register = 0;
test.has_compute_info = true;
return test;
}
TestCase VectorVop3BCarryOutUsesEncodedSdst() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovLiteral(&code, 1, 0xffffffffu);
AppendVMovU32(&code, 2, 1);
AppendVMovU32(&code, 3, 0);
AppendVop3B(&code, 0x30fu, 10, 20, Vgpr(1), Vgpr(2));
AppendStoreSgprAtLaneDwordOffset(&code, 20, 0, 0);
AppendVop3B(&code, 0x310u, 11, 22, Vgpr(3), Vgpr(2));
AppendStoreSgprAtLaneDwordOffset(&code, 22, 0, 4);
AppendVop3B(&code, 0x319u, 12, 24, Vgpr(2), Vgpr(3));
AppendStoreSgprAtLaneDwordOffset(&code, 24, 0, 8);
AppendEnd(&code);
TestCase test;
test.name = "VectorVop3BCarryOutUsesEncodedSdst";
test.code = code;
test.expected = std::vector<u32>(12, 15u);
test.opcodes = {O::V_MOV_B32, O::V_ADD_I32, O::V_SUB_I32,
O::V_SUBREV_I32, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.compute_info.threads_num[0] = 4;
test.compute_info.threads_num[1] = 1;
test.compute_info.threads_num[2] = 1;
test.compute_info.group_id[0] = true;
test.compute_info.thread_ids_num = 1;
test.compute_info.workgroup_register = 0;
test.has_compute_info = true;
return test;
}
TestCase VectorVop3BSubCoU32UsesRdna2Opcode310() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 1, 0);
AppendVMovU32(&code, 2, 1);
AppendVMovU32(&code, 3, 1);
AppendVMovLiteral(&code, 4, 0xffffffffu);
AppendVMovLiteral(&code, 5, 0x80000000u);
code.push_back(EncodeVopc(0xc2, InlineU32(1), 0));
code.push_back(EncodeVop2(0x01, 1, Vgpr(1), Vgpr(3)));
code.push_back(EncodeVopc(0xc2, InlineU32(2), 0));
code.push_back(EncodeVop2(0x01, 1, Vgpr(1), Vgpr(4)));
code.push_back(EncodeVopc(0xc2, InlineU32(3), 0));
code.push_back(EncodeVop2(0x01, 1, Vgpr(1), Vgpr(5)));
code.push_back(EncodeVop2(0x01, 2, Vgpr(2), Vgpr(4)));
AppendVop3B(&code, 0x310u, 10, 20, Vgpr(1), Vgpr(2));
AppendStoreVgprAtLaneDwordOffset(&code, 10, 0, 0);
AppendStoreSgprAtLaneDwordOffset(&code, 20, 0, 4);
AppendEnd(&code);
TestCase test;
test.name = "VectorVop3BSubCoU32UsesRdna2Opcode310";
test.code = code;
test.expected = {0xffffffffu, 0, 0xfffffffeu, 0x80000001u, 9, 9, 9, 9};
test.opcodes = {O::V_MOV_B32, O::V_CMP_EQ_U32, O::V_CNDMASK_B32,
O::V_SUB_I32, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.compute_info.threads_num[0] = 4;
test.compute_info.threads_num[1] = 1;
test.compute_info.threads_num[2] = 1;
test.compute_info.group_id[0] = true;
test.compute_info.thread_ids_num = 1;
test.compute_info.workgroup_register = 0;
test.has_compute_info = true;
return test;
}
TestCase VectorMadU64U32UnsignedCarryOut() {
using O = ShaderOpcode;
struct Input {
u32 lhs;
u32 rhs;
uint64_t addend;
};
constexpr Input inputs[] = {
{0, UINT32_MAX, UINT64_MAX},
{UINT32_MAX, 2, 0xffffffff00000002ull},
{UINT32_MAX, UINT32_MAX, UINT64_MAX},
{0x80000000u, 0x80000000u, 0x4000000000000000ull},
{3, 5, 7},
};
TestCase test;
test.name = "VectorMadU64U32UnsignedCarryOut";
for (const auto &input : inputs) {
test.initial.insert(
test.initial.end(),
{input.lhs, input.rhs, u32(input.addend), u32(input.addend >> 32)});
}
test.expected = test.initial;
auto &code = test.code;
for (u32 i = 0; i < std::size(inputs); ++i) {
for (u32 word = 0; word < 4; ++word) {
AppendVMovU32(&code, 30, (i * 4 + word) * sizeof(u32));
AppendBufferLoadDword(&code, word < 2 ? word + 1 : word + 2, 30);
}
AppendVop3B(&code, 0x176u, 10, 20, Vgpr(1), Vgpr(2), Vgpr(4));
const auto output = static_cast<u32>(test.expected.size());
AppendStoreVgpr(&code, 10, output);
AppendStoreVgpr(&code, 11, output + 1);
AppendStoreSgprPair(&code, 20, output + 2);
const auto product = uint64_t(inputs[i].lhs) * inputs[i].rhs;
const auto sum = product + inputs[i].addend;
test.expected.insert(test.expected.end(),
{u32(sum), u32(sum >> 32), u32(sum < product), 0});
}
AppendEnd(&code);
test.initial.resize(test.expected.size());
test.opcodes = {O::V_MOV_B32, O::BUFFER_LOAD_DWORD, O::V_MAD_U64_U32,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
// NVIDIA rejected the native widening multiply in a large Astro's Playroom
// vertex shader. Keep its 32x32 product portable and its 64-bit addition
// native.
test.required_spirv = {"OpTypeInt 64 0", "OpUMulExtended", "OpIMul %uint",
"OpIAdd %ulong", "OpULessThan"};
test.forbidden_spirv = {"OpIMul %ulong", "OpIMul %long"};
return test;
}
TestCase VectorLaneAndPackedOps() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovLiteral(&code, 0, 0x12345678u);
code.push_back(EncodeVop1(0x01, 1, InlineU32(0)));
AppendVMovLiteral(&code, 4, 0x40003c00u);
AppendVMovLiteral(&code, 5, 0x44004200u);
AppendVMovLiteral(&code, 6, 0x3c003c00u);
AppendVMovLiteral(&code, 7, 0x3f800000u);
AppendVMovLiteral(&code, 8, 0x40000000u);
AppendVMovLiteral(&code, 9, 0x12345678u);
AppendVMovLiteral(&code, 10, 0xabcdef01u);
code.push_back(EncodeVop1(0x02, 0, Vgpr(0)));
AppendVop3(&code, 0x360, 1, Vgpr(0), InlineU32(0));
AppendVop3(&code, 0x361, 1, 0, InlineU32(0));
AppendVop3(&code, 0x377, 2, Vgpr(0), InlineU32(0), InlineU32(0));
code.push_back(EncodeVop2(0x2f, 3, Vgpr(7), 8));
AppendVop3p(&code, 0x0f, 11, Vgpr(4), Vgpr(5), 0, 0x3);
AppendVop3p(&code, 0x10, 12, Vgpr(4), Vgpr(5), 0, 0x3);
AppendVop3p(&code, 0x11, 13, Vgpr(4), Vgpr(5), 0, 0x3);
AppendVop3p(&code, 0x12, 14, Vgpr(4), Vgpr(5), 0, 0x3);
AppendVop3p(&code, 0x0e, 15, Vgpr(4), Vgpr(5), Vgpr(6), 0x7);
AppendVop3(&code, 0x362, 16, Vgpr(7), InlineU32(1));
AppendVop3(&code, 0x36a, 17, Vgpr(9), Vgpr(10));
AppendStoreSgpr(&code, 0, 0);
AppendStoreSgpr(&code, 1, 1);
const u32 results[] = {1, 2, 3, 11, 12, 13, 14, 15, 16, 17};
for (u32 i = 0; i < static_cast<u32>(std::size(results)); i++) {
AppendStoreVgpr(&code, results[i], i + 2u);
}
AppendEnd(&code);
return {"VectorLaneAndPackedOps",
code,
{},
{0x12345678u, 0x12345678u, 0x12345678u, 0x12345678u, 0x40003c00u,
0x46004400u, 0x48004200u, 0x40003c00u, 0x44004200u, 0x48804400u,
0x40000000u, 0xef015678u},
{O::V_MOV_B32, O::V_READFIRSTLANE_B32, O::V_READLANE_B32,
O::V_WRITELANE_B32, O::V_PERMLANE16_B32, O::V_CVT_PKRTZ_F16_F32,
O::V_PK_ADD_F16, O::V_PK_MUL_F16, O::V_PK_MIN_F16, O::V_PK_MAX_F16,
O::V_PK_FMA_F16, O::V_LDEXP_F32, O::V_CVT_PK_U16_U32,
O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}
TestCase Vop2PkFmacF16AccumulatesPackedHalvesIndependently() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovLiteral(&code, 0, 0x50004c00u); // low=16.0h, high=32.0h
AppendVMovLiteral(&code, 1, 0x42003c00u); // low=1.0h, high=3.0h
AppendVMovLiteral(&code, 2, 0x48004400u); // low=4.0h, high=8.0h
code.push_back(EncodeVop2(0x3c, 0, Vgpr(1), 2));
AppendStoreVgpr(&code, 0, 0);
code.push_back(EncodeVop2(0x3c, 0, Vgpr(1), 2));
AppendStoreVgpr(&code, 0, 1);
AppendVMovLiteral(&code, 3, 0x50004c00u);
code.push_back(EncodeVop2(0x3c, 3, 242, 2)); // Inline 1.0h is {1.0h, 0.0h}.
AppendStoreVgpr(&code, 3, 2);
AppendEnd(&code);
return {"Vop2PkFmacF16AccumulatesPackedHalvesIndependently",
code,
{},
{0x53004d00u, 0x55004e00u, 0x50004d00u},
{O::V_MOV_B32, O::V_PK_FMAC_F16, O::BUFFER_STORE_DWORD,
O::S_ENDPGM}};
}
TestCase Vop2PkFmacF16DppNegatesBothPackedHalves() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovLiteral(&code, 0, 0x50004c00u); // low=16.0h, high=32.0h
AppendVMovLiteral(&code, 1, 0x42003c00u); // low=1.0h, high=3.0h
AppendVMovLiteral(&code, 2, 0x48004400u); // low=4.0h, high=8.0h
code.push_back(EncodeVop2(0x3c, 0, 250, 2));
code.push_back(EncodeVop2Dpp(1, 0, 0xf, 0xf, 1));
AppendStoreVgpr(&code, 0, 0);
AppendEnd(&code);
return {"Vop2PkFmacF16DppNegatesBothPackedHalves",
code,
{},
{0x48004a00u},
{O::V_MOV_B32, O::V_PK_FMAC_F16, O::BUFFER_STORE_DWORD,
O::S_ENDPGM}};
}
TestCase Vop3pOpselHiUsesArchitecturalSourceBits() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovLiteral(&code, 0, 0x42003c00u); // low=1.0h, high=3.0h
AppendVMovLiteral(&code, 1, 0x48004400u); // low=4.0h, high=8.0h
AppendVMovLiteral(&code, 2, 0x50004c00u); // low=16.0h, high=32.0h
// Raw V_PK_FMA_F16 words keep this test independent of the encoders above.
code.insert(code.end(), {0xcc0e000au, 0x0c0a0300u}); // src0 high: 3*4+16=28
code.insert(code.end(), {0xcc0e000bu, 0x140a0300u}); // src1 high: 1*8+16=24
code.insert(code.end(), {0xcc0e400cu, 0x040a0300u}); // src2 high: 1*4+32=36
for (u32 i = 0; i < 3; i++) {
AppendStoreVgpr(&code, 10 + i, i);
}
AppendEnd(&code);
return {"Vop3pOpselHiUsesArchitecturalSourceBits",
code,
{},
{0x4f004d00u, 0x4e004d00u, 0x50804d00u},
{O::V_MOV_B32, O::V_PK_FMA_F16, O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}
TestCase Vop3pIntegerNegationCapturedAndSelectedHalves() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovLiteral(&code, 6, 0xfed4ff38u); // (-200, -300)
AppendVMovLiteral(&code, 3, 0x012c00c8u); // (200, 300)
// Captured temporal shader: NEG_LO/HI turns 0x7fff into 0xffff (-1).
// Adding that clamped offset must select (199,299), not collapse to (0,0).
code.insert(code.end(), {0xcc074107u, 0x30020cffu, 0x00007fffu});
code.insert(code.end(), {0xcc0a4010u, 0x18020f03u});
AppendStoreVgpr(&code, 7, 0);
AppendStoreVgpr(&code, 16, 1);
AppendVMovLiteral(&code, 0, 0x80011234u);
AppendVMovLiteral(&code, 1, 0xbeef5678u);
// Independently negate src0 low and src1 high, then reverse both the
// modified sources and src1 half selection. NEG flips the selected sign bit.
AppendVop3p(&code, 0x0a, 10, Vgpr(0), Vgpr(1), 0, 0x3, 0, 0x2, 0x1);
AppendVop3p(&code, 0x0a, 11, Vgpr(0), Vgpr(1), 0, 0x1, 0x2, 0x1, 0x2);
AppendStoreVgpr(&code, 10, 2);
AppendStoreVgpr(&code, 11, 3);
AppendEnd(&code);
return {"Vop3pIntegerNegationCapturedAndSelectedHalves",
code,
{},
{0xffffffffu, 0x012b00c7u, 0xbef0e8acu, 0x56795123u},
{O::V_MOV_B32, O::V_PK_MAX_I16, O::V_PK_ADD_U16,
O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}
TestCase Vop3pInt16Ops() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovLiteral(&code, 1, 0x80007fffu);
AppendVMovLiteral(&code, 2, 0x00010001u);
AppendVMovLiteral(&code, 3, 0xfffe0003u);
AppendVMovLiteral(&code, 4, 0x00130004u);
AppendVMovLiteral(&code, 5, 0x8010f00fu);
AppendVMovLiteral(&code, 6, 0x00050007u);
AppendVop3p(&code, 0x00u, 10, Vgpr(1), Vgpr(3), Vgpr(6), 0x7u);
AppendVop3p(&code, 0x09u, 11, Vgpr(1), Vgpr(3), Vgpr(6), 0x7u);
AppendVop3p(&code, 0x01u, 12, Vgpr(1), Vgpr(3), 0, 0x3u);
AppendVop3p(&code, 0x02u, 13, Vgpr(1), Vgpr(2), 0, 0x3u);
AppendVop3p(&code, 0x0bu, 14, Vgpr(2), Vgpr(3), 0, 0x3u);
AppendVop3p(&code, 0x04u, 15, Vgpr(4), Vgpr(5), 0, 0x3u);
AppendVop3p(&code, 0x05u, 16, Vgpr(4), Vgpr(5), 0, 0x3u);
AppendVop3p(&code, 0x06u, 17, Vgpr(4), Vgpr(5), 0, 0x3u);
AppendVop3p(&code, 0x08u, 18, Vgpr(1), Vgpr(2), 0, 0x3u);
AppendVop3p(&code, 0x0cu, 19, Vgpr(1), Vgpr(2), 0, 0x3u);
AppendVop3p(&code, 0x0du, 20, Vgpr(1), Vgpr(2), 0, 0x3u);
for (u32 i = 0; i < 11; i++) {
AppendStoreVgpr(&code, 10 + i, i);
}
AppendEnd(&code);
TestCase test;
test.name = "Vop3pInt16Ops";
test.code = std::move(code);
test.expected = {0x00058004u, 0x00058004u, 0x00007ffdu, 0x80018000u,
0x0003fffeu, 0x008000f0u, 0x10020f00u, 0xf002ff00u,
0x80000001u, 0x80007fffu, 0x00010001u};
test.opcodes = {O::V_MOV_B32, O::V_PK_MAD_I16, O::V_PK_MAD_U16,
O::V_PK_MUL_LO_U16, O::V_PK_ADD_I16, O::V_PK_SUB_U16,
O::V_PK_LSHLREV_B16, O::V_PK_LSHRREV_B16, O::V_PK_ASHRREV_I16,
O::V_PK_MIN_I16, O::V_PK_MAX_U16, O::V_PK_MIN_U16,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
return test;
}
TestCase CvtPkU8F32PacksSelectedByte() {
using O = ShaderOpcode;
std::vector<u32> code;
// RDNA2 V_CVT_PK_U8_F32: uint8(S0) replaces byte S1[1:0] in S2.
AppendVMovLiteral(&code, 0, 0x414c0000u); // 12.75f truncates to 12.
AppendVMovLiteral(&code, 1, 0x43960000u); // 300.0f saturates to 255.
AppendVMovLiteral(&code, 2, 0xbf800000u); // Negative values saturate to zero.
AppendVMovLiteral(&code, 3, 0x7fc00000u); // NaN saturates to zero.
AppendVMovLiteral(&code, 4, 0x11223344u);
AppendVop3(&code, 0x15eu, 10, Vgpr(0), InlineU32(0), Vgpr(4));
AppendVop3(&code, 0x15eu, 11, Vgpr(1), InlineU32(5), Vgpr(4));
AppendVop3(&code, 0x15eu, 12, Vgpr(2), InlineU32(2), Vgpr(4));
AppendVop3(&code, 0x15eu, 13, Vgpr(3), InlineU32(3), Vgpr(4));
for (u32 i = 0; i < 4; i++) {
AppendStoreVgpr(&code, 10 + i, i);
}
AppendEnd(&code);
return {
"CvtPkU8F32PacksSelectedByte",
code,
{},
{0x1122330cu, 0x1122ff44u, 0x11003344u, 0x00223344u},
{O::V_MOV_B32, O::V_CVT_PK_U8_F32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}
TestCase CvtPkrtzF16F32SubnormalRoundsTowardZero() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovLiteral(&code, 0, 0x33c00000u);
AppendVMovLiteral(&code, 1, 0xb3c00000u);
code.push_back(EncodeVop2(0x2f, 2, Vgpr(0), 1));
AppendStoreVgpr(&code, 2, 0);
AppendEnd(&code);
return {"CvtPkrtzF16F32SubnormalRoundsTowardZero",
code,
{},
{0x80010001u},
{O::V_MOV_B32, O::V_CVT_PKRTZ_F16_F32, O::BUFFER_STORE_DWORD,
O::S_ENDPGM}};
}
TestCase CvtPkrtzF16F32SdwaAndOutputModifiers() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovLiteral(&code, 0, 0x3f000000u); // 0.5f
AppendVMovLiteral(&code, 1, 0x40000000u); // 2.0f
AppendVMovLiteral(&code, 2, 0x3f802000u); // exactly 0x3c01h
AppendVMovLiteral(&code, 3, 0x00000000u);
AppendVMovLiteral(&code, 4, 0xbf000000u); // -0.5f
for (u32 dst = 10; dst <= 18; dst++) {
AppendVMovLiteral(&code, dst, 0xa1b2c3d4u);
}
code.push_back(EncodeVop2(0x2f, 10, 249, 1));
code.push_back(EncodeVop2Sdwa(0, 6, 2, 6, 6, 0, 0, 0, 0, 0, 0, 0, 0,
1));
code.push_back(EncodeVop2(0x2f, 11, 249, 3));
code.push_back(EncodeVop2Sdwa(2, 0, 0));
code.push_back(EncodeVop2(0x2f, 12, 249, 1));
code.push_back(EncodeVop2Sdwa(0, 6, 0, 3, 5));
code.push_back(EncodeVop2(0x2f, 13, 249, 1));
code.push_back(EncodeVop2Sdwa(4, 6, 0, 6, 6, 0, 0, 0, 1, 1));
code.push_back(EncodeVop2(0x2f, 14, 250, 1));
code.push_back(EncodeVop2Dpp(0, 0, 0xf, 0xf, 1));
AppendVop3(&code, 0x12f, 15, Vgpr(0), Vgpr(1), 0, 0, 0, true);
code.push_back(EncodeVop2(0x2f, 16, 249, 1));
code.push_back(EncodeVop2Sdwa(0, 4, 2));
code.push_back(EncodeVop2(0x2f, 17, 249, 1));
code.push_back(EncodeVop2Sdwa(0, 6, 2, 6, 6, 0, 0, 0, 0, 0, 0, 0, 0,
0, 1));
AppendVop3(&code, 0x12f, 18, Vgpr(0), Vgpr(1), 0, 0, 0, false, 1);
for (u32 dst = 10; dst <= 18; dst++) {
AppendStoreVgpr(&code, dst, dst - 10);
}
AppendEnd(&code);
return {"CvtPkrtzF16F32SdwaAndOutputModifiers",
code,
{},
{0x3c003800u, 0x00000001u, 0x00000000u, 0xc0003800u,
0x4000b800u, 0x3c003800u, 0xa1b23800u, 0x44003c00u,
0x44003c00u},
{O::V_MOV_B32, O::V_CVT_PKRTZ_F16_F32, O::BUFFER_STORE_DWORD,
O::S_ENDPGM}};
}
TestCase PackedMinMaxF16NanAndSignedZeroEdges() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovLiteral(&code, 0, 0x80004000u);
AppendVMovLiteral(&code, 1, 0x00007e00u);
AppendVMovLiteral(&code, 2, 0x00004000u);
AppendVMovLiteral(&code, 3, 0x80007e00u);
AppendVMovLiteral(&code, 4, 0x40007d01u);
AppendVMovLiteral(&code, 5, 0x7d014000u);
AppendVMovLiteral(&code, 6, 0x40007e01u);
AppendVMovLiteral(&code, 7, 0x7e013c00u);
AppendVop3p(&code, 0x11, 10, Vgpr(0), Vgpr(1), 0, 0x3);
AppendVop3p(&code, 0x12, 11, Vgpr(2), Vgpr(3), 0, 0x3);
AppendVop3p(&code, 0x11, 12, Vgpr(4), Vgpr(5), 0, 0x3);
AppendVop3p(&code, 0x12, 13, Vgpr(4), Vgpr(5), 0, 0x3);
AppendVop3p(&code, 0x11, 14, Vgpr(6), Vgpr(7), 0, 0x3);
AppendVop3p(&code, 0x12, 15, Vgpr(6), Vgpr(7), 0, 0x3);
for (u32 i = 0; i < 6; i++) {
AppendStoreVgpr(&code, 10 + i, i);
}
AppendEnd(&code);
return {"PackedMinMaxF16NanAndSignedZeroEdges",
code,
{},
{0x80004000u, 0x00004000u, 0x7f017f01u, 0x7f017f01u, 0x40003c00u, 0x40003c00u},
{O::V_MOV_B32, O::V_PK_MIN_F16, O::V_PK_MAX_F16,
O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}
TestCase VectorMinMaxF16Ops() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovLiteral(&code, 0, 0xaaaa4000u); // low=2.0h
AppendVMovLiteral(&code, 1, 0xbbbb3c00u); // low=1.0h
AppendVMovLiteral(&code, 2, 0x12345678u);
AppendVMovLiteral(&code, 3, 0x87654321u);
AppendVMovLiteral(&code, 4, 0x5555aaaau);
AppendVMovLiteral(&code, 5, 0x6666bbbbu);
AppendVMovLiteral(&code, 6, 0x7777bbbbu);
AppendVMovLiteral(&code, 7, 0x8888bbbbu);
AppendVMovLiteral(&code, 8, 0x9999bbbbu);
AppendVMovLiteral(&code, 9, 0x99990000u);
code.push_back(EncodeVop2(0x32, 5, Vgpr(0), 1));
code.push_back(EncodeVop2(0x33, 6, Vgpr(0), 1));
code.push_back(EncodeVop2(0x34, 7, Vgpr(0), 1));
code.push_back(EncodeVop2(0x35, 4, Vgpr(0), 1));
code.push_back(EncodeVop2(0x39, 2, Vgpr(0), 1));
code.push_back(EncodeVop2(0x3a, 3, Vgpr(0), 1));
code.push_back(EncodeVop2(0x35, 9, 249, 1));
code.push_back(EncodeVop2Sdwa(0, 4, 2, 4, 4));
code.push_back(0x660000f9u);
code.push_back(EncodeVop2Sdwa(0, 4, 2, 4, 4));
AppendStoreVgpr(&code, 5, 0);
AppendStoreVgpr(&code, 6, 1);
AppendStoreVgpr(&code, 7, 2);
AppendStoreVgpr(&code, 0, 3);
AppendStoreVgpr(&code, 4, 4);
AppendStoreVgpr(&code, 2, 5);
AppendStoreVgpr(&code, 3, 6);
AppendStoreVgpr(&code, 9, 7);
AppendEnd(&code);
return {"VectorMinMaxF16Ops",
code,
{},
{0x66664200u, 0x77773c00u, 0x8888bc00u, 0xaaaa0000u, 0x55554000u,
0x12344000u, 0x87653c00u, 0x99994000u},
{O::V_MOV_B32, O::V_ADD_F16, O::V_SUB_F16, O::V_SUBREV_F16,
O::V_MUL_F16, O::V_MAX_F16, O::V_MIN_F16, O::BUFFER_STORE_DWORD,
O::S_ENDPGM}};
}
TestCase VectorCvtF16U16ByteSource() {
using O = ShaderOpcode;
constexpr std::array inputs = {0xff123456u, 0x80123456u, 0x00123456u};
std::vector<u32> code;
AppendVMovLiteral(&code, 16, 0xa5a55a5au);
for (u32 i = 0; i < inputs.size(); i++) {
AppendVMovU32(&code, 30, i * 4u);
AppendBufferLoadDword(&code, 18, 30);
// Convert unsigned byte3 to FP16 word0 while preserving word1 of v16.
code.insert(code.end(), {0x7e20a0f9u, 0x00031412u});
AppendStoreVgpr(&code, 16, i);
}
AppendEnd(&code);
TestCase test;
test.name = "VectorCvtF16U16ByteSource";
test.code = std::move(code);
test.initial.assign(inputs.begin(), inputs.end());
test.expected = {0xa5a55bf8u, 0xa5a55800u, 0xa5a50000u};
test.opcodes = {O::V_MOV_B32, O::BUFFER_LOAD_DWORD, O::V_CVT_F16_U16,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.required_spirv = {"OpBitFieldUExtract", "OpConvertUToF"};
return test;
}
TestCase VectorCvtU16F16Sdwa() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovLiteral(&code, 0, 0x45004200u); // low=3.0h, high=5.0h
AppendVMovLiteral(&code, 1, 0xbc007c00u); // low=+inf, high=-1.0h
AppendVMovLiteral(&code, 2, 0x00007e00u); // low=qNaN
AppendVMovLiteral(&code, 10, 0x12345678u);
AppendVMovLiteral(&code, 11, 0x87654321u);
AppendVMovLiteral(&code, 12, 0xabcd1111u);
AppendVMovLiteral(&code, 13, 0x77772222u);
AppendVMovLiteral(&code, 14, 0x5555aaaau);
code.push_back(EncodeVop1(0x52, 10, 249));
code.push_back(EncodeVop1Sdwa(0, 4, 2, 4));
code.push_back(EncodeVop1(0x52, 11, 249));
code.push_back(EncodeVop1Sdwa(0, 5, 2, 5));
code.push_back(EncodeVop1(0x52, 12, 249));
code.push_back(EncodeVop1Sdwa(1, 4, 2, 4));
code.push_back(EncodeVop1(0x52, 13, 249));
code.push_back(EncodeVop1Sdwa(1, 4, 2, 5));
code.push_back(EncodeVop1(0x52, 14, 249));
code.push_back(EncodeVop1Sdwa(2, 4, 2, 4));
for (u32 i = 0; i < 5; i++) {
AppendStoreVgpr(&code, 10 + i, i);
}
AppendEnd(&code);
return {"VectorCvtU16F16Sdwa",
code,
{},
{0x12340003u, 0x00054321u, 0xabcdffffu, 0x77770000u, 0x55550000u},
{O::V_MOV_B32, O::V_CVT_U16_F16, O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}
TestCase NativeAndSdwa16BitDestinationWrites() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovLiteral(&code, 0, 0x3fc00000u); // 1.5f -> 0x3e00h
AppendVMovLiteral(&code, 1, 0x00004200u); // low=3.0h
AppendVMovLiteral(&code, 2, 0x0000c000u); // low=-2.0h
AppendVMovLiteral(&code, 3, 0x00003c00u); // low=1.0h
AppendVMovLiteral(&code, 4, 0x00004000u); // low=2.0h
const u32 sentinels[] = {
0x3c001234u, 0xbbbb2345u, 0xcccc3456u, 0xdddd4567u, 0xeeee5678u,
0xffff6789u, 0xf00d789au, 0x11118888u, 0x22229999u, 0x3333aaaau,
0xaaaa0000u, 0xbbbb0000u, 0xcccc0000u, 0xabcd1234u, 0xbcde2345u,
0xcdef3456u, 0xdef01234u, 0xef012345u, 0xf0123456u, 0x01234567u,
0xaaaa1234u, 0xbbbb5678u, 0x24681234u, 0x13572468u, 0x89abcdefu,
0xfedcba98u, 0x0bad1234u,
};
// AppendStoreVgpr uses v31 as its address scratch register.
const auto output_vgpr = [](u32 index) { return 10u + index + (index >= 21u ? 1u : 0u); };
for (u32 i = 0; i < static_cast<u32>(std::size(sentinels)); i++) {
AppendVMovLiteral(&code, output_vgpr(i), sentinels[i]);
}
code.push_back(EncodeVop1(0x0a, 10, Vgpr(0)));
code.push_back(EncodeVop1(0x0a, 11, 250));
code.push_back(EncodeVop1Dpp(0));
AppendVop3(&code, 0x18a, 12, Vgpr(0), 0);
code.push_back(EncodeVop1(0x52, 13, Vgpr(1)));
code.push_back(EncodeVop1(0x53, 14, Vgpr(2)));
code.push_back(EncodeVop1(0x0a, 15, 249));
code.push_back(EncodeVop1Sdwa(0));
code.push_back(EncodeVop1(0x54, 16, 249));
code.push_back(EncodeVop1Sdwa(10, 4, 2, 5));
code.push_back(EncodeVop1(0x52, 17, 249));
code.push_back(EncodeVop1Sdwa(1, 6, 0, 4));
code.push_back(EncodeVop1(0x52, 18, 249));
code.push_back(EncodeVop1Sdwa(1, 4, 0, 4));
code.push_back(EncodeVop1(0x53, 19, 249));
code.push_back(EncodeVop1Sdwa(2, 4, 1, 4));
code.push_back(EncodeVop2(0x32, 20, 250, 4));
code.push_back(EncodeVop2Dpp(3));
AppendVop3(&code, 0x132, 21, Vgpr(3), Vgpr(4));
code.push_back(EncodeVop2(0x32, 22, 249, 4));
code.push_back(EncodeVop2Sdwa(3, 4, 0, 4, 4));
code.push_back(EncodeVop1(0x0a, 23, 249));
code.push_back(EncodeVop1Sdwa(0, 4, 2, 3));
code.push_back(EncodeVop1(0x0a, 24, 249));
code.push_back(EncodeVop1Sdwa(0, 4, 2, 5));
code.push_back(EncodeVop1(0x0a, 25, 249));
code.push_back(EncodeVop1Sdwa(0, 4, 2, 6, 0, 1, 1));
code.push_back(EncodeVop1(0x0a, 26, 249));
code.push_back(EncodeVop1Sdwa(0, 4, 2, 6, 0, 0, 0, 0, 0, 1));
code.push_back(EncodeVop1(0x0a, 27, 249));
code.push_back(EncodeVop1Sdwa(0, 4, 2, 6, 0, 0, 0, 0, 1));
AppendVop3(&code, 0x18a, 28, Vgpr(0), 0, 0, 0, 0, false, 1);
AppendVop3(&code, 0x18a, 29, Vgpr(0), 0, 0, 0, 0, true);
AppendVop3(&code, 0x34b, 30, Vgpr(3), Vgpr(4), Vgpr(3));
AppendVop3(&code, 0x34b, 32, Vgpr(3), Vgpr(4), Vgpr(3), 0, 0x8);
code.push_back(EncodeVop1(0x52, 33, 249));
code.push_back(EncodeVop1Sdwa(1, 4, 2, 4, 0, 1, 1, 0, 1));
AppendVop3(&code, 0x1d3, 34, Vgpr(1), 0, 0, 1, 0, true, 0, 1);
code.push_back(EncodeVop1(0x07, 35, 249));
code.push_back(EncodeVop1Sdwa(0, 6, 0, 6, 0, 1, 1, 0, 1));
AppendVop3(&code, 0x188, 36, Vgpr(0), 0, 0, 1, 0, true, 0, 1);
code.push_back(EncodeVop1(0x54, 37, 249));
code.push_back(EncodeVop1Sdwa(4, 4, 2, 4, 0, 1, 1));
for (u32 i = 0; i < static_cast<u32>(std::size(sentinels)); i++) {
AppendStoreVgpr(&code, output_vgpr(i), i);
}
AppendEnd(&code);
return {"NativeAndSdwa16BitDestinationWrites",
code,
{},
{0x3c003e00u, 0xbbbb3e00u, 0xcccc3e00u, 0xdddd0003u,
0xeeeefffeu, 0x00003e00u, 0xf00d3c00u, 0x00000003u,
0x00000003u, 0xfffffffeu, 0xaaaa4200u, 0xbbbb4200u,
0x00004200u, 0xabcd0000u, 0xbcde0000u, 0xcdefbe00u,
0xdef04200u, 0xef013c00u, 0xf0124200u, 0x01233c00u,
0xaaaa4200u, 0x42005678u, 0x24680000u, 0x1357fffdu,
0x00000000u, 0xffffffffu, 0x0badb800u},
{O::V_MOV_B32, O::V_CVT_F16_F32, O::V_CVT_U32_F32,
O::V_CVT_I32_F32, O::V_CVT_U16_F16, O::V_CVT_I16_F16,
O::V_RCP_F16, O::V_ADD_F16, O::V_FMA_F16, O::BUFFER_STORE_DWORD,
O::S_ENDPGM}};
}
TestCase VectorMinMaxMed3F16Ops() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovLiteral(&code, 0, 0x40003c00u); // low=1.0h, high=2.0h
AppendVMovLiteral(&code, 1, 0x44004200u); // low=3.0h, high=4.0h
AppendVMovLiteral(&code, 2, 0xc8004000u); // low=2.0h, high=-8.0h
AppendVMovLiteral(&code, 3, 0x00007e00u); // low=qNaN
AppendVMovLiteral(&code, 4, 0x00004000u); // low=2.0h
AppendVMovLiteral(&code, 5, 0x00003c00u); // low=1.0h
AppendVMovLiteral(&code, 10, 0xaaaa5555u);
AppendVMovLiteral(&code, 11, 0x12345678u);
AppendVMovLiteral(&code, 12, 0x77772222u);
AppendVop3(&code, 0x351, 10, Vgpr(0), Vgpr(1), Vgpr(2));
AppendVop3(&code, 0x354, 11, Vgpr(0), Vgpr(1), Vgpr(2), 0, 0xf, false, 0,
0x4);
AppendVop3(&code, 0x357, 12, Vgpr(3), Vgpr(4), Vgpr(5));
for (u32 i = 0; i < 3; i++) {
AppendStoreVgpr(&code, 10 + i, i);
}
AppendEnd(&code);
return {"VectorMinMaxMed3F16Ops",
code,
{},
{0xaaaa3c00u, 0x48005678u, 0x77773c00u},
{O::V_MOV_B32, O::V_MIN3_F16, O::V_MAX3_F16, O::V_MED3_F16,
O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}
TestCase VectorSpecialF16Ops() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovLiteral(&code, 0, 0xc000fc00u); // low=-inf, high=-2.0h
AppendVMovLiteral(&code, 1, 0x00004400u); // low=+4.0h
AppendVMovLiteral(&code, 2, 0x00003c00u); // low=+1.0h
AppendVMovLiteral(&code, 3, 0x0000bc00u); // low=-1.0h
AppendVMovLiteral(&code, 4, 0x0000fc00u); // low=-inf
AppendVMovLiteral(&code, 11, 0x12345678u);
code.push_back(EncodeVop1(0x54, 10, Vgpr(0)));
code.push_back(EncodeVop1(0x54, 11, 249));
code.push_back(EncodeVop1Sdwa(0, 5, 2, 5));
code.push_back(EncodeVop1(0x56, 12, Vgpr(1)));
code.push_back(EncodeVop1(0x57, 13, Vgpr(2)));
code.push_back(EncodeVop1(0x57, 14, Vgpr(3)));
code.push_back(EncodeVop1(0x58, 15, Vgpr(4)));
AppendVop3(&code, 0x1d7, 16, Vgpr(3), 0, 0, 0, 0, true);
for (u32 i = 0; i < 7; i++) {
AppendStoreVgpr(&code, 10 + i, i);
}
AppendEnd(&code);
return {"VectorSpecialF16Ops",
code,
{},
{0x00008000u, 0xb8005678u, 0x00003800u, 0x00000000u, 0x0000fe00u,
0x00000000u, 0x00000000u},
{O::V_MOV_B32, O::V_RCP_F16, O::V_RSQ_F16, O::V_LOG_F16, O::V_EXP_F16,
O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}
TestCase VectorFractF16CapturedAndEdges() {
using O = ShaderOpcode;
// RDNA2 section 12.8: x - floor(x), rounded to half precision.
// Load at runtime to exercise the emitted operation, including half subnormals.
const std::array<u32, 18> inputs{
0x3d00u, 0xbd00u, 0x0000u, 0x8000u, 0x0001u, 0x8001u,
0x03ffu, 0x83ffu, 0x7bffu, 0xfbffu, 0x3800u, 0xb800u,
0x8c00u, 0x8c01u, 0x7c00u, 0xfc00u, 0x7e55u, 0x7d01u};
const std::array<u32, 18> fractions{
0x3400u, 0x3a00u, 0x0000u, 0x0000u, 0x0001u, 0x3c00u,
0x03ffu, 0x3c00u, 0x0000u, 0x0000u, 0x3800u, 0x3800u,
0x3c00u, 0x3bffu, 0x7e00u, 0x7e00u, 0x7e00u, 0x7e00u};
TestCase test;
test.name = "VectorFractF16CapturedAndEdges";
for (u32 bits : inputs) {
test.initial.push_back(0x42000000u | bits); // Distinct high half must be ignored.
}
test.expected = test.initial;
auto &code = test.code;
for (u32 i = 0; i < inputs.size(); i++) {
AppendVMovU32(&code, 30, i * 4u);
AppendBufferLoadDword(&code, 7, 30);
AppendVMovLiteral(&code, 9, 0xa5a51234u);
code.push_back(0x7e12bf07u); // Captured v_fract_f16 v9, v7.
if (i >= 14u) {
// Check quiet NaN class without constraining its sign or payload.
code.push_back(EncodeVop2(0x1b, 9, 255u, 9));
code.push_back(0xffff7e00u);
}
AppendStoreVgpr(&code, 9, static_cast<u32>(inputs.size()) + i);
test.expected.push_back(0xa5a50000u | fractions[i]);
}
AppendEnd(&code);
test.opcodes = {O::V_MOV_B32, O::BUFFER_LOAD_DWORD, O::V_FRACT_F16,
O::V_AND_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.decoded_counts = {{"V_FRACT_F16 v9, v7", inputs.size()}};
test.ir_counts = {{" = FPFract32 ", inputs.size()}};
test.required_spirv = {" Fract ", " PackHalf2x16 ", " UnpackHalf2x16 "};
test.forbidden_spirv = {"OpCapability Float16"};
return test;
}
TestCase VectorFractF16Modifiers() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovLiteral(&code, 7, 0xbd003d00u); // high=-1.25h, low=+1.25h
for (u32 dst = 10; dst <= 19; dst++) {
AppendVMovLiteral(&code, dst, 0xabcd1234u);
}
code.push_back(EncodeVop1(0x5f, 10, 249));
code.push_back(EncodeVop1Sdwa(7, 5, 2, 5)); // high -> high, preserve low
code.push_back(EncodeVop1(0x5f, 11, 249));
code.push_back(EncodeVop1Sdwa(7, 4, 0, 5, 0, 1, 1)); // -abs(-1.25), pad high
code.push_back(EncodeVop1(0x5f, 12, 249));
code.push_back(EncodeVop1Sdwa(7, 4, 2, 5, 0, 0, 1)); // abs(-1.25)
code.push_back(EncodeVop1(0x5f, 13, 249));
code.push_back(EncodeVop1Sdwa(7, 4, 2, 5, 0, 0, 0, 0, 1, 1)); // *2, clamp
code.push_back(EncodeVop1(0x5f, 14, 250));
code.push_back(EncodeVop1Dpp(7, 0x0e4) | (1u << 20u)); // DPP identity, negate
AppendVop3(&code, 0x1df, 15, Vgpr(7), 0, 0, 1, 0, true, 1, 1);
code.push_back(EncodeVop1(0x5f, 16, 255));
code.push_back(0x4200bd00u); // literal -1.25h
code.push_back(EncodeVop1(0x5f, 17, 240)); // inline +0.5, interpreted as FP16
AppendSMovLiteral(&code, 20, 0x4200bd00u);
code.push_back(EncodeVop1(0x5f, 18, 20)); // SGPR source
AppendVop3(&code, 0x1df, 19, Vgpr(7), 0, 0, 0, 0, false, 3); // *0.5
for (u32 i = 0; i < 10; i++) {
AppendStoreVgpr(&code, 10 + i, i);
}
AppendEnd(&code);
TestCase test{"VectorFractF16Modifiers", code, {},
{0x3a001234u, 0x00003a00u, 0xabcd3400u, 0xabcd3c00u,
0xabcd3a00u, 0xabcd3c00u, 0xabcd3a00u, 0xabcd3800u,
0xabcd3a00u, 0xabcd3000u},
{O::V_MOV_B32, O::S_MOV_B32, O::V_FRACT_F16,
O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
test.decoded_counts = {{"V_FRACT_F16", 10}};
test.ir_counts = {{" = FPFract32 ", 10}};
test.required_spirv = {" Fract ", " PackHalf2x16 "};
return test;
}
TestCase VectorCosF16CapturedSdwaAndEdges() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovLiteral(&code, 4, 0x34001234u); // high=+0.25h, low sentinel
code.push_back(0x7e08c2f9u);
code.push_back(0x00051504u); // captured v_cos_f16 v4.word1, v4.word1
AppendVMovLiteral(&code, 5, 0x00008000u); // -0.0h
code.push_back(EncodeVop1(0x61, 6, Vgpr(5)));
AppendVMovLiteral(&code, 7, 0x00007bffu); // max finite
code.push_back(EncodeVop1(0x61, 8, Vgpr(7)));
AppendVMovLiteral(&code, 9, 0x00007c00u); // +inf
code.push_back(EncodeVop1(0x61, 10, Vgpr(9)));
AppendVMovLiteral(&code, 11, 0x00000001u); // minimum positive denormal
code.push_back(EncodeVop1(0x61, 12, Vgpr(11)));
AppendVMovLiteral(&code, 13, 0x00003800u); // +0.5h
code.push_back(EncodeVop1(0x61, 14, Vgpr(13)));
for (u32 i = 0; i < 6; i++) {
AppendStoreVgpr(&code, 4 + i * 2, i);
}
AppendEnd(&code);
TestCase test{"VectorCosF16CapturedSdwaAndEdges",
code,
{},
{0x00001234u, 0x00003c00u, 0x00003c00u, 0x0000fe00u,
0x00003c00u, 0x0000bc00u},
{O::V_MOV_B32, O::V_COS_F16, O::BUFFER_STORE_DWORD,
O::S_ENDPGM}};
test.decoded_counts = {{"V_COS_F16", 6},
{"V_COS_F16 v4.sdwa(sel=5,sext=0), "
"v4.sdwa(sel=5,sext=0)",
1}};
test.ir_counts = {{" = FPCos ", 6}};
test.required_spirv = {" Fract ", " Cos ", " PackHalf2x16 "};
test.forbidden_spirv = {"OpCapability Float16"};
return test;
}
TestCase VectorSinF16SdwaAndEdges() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovLiteral(&code, 4, 0x34001234u); // high=+0.25h, low sentinel
code.push_back(0x7e08c0f9u);
code.push_back(0x00051504u); // v_sin_f16 v4.word1, v4.word1
AppendVMovLiteral(&code, 5, 0x00008000u); // -0.0h
code.push_back(EncodeVop1(0x60, 6, Vgpr(5)));
AppendVMovLiteral(&code, 7, 0x00007bffu); // max finite
code.push_back(EncodeVop1(0x60, 8, Vgpr(7)));
AppendVMovLiteral(&code, 9, 0x0000fbffu); // minimum finite
code.push_back(EncodeVop1(0x60, 10, Vgpr(9)));
AppendVMovLiteral(&code, 11, 0x00007c00u); // +inf
code.push_back(EncodeVop1(0x60, 12, Vgpr(11)));
AppendVMovLiteral(&code, 13, 0x0000fc00u); // -inf
code.push_back(EncodeVop1(0x60, 14, Vgpr(13)));
AppendVMovLiteral(&code, 15, 0x00003800u); // +0.5h
code.push_back(EncodeVop1(0x60, 16, Vgpr(15)));
AppendVMovLiteral(&code, 17, 0x00003a00u); // +0.75h
code.push_back(EncodeVop1(0x60, 18, Vgpr(17)));
AppendVMovLiteral(&code, 19, 0x00003400u); // +0.25h
AppendVop3(&code, 0x1e0, 20, Vgpr(19), 0);
for (u32 i = 0; i < 9; i++) {
AppendStoreVgpr(&code, 4 + i * 2, i);
}
AppendEnd(&code);
TestCase test{"VectorSinF16SdwaAndEdges",
code,
{},
{0x3c001234u, 0x00008000u, 0x00000000u, 0x00000000u,
0x0000fe00u, 0x0000fe00u, 0x00000000u, 0x0000bc00u,
0x00003c00u},
{O::V_MOV_B32, O::V_SIN_F16, O::BUFFER_STORE_DWORD,
O::S_ENDPGM}};
test.decoded_counts = {{"V_SIN_F16", 9},
{"V_SIN_F16 v4.sdwa(sel=5,sext=0), "
"v4.sdwa(sel=5,sext=0)",
1}};
test.ir_counts = {{" = FPSin ", 9}};
test.required_spirv = {" Fract ", " Sin ", " PackHalf2x16 "};
test.forbidden_spirv = {"OpCapability Float16"};
return test;
}
TestCase VectorWritelaneIgnoresExecMask() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovLiteral(&code, 2, 0xaaaaaaaau);
AppendSMovLiteral(&code, 4, 0x12345678u);
code.push_back(EncodeSMovB32(126, InlineU32(1)));
code.push_back(EncodeSMovB32(127, InlineU32(0)));
AppendVop3(&code, 0x361, 2, 4, InlineU32(1));
code.push_back(EncodeSMovB32(126, InlineU32(0xf)));
code.push_back(EncodeSMovB32(127, InlineU32(0)));
code.push_back(EncodeVop2(0x1a, 3, InlineU32(2), 0));
AppendBufferStoreDword(&code, 2, 3);
AppendEnd(&code);
TestCase test;
test.name = "VectorWritelaneIgnoresExecMask";
test.code = code;
test.expected = {0xaaaaaaaau, 0x12345678u, 0xaaaaaaaau, 0xaaaaaaaau};
test.opcodes = {O::V_MOV_B32, O::S_MOV_B32, O::V_WRITELANE_B32,
O::V_LSHLREV_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.compute_info.threads_num[0] = 4;
test.compute_info.threads_num[1] = 1;
test.compute_info.threads_num[2] = 1;
test.compute_info.thread_ids_num = 1;
test.has_compute_info = true;
return test;
}
TestCase VectorReadlaneFromInactiveWrittenLane() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendSMovLiteral(&code, 4, 0x12345678u);
AppendVop3(&code, 0x361, 2, 4, InlineU32(4));
AppendSMovLiteral(&code, 4, 0xdeadbeefu);
AppendVop3(&code, 0x360, 5, Vgpr(2), InlineU32(4));
AppendStoreSgpr(&code, 5, 0);
AppendEnd(&code);
TestCase test;
test.name = "VectorReadlaneInactiveWrittenLane";
test.code = code;
test.expected = {0x12345678u};
test.opcodes = {O::S_MOV_B32, O::V_WRITELANE_B32, O::V_READLANE_B32,
O::V_MOV_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.forbidden_spirv = {"OpGroupNonUniformShuffle"};
test.compute_info.threads_num[0] = 4;
test.compute_info.threads_num[1] = 1;
test.compute_info.threads_num[2] = 1;
test.compute_info.thread_ids_num = 1;
test.has_compute_info = true;
return test;
}
TestCase VectorLaneWave32RuntimeSelectorWraps() {
using O = ShaderOpcode;
std::vector<u32> code;
code.push_back(EncodeSop2(0x00, 5, 4, InlineU32(33)));
AppendSMovLiteral(&code, 6, 0x12345678u);
AppendVMovU32(&code, 2, 0);
AppendVop3(&code, 0x361, 2, 6, 5);
AppendVop3(&code, 0x360, 7, Vgpr(2), InlineU32(1));
AppendStoreSgpr(&code, 7, 0);
AppendEnd(&code);
TestCase test;
test.name = "VectorLaneWave32RuntimeSelectorWraps";
test.code = code;
test.expected = {0x12345678u};
test.opcodes = {O::S_ADD_U32, O::S_MOV_B32, O::V_WRITELANE_B32,
O::V_READLANE_B32, O::V_MOV_B32, O::BUFFER_STORE_DWORD,
O::S_ENDPGM};
test.required_spirv = {"OpGroupNonUniformShuffle"};
test.compute_info.threads_num[0] = 4;
test.compute_info.threads_num[1] = 1;
test.compute_info.threads_num[2] = 1;
test.compute_info.group_id[0] = true;
test.compute_info.wave_size = 32;
test.compute_info.thread_ids_num = 1;
test.compute_info.workgroup_register = 4;
test.has_compute_info = true;
return test;
}
TestCase VectorReadlaneSelectsTwoKeysWithinWave() {
using O = ShaderOpcode;
std::vector<u32> code;
code.push_back(EncodeVop2(0x16, 1, InlineU32(4), 0)); // lanes 0..15: 0, 16..31: 1
AppendVop3(&code, 0x360, 20, Vgpr(1), InlineU32(0));
AppendVop3(&code, 0x360, 21, Vgpr(1), InlineU32(16));
AppendVMovU32(&code, 2, 0);
AppendVMovU32(&code, 3, 11);
AppendVMovU32(&code, 4, 22);
code.push_back(EncodeVopc(0xc2, 20, 1)); // selected lane 0 key == local key
code.push_back(EncodeVop2(0x01, 2, Vgpr(2), 3));
code.push_back(EncodeVopc(0xc2, 21, 1)); // selected lane 16 key == local key
code.push_back(EncodeVop2(0x01, 2, Vgpr(2), 4));
AppendStoreVgprAtLaneDwordOffset(&code, 2, 0, 0);
AppendEnd(&code);
TestCase test;
test.name = "VectorReadlaneSelectsTwoKeysWithinWave";
test.code = std::move(code);
test.expected.resize(32);
std::fill_n(test.expected.begin(), 16, 11u);
std::fill_n(test.expected.begin() + 16, 16, 22u);
test.opcodes = {O::V_LSHRREV_B32, O::V_READLANE_B32, O::V_MOV_B32,
O::V_CMP_EQ_U32, O::V_CNDMASK_B32,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.decoded_counts = {{"V_READLANE_B32", 2}};
test.required_spirv = {"OpGroupNonUniformShuffle"};
test.compute_info.threads_num[0] = 32;
test.compute_info.threads_num[1] = 1;
test.compute_info.threads_num[2] = 1;
test.compute_info.wave_size = 32;
test.compute_info.thread_ids_num = 1;
test.has_compute_info = true;
return test;
}
TestCase VectorPermlanex16() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovLiteral(&code, 3, 0x76543210u);
AppendVMovLiteral(&code, 4, 0xfedcba98u);
code.push_back(EncodeVop2(0x1a, 2, InlineU32(2), 0));
AppendVMovLiteral(&code, 5, 0xfeedbabeu);
AppendVop3(&code, 0x378, 1, Vgpr(5), Vgpr(3), Vgpr(4));
AppendBufferStoreDword(&code, 1, 2);
AppendEnd(&code);
TestCase test;
test.name = "VectorPermlanex16";
test.code = code;
test.expected = std::vector<u32>(32, 0xfeedbabeu);
test.opcodes = {O::V_MOV_B32, O::V_PERMLANEX16_B32, O::V_LSHLREV_B32,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.compute_info.threads_num[0] = 32;
test.compute_info.threads_num[1] = 1;
test.compute_info.threads_num[2] = 1;
test.compute_info.thread_ids_num = 1;
test.has_compute_info = true;
return test;
}
TestCase VectorPermlane16FetchInactiveZero() {
using O = ShaderOpcode;
std::vector<u32> code = {
EncodeVop1(0x01, 1, 0),
EncodeVop2(0x25, 1, InlineU32(10), 1),
EncodeVopc(0xc2, InlineU32(0), 0),
EncodeSop1(0x04, 126, 106),
EncodeSMovB32(0, InlineU32(1)),
EncodeSMovB32(1, InlineU32(0)),
};
AppendVop3(&code, 0x377, 2, Vgpr(1), 0, 1);
AppendStoreVgpr(&code, 2, 0);
AppendEnd(&code);
TestCase test;
test.name = "VectorPermlane16FetchInactiveZero";
test.code = code;
test.expected = {0};
test.opcodes = {O::V_MOV_B32, O::V_ADD_NC_U32, O::V_CMP_EQ_U32,
O::S_MOV_B64, O::S_MOV_B32, O::V_PERMLANE16_B32,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.compute_info.threads_num[0] = 4;
test.compute_info.threads_num[1] = 1;
test.compute_info.threads_num[2] = 1;
test.compute_info.thread_ids_num = 1;
test.has_compute_info = true;
return test;
}
TestCase VectorPermlane16FetchInactiveFi() {
using O = ShaderOpcode;
std::vector<u32> code = {
EncodeVop1(0x01, 1, Vgpr(0)), EncodeVop2(0x25, 1, InlineU32(10), 1),
EncodeVopc(0xc2, InlineU32(0), 0), EncodeSop1(0x04, 126, 106),
EncodeSMovB32(0, InlineU32(1)), EncodeSMovB32(1, InlineU32(0)),
};
AppendVop3(&code, 0x377, 2, Vgpr(1), 0, 1, 0, 1);
AppendStoreVgpr(&code, 2, 0);
AppendEnd(&code);
TestCase test;
test.name = "VectorPermlane16FetchInactiveFi";
test.code = code;
test.expected = {11};
test.opcodes = {O::V_MOV_B32, O::V_ADD_NC_U32, O::V_CMP_EQ_U32,
O::S_MOV_B64, O::S_MOV_B32, O::V_PERMLANE16_B32,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.compute_info.threads_num[0] = 4;
test.compute_info.threads_num[1] = 1;
test.compute_info.threads_num[2] = 1;
test.compute_info.thread_ids_num = 1;
test.has_compute_info = true;
return test;
}
TestCase VectorDpp8Captured(bool masked_exec, u32 wave_size = 64, bool fi = false) {
using O = ShaderOpcode;
constexpr u32 sentinel = 0xaaaaaaaau;
// Both wave halves contain active and inactive lane-3 sources for the ADD.
const u32 masks[] = {masked_exec ? 0x0f0ff0f0u : 0xffffffffu,
masked_exec ? 0xf0f00f0fu : 0xffffffffu};
std::vector<u32> code;
AppendVMovU32(&code, 5, 100);
code.push_back(EncodeVop2(0x25, 5, Vgpr(0), 5));
code.push_back(EncodeVop1(0x01, 7, Vgpr(5)));
AppendVMovLiteral(&code, 2, sentinel);
AppendVMovLiteral(&code, 3, sentinel);
AppendVMovLiteral(&code, 8, sentinel);
AppendSMovLiteral(&code, 126, masks[0]);
if (wave_size == 64) AppendSMovLiteral(&code, 127, masks[1]);
// Captured DPP8 selectors broadcast each group's upper/lower four lanes.
const u32 encoding = fi ? 234u : 233u;
code.push_back(EncodeVop1(0x01, 2, encoding));
code.push_back(0xfacfac05u);
code.push_back(EncodeVop1(0x01, 3, encoding));
code.push_back(0x68868805u);
// CS 6babb2a8915d35c4, pc 0xcc: add lane 3 of each group to its own v7.
code.push_back(EncodeVop2(0x25, 8, encoding, 7));
code.push_back(0x6db6db07u);
AppendSMovLiteral(&code, 126, 0xffffffffu);
if (wave_size == 64) AppendSMovLiteral(&code, 127, 0xffffffffu);
code.push_back(EncodeVop2(0x1a, 6, InlineU32(2), 0));
AppendBufferStoreDword(&code, 2, 6);
AppendVMovU32(&code, 7, wave_size * sizeof(u32));
code.push_back(EncodeVop2(0x25, 6, Vgpr(7), 6));
AppendBufferStoreDword(&code, 3, 6);
code.push_back(EncodeVop2(0x25, 6, Vgpr(7), 6));
AppendBufferStoreDword(&code, 8, 6);
AppendEnd(&code);
TestCase test;
const char *names[2][2] = {
{"VectorDpp8Wave32MaskedExec", "VectorDpp8Wave64MaskedExec"},
{"VectorDpp8FiWave32MaskedExec", "VectorDpp8FiWave64MaskedExec"}};
test.name = masked_exec ? names[fi][wave_size == 64] : "VectorDpp8Captured";
test.code = std::move(code);
test.expected.resize(3 * wave_size, sentinel);
for (u32 lane = 0; lane < wave_size; ++lane) {
const auto mask = masks[lane / 32u];
if ((mask & (1u << (lane % 32u))) == 0) {
continue;
}
const u32 sources[] = {(lane & ~7u) | 4u | (lane & 3u),
(lane & ~7u) | (lane & 3u)};
for (u32 permutation = 0; permutation < 2; ++permutation) {
const auto source = sources[permutation];
test.expected[permutation * wave_size + lane] =
fi || (mask & (1u << (source % 32u))) != 0 ? 100u + source : 0u;
}
const u32 source = (lane & ~7u) | 3u;
test.expected[2 * wave_size + lane] = 100u + lane;
if (fi || (mask & (1u << (source % 32u))) != 0) {
test.expected[2 * wave_size + lane] += 100u + source;
}
}
test.opcodes = {O::V_MOV_B32, O::V_ADD_NC_U32, O::S_MOV_B32,
O::V_LSHLREV_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.decoded_counts = {{".dpp8(", 3}};
test.compute_info.wave_size = wave_size;
test.compute_info.threads_num[0] = wave_size;
test.compute_info.threads_num[1] = 1;
test.compute_info.threads_num[2] = 1;
test.compute_info.thread_ids_num = 1;
test.has_compute_info = true;
return test;
}
TestCase VectorDppQuadPermuteReverse() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 1, 100);
code.push_back(EncodeVop2(0x25, 2, 250, 1));
code.push_back(EncodeVop2Dpp(0, 0x01b));
code.push_back(EncodeVop2(0x1a, 3, InlineU32(2), 0));
AppendBufferStoreDword(&code, 2, 3);
AppendEnd(&code);
TestCase test;
test.name = "VectorDppQuadPermuteReverse";
test.code = code;
test.expected = {103, 102, 101, 100, 107, 106, 105, 104};
test.opcodes = {O::V_MOV_B32, O::V_ADD_NC_U32, O::V_LSHLREV_B32,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.compute_info.threads_num[0] = 8;
test.compute_info.threads_num[1] = 1;
test.compute_info.threads_num[2] = 1;
test.compute_info.thread_ids_num = 1;
test.has_compute_info = true;
return test;
}
TestCase VectorDppRowXmask() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 1, 100);
code.push_back(EncodeVop2(0x25, 2, 250, 1));
code.push_back(EncodeVop2Dpp(0, 0x164));
code.push_back(EncodeVop2(0x1a, 3, InlineU32(2), 0));
AppendBufferStoreDword(&code, 2, 3);
AppendEnd(&code);
TestCase test;
test.name = "VectorDppRowXmask";
test.code = code;
test.expected = {104, 105, 106, 107, 100, 101, 102, 103};
test.opcodes = {O::V_MOV_B32, O::V_ADD_NC_U32, O::V_LSHLREV_B32,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.compute_info.threads_num[0] = 8;
test.compute_info.threads_num[1] = 1;
test.compute_info.threads_num[2] = 1;
test.compute_info.thread_ids_num = 1;
test.has_compute_info = true;
return test;
}
TestCase VectorDppRowShare(u32 wave_size) {
using O = ShaderOpcode;
constexpr u32 sentinel = 0xaaaaaaaau;
TestCase test;
test.name = wave_size == 32 ? "VectorDppRowShareWave32" : "VectorDppRowShareWave64";
auto &code = test.code;
AppendVMovU32(&code, 1, 100);
code.push_back(EncodeVop2(0x25, 1, Vgpr(0), 1)); // v1 = 100 + lane.
code.push_back(EncodeSop1(0x04, 10, 126));
for (u32 selector = 0; selector < 16; ++selector) {
for (const bool masked : {false, true}) {
const u32 row_mask = masked ? 0xau : 0xfu;
const u32 bank_mask = masked ? 0x5u : 0xfu;
// Keep the selected source active in every row while masking destination
// lanes. Row and bank masks affect destinations, not the shared source.
const u32 source_lanes = 0x00010001u << selector;
const u32 exec[] = {masked ? 0xaaaaaaaau | source_lanes : 0xffffffffu,
masked ? 0x55555555u | source_lanes : 0xffffffffu};
AppendVMovLiteral(&code, 2, sentinel);
AppendVMovLiteral(&code, 3, sentinel);
AppendSMovLiteral(&code, 126, exec[0]);
if (wave_size == 64) AppendSMovLiteral(&code, 127, exec[1]);
code.push_back(EncodeVop1(0x01, 2, 250));
code.push_back(EncodeVop1Dpp(1, 0x150 + selector, row_mask, bank_mask));
code.push_back(EncodeVop2(0x25, 3, 250, 0));
code.push_back(EncodeVop2Dpp(1, 0x150 + selector, row_mask, bank_mask));
code.push_back(EncodeSop1(0x04, 126, 10));
for (const u32 reg : {2u, 3u}) {
AppendStoreVgprAtLaneDwordOffset(&code, reg, 0, u32(test.expected.size()));
for (u32 lane = 0; lane < wave_size; ++lane) {
const bool write = ((row_mask >> (lane / 16)) & 1u) &&
((bank_mask >> ((lane / 4) % 4)) & 1u) &&
((exec[lane / 32] >> (lane % 32)) & 1u);
const u32 source = (lane / 16) * 16 + selector;
test.expected.push_back(write ? 100 + source + (reg == 3 ? lane : 0) : sentinel);
}
}
}
}
AppendEnd(&code);
test.opcodes = {O::V_MOV_B32, O::V_ADD_NC_U32, O::S_MOV_B32, O::S_MOV_B64,
O::V_LSHLREV_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.compute_info.wave_size = wave_size;
test.compute_info.threads_num[0] = wave_size;
test.compute_info.threads_num[1] = test.compute_info.threads_num[2] = 1;
test.compute_info.thread_ids_num = 1;
test.has_compute_info = true;
return test;
}
TestCase VectorDppBankMaskPreservesDestination() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovLiteral(&code, 2, 0xaaaaaaaau);
AppendVMovU32(&code, 1, 100);
code.push_back(EncodeVop2(0x25, 2, 250, 1));
code.push_back(EncodeVop2Dpp(0, 0x0e4, 0xf, 0xe));
code.push_back(EncodeVop2(0x1a, 3, InlineU32(2), 0));
AppendBufferStoreDword(&code, 2, 3);
AppendEnd(&code);
TestCase test;
test.name = "VectorDppBankMaskPreservesDestination";
test.code = code;
test.expected = {0xaaaaaaaau, 0xaaaaaaaau, 0xaaaaaaaau, 0xaaaaaaaau,
104, 105, 106, 107};
test.opcodes = {O::V_MOV_B32, O::V_ADD_NC_U32, O::V_LSHLREV_B32,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.compute_info.threads_num[0] = 8;
test.compute_info.threads_num[1] = 1;
test.compute_info.threads_num[2] = 1;
test.compute_info.thread_ids_num = 1;
test.has_compute_info = true;
return test;
}
TestCase VectorDppBoundsControlZeroPreservesDestination() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovLiteral(&code, 2, 0xaaaaaaaau);
AppendVMovU32(&code, 1, 100);
code.push_back(EncodeVop2(0x25, 2, 250, 1));
code.push_back(EncodeVop2Dpp(0, 0x111));
code.push_back(EncodeVop2(0x1a, 3, InlineU32(2), 0));
AppendBufferStoreDword(&code, 2, 3);
AppendEnd(&code);
TestCase test;
test.name = "VectorDppBoundsControlZeroPreservesDestination";
test.code = code;
test.expected = {0xaaaaaaaau, 100, 101, 102, 103, 104, 105, 106};
test.opcodes = {O::V_MOV_B32, O::V_ADD_NC_U32, O::V_LSHLREV_B32,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.compute_info.threads_num[0] = 8;
test.compute_info.threads_num[1] = 1;
test.compute_info.threads_num[2] = 1;
test.compute_info.thread_ids_num = 1;
test.has_compute_info = true;
return test;
}
TestCase VectorMadF32RoundsProduct() {
using O = ShaderOpcode;
std::vector<u32> code;
for (u32 source = 0; source < 3; source++) {
AppendVMovU32(&code, 30, source * sizeof(u32));
AppendBufferLoadDword(&code, source, 30);
}
AppendVMovLiteral(&code, 10, 0xbf800000u);
AppendVMovLiteral(&code, 11, 0xbf800000u);
AppendVMovLiteral(&code, 15, 0xbf800000u);
AppendVMovLiteral(&code, 16, 0xbf800000u);
code.push_back(EncodeVop2(0x1f, 10, Vgpr(0), 1));
AppendVop3(&code, 0x11f, 11, Vgpr(0), Vgpr(1), 0);
AppendVop3(&code, 0x141, 12, Vgpr(0), Vgpr(1), Vgpr(2));
code.push_back(EncodeVop2(0x20, 13, Vgpr(0), 2));
code.push_back(0x3f7ffffeu);
code.push_back(EncodeVop2(0x21, 14, Vgpr(0), 1));
code.push_back(0xbf800000u);
code.push_back(EncodeVop2(0x2b, 15, Vgpr(0), 1));
AppendVop3(&code, 0x12b, 16, Vgpr(0), Vgpr(1), 0);
AppendVop3(&code, 0x14b, 17, Vgpr(0), Vgpr(1), Vgpr(2));
code.push_back(EncodeVop2(0x2c, 18, Vgpr(0), 2));
code.push_back(0x3f7ffffeu);
code.push_back(EncodeVop2(0x2d, 19, Vgpr(0), 1));
code.push_back(0xbf800000u);
for (u32 i = 0; i < 10; i++) {
AppendStoreVgpr(&code, 10 + i, i);
}
AppendEnd(&code);
TestCase test;
test.name = "VectorMadF32RoundsProduct";
test.code = std::move(code);
test.initial = {0x3f800001u, 0x3f7ffffeu, 0xbf800000u};
// (1+2^-23)*(1-2^-23)-1 rounds to zero for MAD, but -2^-46 for FMA.
test.expected = {0u, 0u, 0u, 0u, 0u, 0xa8800000u, 0xa8800000u,
0xa8800000u, 0xa8800000u, 0xa8800000u};
test.opcodes = {O::V_MOV_B32, O::BUFFER_LOAD_DWORD, O::V_MAC_F32,
O::V_MAD_F32, O::V_MADMK_F32, O::V_MADAK_F32,
O::V_FMAC_F32, O::V_FMA_F32, O::V_FMAMK_F32, O::V_FMAAK_F32,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.required_spirv = {"OpFMul", "OpFAdd", "NoContraction", "Fma"};
return test;
}
TestCase VectorMadF32FlushesDenorms() {
using O = ShaderOpcode;
std::vector<u32> code;
for (u32 sample = 0; sample < 5; sample++) {
for (u32 source = 0; source < 3; source++) {
AppendVMovU32(&code, 30, (sample * 3 + source) * sizeof(u32));
AppendBufferLoadDword(&code, source, 30);
}
AppendVop3(&code, 0x141, 10 + sample, Vgpr(0), Vgpr(1), Vgpr(2));
}
for (u32 sample = 0; sample < 5; sample++) {
AppendStoreVgpr(&code, 10 + sample, sample);
}
AppendEnd(&code);
TestCase test;
test.name = "VectorMadF32FlushesDenorms";
test.code = std::move(code);
// Input denorms in each operand, an intermediate denorm, and a negative result denorm.
test.initial = {0x00400000u, 0x40000000u, 0u,
0x40000000u, 0x00400000u, 0u,
0u, 0x3f800000u, 0x00400000u,
0x00800000u, 0x3f000000u, 0x00800000u,
0x00800000u, 0x3f800000u, 0x80800001u};
test.expected = {0u, 0u, 0u, 0x00800000u, 0x80000000u};
test.opcodes = {O::V_MOV_B32, O::BUFFER_LOAD_DWORD, O::V_MAD_F32,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
// Legacy MAD always flushes, even when the shader requests denorm preservation.
test.compute_info.float_mode = 0xf0;
test.has_compute_info = true;
return test;
}
TestCase Vop3FmacF32NegatedSourceAccumulates() {
using O = ShaderOpcode;
std::vector<u32> code;
constexpr u32 registers[] = {13, 20, 29};
for (u32 sample = 0; sample < 2; sample++) {
for (u32 source = 0; source < 3; source++) {
AppendVMovU32(&code, 30, (sample * 3 + source) * sizeof(u32));
AppendBufferLoadDword(&code, registers[source], 30);
}
code.push_back(0xd52b001du);
code.push_back(0x4002290du); // v_fmac_f32_e64 v29, v13, -v20
AppendStoreVgpr(&code, 29, sample);
}
AppendEnd(&code);
TestCase test;
test.name = "Vop3FmacF32NegatedSourceAccumulates";
test.code = std::move(code);
test.initial = {0x40000000u, 0x40400000u, 0x41200000u,
0x3f800001u, 0x3f7ffffeu, 0x3f800000u};
// 10 - 2*3 = 4; 1 - (1+2^-23)*(1-2^-23) = 2^-46, not rounded zero.
test.expected = {0x40800000u, 0x28800000u};
test.opcodes = {O::V_MOV_B32, O::BUFFER_LOAD_DWORD, O::V_FMAC_F32,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.required_spirv = {"OpFNegate", "Fma"};
return test;
}
TestCase Vop3LdexpSourceModifier() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovLiteral(&code, 9, 0xbfc00000u);
code.push_back(0xd7620107u);
code.push_back(0x00018509u);
AppendStoreVgpr(&code, 7, 0);
AppendEnd(&code);
return {"Vop3LdexpSourceModifier",
code,
{},
{0x3ec00000u},
{O::V_MOV_B32, O::V_LDEXP_F32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}
TestCase Vop1MoveRelSource() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovLiteral(&code, 12, 0x11111111u);
AppendVMovLiteral(&code, 13, 0x22222222u);
AppendVMovLiteral(&code, 14, 0x33333333u);
code.push_back(EncodeSMovB32(124, InlineU32(2)));
code.push_back(0x7e6e870cu);
AppendStoreVgpr(&code, 55, 0);
AppendEnd(&code);
return {"Vop1MoveRelSource",
code,
{},
{0x33333333u},
{O::V_MOV_B32, O::S_MOV_B32, O::V_MOVRELS_B32, O::BUFFER_STORE_DWORD,
O::S_ENDPGM}};
}
TestCase Vop1MoveRelDestination() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovLiteral(&code, 1, 0x12345678u);
AppendVMovLiteral(&code, 5, 0xaaaaaaaau);
AppendVMovLiteral(&code, 6, 0xbbbbbbbbu);
AppendVMovLiteral(&code, 7, 0xccccccccu);
code.push_back(EncodeSMovB32(124, InlineU32(2)));
code.push_back(EncodeSop1(0x04, 8, 126));
code.push_back(EncodeVopc(0xc2, InlineU32(0), 0));
code.push_back(EncodeSop1(0x04, 126, 106));
code.push_back(EncodeVop1(0x42, 5, Vgpr(1)));
code.push_back(EncodeSop1(0x04, 126, 8));
code.push_back(EncodeVop2(0x1a, 4, InlineU32(2), 0));
AppendBufferStoreDword(&code, 7, 4);
AppendEnd(&code);
TestCase test;
test.name = "Vop1MoveRelDestination";
test.code = code;
test.expected = {0x12345678u, 0xccccccccu, 0xccccccccu, 0xccccccccu};
test.opcodes = {O::V_MOV_B32, O::S_MOV_B32, O::V_CMP_EQ_U32,
O::S_MOV_B64, O::V_MOVRELD_B32, O::V_LSHLREV_B32,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.compute_info.threads_num[0] = 4;
test.compute_info.threads_num[1] = 1;
test.compute_info.threads_num[2] = 1;
test.compute_info.thread_ids_num = 1;
test.has_compute_info = true;
return test;
}
TestCase CubeIdCapturedNegationAndOutputScale() {
using O = ShaderOpcode;
constexpr std::array directions = {
std::array{4.0f, 2.0f, 1.0f}, std::array{-4.0f, 2.0f, 1.0f},
std::array{1.0f, -4.0f, 2.0f}, std::array{1.0f, 4.0f, 2.0f},
std::array{1.0f, 2.0f, -4.0f}, std::array{1.0f, 2.0f, 4.0f}};
constexpr std::array registers = {47u, 49u, 48u};
TestCase test;
test.name = "CubeIdCapturedNegationAndOutputScale";
for (u32 i = 0; i < directions.size(); i++) {
for (u32 component = 0; component < registers.size(); component++) {
test.initial.push_back(std::bit_cast<u32>(directions[i][component]));
AppendVMovU32(&test.code, 30, (i * 3u + component) * 4u);
AppendBufferLoadDword(&test.code, registers[component], 30);
}
// The captured instruction negates Y/Z and divides the floating face ID by two.
test.code.insert(test.code.end(), {0xd544000bu, 0xdcc2632fu});
AppendStoreVgpr(&test.code, 11, i);
}
AppendEnd(&test.code);
test.expected = {0x00000000u, 0x3f000000u, 0x3f800000u,
0x3fc00000u, 0x40000000u, 0x40200000u};
test.opcodes = {O::V_MOV_B32, O::BUFFER_LOAD_DWORD, O::V_CUBEID_F32,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.decoded_counts = {
{"V_CUBEID_F32 v11.omod(3), v47, v49.neg, v48.neg", directions.size()}};
test.required_spirv = {"OpFNegate", "OpFMul"};
return test;
}
TestCase VectorFloatSpecialOps() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovLiteral(&code, 0, 0x3f800000u);
AppendVMovLiteral(&code, 1, 0x40000000u);
AppendVMovLiteral(&code, 2, 0x40400000u);
AppendVMovLiteral(&code, 3, 0x40800000u);
AppendVMovLiteral(&code, 4, 0x40003c00u);
AppendVMovLiteral(&code, 5, 0x44004200u);
AppendVMovLiteral(&code, 6, 0x3f800000u);
AppendVMovLiteral(&code, 7, 0x3f800000u);
AppendVMovLiteral(&code, 8, 0xbf800000u);
AppendVMovLiteral(&code, 10, 0x3f000000u);
AppendVMovLiteral(&code, 12, 0xaaaabbbbu);
AppendVMovLiteral(&code, 13, 0xaaaabbbbu);
code.push_back(EncodeVop2(0x02, 6, Vgpr(4), 5));
AppendVop3(&code, 0x368, 9, Vgpr(7), Vgpr(8));
AppendVop3(&code, 0x369, 11, Vgpr(7), Vgpr(10));
AppendVop3p(&code, 0x21, 12, Vgpr(1), Vgpr(2), Vgpr(0));
AppendVop3p(&code, 0x22, 13, Vgpr(1), Vgpr(2), Vgpr(0));
AppendVop3(&code, 0x144, 14, Vgpr(0), Vgpr(1), Vgpr(2));
AppendVop3(&code, 0x145, 15, Vgpr(0), Vgpr(1), Vgpr(2));
AppendVop3(&code, 0x146, 16, Vgpr(0), Vgpr(1), Vgpr(2));
AppendVop3(&code, 0x147, 17, Vgpr(0), Vgpr(1), Vgpr(2));
const u32 results[] = {6, 9, 11, 12, 13, 14, 15, 16, 17};
for (u32 i = 0; i < static_cast<u32>(std::size(results)); i++) {
AppendStoreVgpr(&code, results[i], i);
}
AppendEnd(&code);
return {"VectorFloatSpecialOps",
code,
{},
{0x41400000u, 0x80017fffu, 0x8000ffffu, 0xaaaa4700u, 0x4700bbbbu,
0x40800000u, 0x3f800000u, 0xc0000000u, 0x40c00000u},
{O::V_MOV_B32, O::V_DOT2C_F32_F16, O::V_CVT_PKNORM_I16_F32,
O::V_CVT_PKNORM_U16_F32, O::V_MAD_MIXLO_F16, O::V_MAD_MIXHI_F16,
O::V_CUBEID_F32, O::V_CUBESC_F32, O::V_CUBETC_F32, O::V_CUBEMA_F32,
O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}
TestCase MadMixF16LiteralHalfSourceUsesOpsel() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovLiteral(&code, 0, 0x3f800000u);
AppendVMovLiteral(&code, 12, 0xaaaabbbbu);
AppendVMovLiteral(&code, 13, 0x11112222u);
AppendVop3p(&code, 0x21, 12, 255u, Vgpr(0), Vgpr(0), 0x1, 0x1);
code.push_back(0x40003c00u);
AppendVop3p(&code, 0x22, 13, 255u, Vgpr(0), Vgpr(0), 0x1, 0x0);
code.push_back(0x40003c00u);
AppendStoreVgpr(&code, 12, 0);
AppendStoreVgpr(&code, 13, 1);
AppendEnd(&code);
return {"MadMixF16LiteralHalfSourceUsesOpsel",
code,
{},
{0xaaaa4200u, 0x40002222u},
{O::V_MOV_B32, O::V_MAD_MIXLO_F16, O::V_MAD_MIXHI_F16,
O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}
TestCase MadMixF16NegHiIsAbsAndNegIsIndependent() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovLiteral(&code, 0, 0x3f800000u);
AppendVMovLiteral(&code, 1, 0xc0000000u);
AppendVMovLiteral(&code, 2, 0x40000000u);
AppendVMovLiteral(&code, 12, 0xaaaabbbbu);
AppendVMovLiteral(&code, 13, 0x11112222u);
AppendVop3p(&code, 0x21, 12, Vgpr(1), Vgpr(0), Vgpr(0), 0, 0, 0x1);
AppendVop3p(&code, 0x22, 13, Vgpr(2), Vgpr(0), Vgpr(0), 0, 0, 0, 0x1);
AppendStoreVgpr(&code, 12, 0);
AppendStoreVgpr(&code, 13, 1);
AppendEnd(&code);
return {"MadMixF16NegHiIsAbsAndNegIsIndependent",
code,
{},
{0xaaaa4200u, 0xbc002222u},
{O::V_MOV_B32, O::V_MAD_MIXLO_F16, O::V_MAD_MIXHI_F16,
O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}
TestCase VectorVop3FmaF16UsesRdna2Opcode34b() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovLiteral(&code, 1, 0x40003c00u);
AppendVMovLiteral(&code, 2, 0x44004200u);
AppendVMovLiteral(&code, 3, 0x3c003c00u);
AppendVMovLiteral(&code, 10, 0xaaaa5555u);
AppendVMovLiteral(&code, 11, 0xbbbb5555u);
AppendVop3(&code, 0x34bu, 10, Vgpr(1), Vgpr(2), Vgpr(3));
AppendVop3(&code, 0x34bu, 11, Vgpr(1), Vgpr(2), Vgpr(3), 0, 0xfu);
AppendStoreVgpr(&code, 10, 0);
AppendStoreVgpr(&code, 11, 1);
AppendEnd(&code);
return {"VectorVop3FmaF16UsesRdna2Opcode34b",
code,
{},
{0xaaaa4400u, 0x48805555u},
{O::V_MOV_B32, O::V_FMA_F16, O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}
TestCase FloatInlineConstantF16UsesNumericValue() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovLiteral(&code, 1, 0x38003800u); // 0.5h, 0.5h
AppendVMovLiteral(&code, 2, 0x3c003c00u); // 1.0h, 1.0h
AppendVMovLiteral(&code, 10, 0xaaaa0000u);
AppendVMovLiteral(&code, 12, 0xbbbb0000u);
AppendVMovLiteral(&code, 14, 0xcccc0000u);
AppendVMovLiteral(&code, 15, 0xdddd0000u);
AppendVop3(&code, 0x34bu, 10, 246u, Vgpr(1), Vgpr(2));
AppendVop3p(&code, 0x0eu, 11, 246u, Vgpr(1), Vgpr(2), 0x1u);
AppendVop3(&code, 0x34bu, 12, 248u, 242u, 128u);
AppendVop3(&code, 0x311u, 13, 242u, 248u);
AppendVop3(&code, 0x34bu, 14, 246u, Vgpr(1), Vgpr(2), 0, 1);
AppendVop3(&code, 0x303u, 15, 240u, InlineU32(1));
AppendStoreVgpr(&code, 10, 0);
AppendStoreVgpr(&code, 11, 1);
AppendStoreVgpr(&code, 12, 2);
AppendStoreVgpr(&code, 13, 3);
AppendStoreVgpr(&code, 14, 4);
AppendStoreVgpr(&code, 15, 5);
AppendEnd(&code);
TestCase test;
test.name = "FloatInlineConstantF16UsesNumericValue";
test.code = std::move(code);
test.expected = {0xaaaa4200u, 0x3c004200u, 0xbbbb3118u,
0x31183c00u, 0xcccc3c00u, 0xdddd3801u};
test.opcodes = {O::V_MOV_B32, O::V_FMA_F16, O::V_PK_FMA_F16,
O::V_PACK_B32_F16, O::V_ADD_NC_U16,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.decoded_counts = {{"4.000000", 3}, {"0.159155", 2}};
return test;
}
TestCase Vop3Int16Ops() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovLiteral(&code, 1, 0xabcd0001u);
AppendVMovLiteral(&code, 2, 0x12340002u);
AppendVMovLiteral(&code, 3, 0x55557fffu);
AppendVMovLiteral(&code, 4, 0x66668000u);
AppendVMovLiteral(&code, 5, 0x77770013u);
AppendVMovLiteral(&code, 6, 0x88888010u);
AppendVMovLiteral(&code, 10, 0xaaaa0000u);
AppendVMovLiteral(&code, 11, 0xbbbb0000u);
AppendVMovLiteral(&code, 12, 0xcccc0000u);
AppendVMovLiteral(&code, 13, 0xdddd0000u);
AppendVMovLiteral(&code, 14, 0xeeee0000u);
AppendVMovLiteral(&code, 15, 0x11110000u);
AppendVop3(&code, 0x304u, 10, Vgpr(1), Vgpr(2));
AppendVop3(&code, 0x30du, 11, Vgpr(3), Vgpr(1));
AppendVop3(&code, 0x308u, 12, Vgpr(5), Vgpr(6));
AppendVop3(&code, 0x309u, 13, Vgpr(3), Vgpr(4));
AppendVop3(&code, 0x30bu, 14, Vgpr(3), Vgpr(4));
AppendVop3(&code, 0x308u, 15, Vgpr(5), Vgpr(3));
for (u32 i = 0; i < 6; i++) {
AppendStoreVgpr(&code, 10 + i, i);
}
AppendEnd(&code);
TestCase test;
test.name = "Vop3Int16Ops";
test.code = std::move(code);
test.expected = {0xaaaaffffu, 0xbbbb8000u, 0xccccf002u,
0xdddd8000u, 0xeeee7fffu, 0x11110fffu};
test.opcodes = {O::V_MOV_B32, O::V_SUB_NC_U16, O::V_ADD_NC_I16,
O::V_ASHRREV_I16, O::V_MAX_U16, O::V_MIN_U16,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
return test;
}
TestCase VectorFloatControlContractPreservesInfNan() {
using O = ShaderOpcode;
std::vector<u32> code;
code.push_back(EncodeSop2(0x24, 0, InlineU32(8), InlineU32(23)));
code.push_back(EncodeVop1(0x01, 0, InlineU32(0)));
AppendVMovU32(&code, 1, 1);
AppendVMovU32(&code, 2, 3);
AppendVop3(&code, 0x108, 10, 0, Vgpr(0));
code.push_back(EncodeVopc(0x88, Vgpr(10), 2));
code.push_back(EncodeVop2(0x01, 3, InlineU32(0), 1));
AppendStoreVgpr(&code, 3, 0);
AppendEnd(&code);
TestCase test;
test.name = "VectorFloatControlContractPreservesInfNan";
test.code = std::move(code);
test.expected = {1u};
test.opcodes = {O::S_BFM_B32, O::V_MOV_B32, O::V_MUL_F32,
O::V_CMP_CLASS_F32, O::V_CNDMASK_B32, O::BUFFER_STORE_DWORD,
O::S_ENDPGM};
test.required_spirv = {"SPV_KHR_float_controls",
"SignedZeroInfNanPreserve 32"};
return test;
}
TestCase VectorFloatArithmeticOps() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovLiteral(&code, 0, 0x3f800000u);
AppendVMovLiteral(&code, 1, 0x40000000u);
AppendVMovLiteral(&code, 2, 0x40800000u);
AppendVMovLiteral(&code, 3, 0x40a00000u);
AppendVMovLiteral(&code, 4, 0x40400000u);
AppendVMovLiteral(&code, 11, 0x40800000u);
AppendVMovLiteral(&code, 17, 0x40800000u);
code.push_back(EncodeVop2(0x03, 5, Vgpr(0), 1));
code.push_back(EncodeVop2(0x04, 6, Vgpr(3), 1));
code.push_back(EncodeVop2(0x05, 7, Vgpr(1), 3));
code.push_back(EncodeVop2(0x08, 8, Vgpr(1), 2));
code.push_back(EncodeVop1(0x01, 21, Vgpr(8)));
code.push_back(EncodeVop2(0x0f, 9, Vgpr(3), 1));
code.push_back(EncodeVop2(0x10, 10, Vgpr(3), 1));
code.push_back(EncodeVop2(0x1f, 11, Vgpr(1), 4));
AppendVop3(&code, 0x141, 12, Vgpr(1), Vgpr(4), Vgpr(2));
AppendVop3(&code, 0x14b, 13, Vgpr(1), Vgpr(4), Vgpr(2));
AppendVop3(&code, 0x151, 14, Vgpr(3), Vgpr(1), Vgpr(2));
AppendVop3(&code, 0x154, 15, Vgpr(3), Vgpr(1), Vgpr(2));
AppendVop3(&code, 0x157, 16, Vgpr(3), Vgpr(1), Vgpr(2));
code.push_back(EncodeVop2(0x20, 18, Vgpr(1), 17));
code.push_back(0x3f800000u);
code.push_back(EncodeVop2(0x21, 19, Vgpr(1), 4));
code.push_back(0x40000000u);
code.push_back(EncodeVop2(0x03, 20, 249, 1));
code.push_back(EncodeVop2Sdwa(0));
AppendSMovLiteral(&code, 70, 0x40000000u);
code.push_back(0x06088cf9u);
code.push_back(0x868606f2u);
AppendSMovLiteral(&code, 26, 0x40000000u);
code.push_back(0x081034f9u);
code.push_back(0x868606f2u);
code.push_back(0xd51f8011u);
code.push_back(0x00020d0bu);
AppendVMovLiteral(&code, 22, 0xbf800000u);
code.push_back(EncodeVop2(0x05, 23, 249, 26));
code.push_back(EncodeVop2Sdwa(22, 6, 0, 6, 6, 0, 0, 0, 1, 0, 0, 0, 1));
AppendVMovLiteral(&code, 24, 0x3f800000u);
AppendVMovLiteral(&code, 25, 0x3f800000u);
code.push_back(EncodeVop2(0x03, 26, 249, 25));
code.push_back(EncodeVop2Sdwa(24, 6, 0, 6, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1));
AppendVMovLiteral(&code, 27, 0xbf800000u);
AppendVMovLiteral(&code, 28, 0x40000000u);
AppendVop3(&code, 0x104, 29, Vgpr(27), Vgpr(28), 0, 1);
AppendVop3(&code, 0x103, 30, Vgpr(0), Vgpr(1), 0, 0, 0, false, 1);
AppendVop3(&code, 0x151, 32, Vgpr(0), Vgpr(1), Vgpr(2), 0, 0, false, 1);
AppendVMovLiteral(&code, 33, 0xbf800000u);
AppendVMovLiteral(&code, 34, 0x40000000u);
code.push_back(EncodeVop2(0x03, 35, 250, 34));
code.push_back(EncodeVop2Dpp(33, 0, 0xf, 0xf, 0, 1));
const u32 results[] = {5, 6, 7, 21, 9, 10, 11, 12, 13, 14, 15, 16,
18, 19, 20, 4, 8, 17, 23, 26, 29, 30, 32, 35};
for (u32 i = 0; i < static_cast<u32>(std::size(results)); i++) {
AppendStoreVgpr(&code, results[i], i);
}
AppendEnd(&code);
return {"VectorFloatArithmeticOps",
code,
{},
{0x40400000u, 0x40400000u, 0x40400000u, 0x41000000u, 0x40000000u,
0x40a00000u, 0x41200000u, 0x41200000u, 0x41200000u, 0x40000000u,
0x40a00000u, 0x40800000u, 0x40c00000u, 0x41000000u, 0x40400000u,
0x40400000u, 0xbf800000u, 0x3f800000u, 0x3f800000u, 0x40800000u,
0xbf800000u, 0x40c00000u, 0x40000000u, 0x40400000u},
{O::S_MOV_B32, O::V_MOV_B32, O::V_ADD_F32, O::V_SUB_F32,
O::V_SUBREV_F32, O::V_MUL_F32, O::V_MIN_F32, O::V_MAX_F32,
O::V_MAC_F32, O::V_MAD_F32, O::V_FMA_F32, O::V_MIN3_F32,
O::V_MAX3_F32, O::V_MED3_F32, O::V_MADMK_F32, O::V_MADAK_F32,
O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}
TestCase VectorMinMaxF32NanAndSignedZeroEdges() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovLiteral(&code, 0, 0x40000000u);
AppendVMovLiteral(&code, 1, 0x7fc00000u);
AppendVMovLiteral(&code, 2, 0x00000000u);
AppendVMovLiteral(&code, 3, 0x80000000u);
AppendVMovLiteral(&code, 4, 0x40400000u);
AppendVMovLiteral(&code, 5, 0x3f800000u);
AppendVMovLiteral(&code, 6, 0x7fa00001u);
code.push_back(EncodeVop2(0x0f, 10, Vgpr(0), 1));
code.push_back(EncodeVop2(0x10, 11, Vgpr(0), 1));
code.push_back(EncodeVop2(0x0f, 12, Vgpr(3), 2));
code.push_back(EncodeVop2(0x10, 13, Vgpr(2), 3));
code.push_back(EncodeVop2(0x0f, 14, Vgpr(6), 0));
code.push_back(EncodeVop2(0x10, 15, Vgpr(0), 6));
AppendVop3(&code, 0x151, 16, Vgpr(0), Vgpr(1), Vgpr(4));
AppendVop3(&code, 0x154, 17, Vgpr(0), Vgpr(1), Vgpr(5));
for (u32 i = 0; i < 8; i++) {
AppendStoreVgpr(&code, 10 + i, i);
}
AppendEnd(&code);
return {"VectorMinMaxF32NanAndSignedZeroEdges",
code,
{},
{0x40000000u, 0x40000000u, 0x80000000u, 0x00000000u, 0x40000000u,
0x40000000u, 0x40000000u, 0x40000000u},
{O::V_MOV_B32, O::V_MIN_F32, O::V_MAX_F32, O::V_MIN3_F32,
O::V_MAX3_F32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}
TestCase VectorMed3F32NanUsesMin3Path() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovLiteral(&code, 0, 0x40000000u);
AppendVMovLiteral(&code, 1, 0x40400000u);
AppendVMovLiteral(&code, 2, 0x7fa00001u);
AppendVMovLiteral(&code, 3, 0x7fc00000u);
AppendVop3(&code, 0x157, 10, Vgpr(0), Vgpr(1), Vgpr(2));
AppendVop3(&code, 0x157, 11, Vgpr(0), Vgpr(1), Vgpr(3));
AppendVop3(&code, 0x157, 12, Vgpr(2), Vgpr(0), Vgpr(1));
AppendStoreVgpr(&code, 10, 0);
AppendStoreVgpr(&code, 11, 1);
AppendStoreVgpr(&code, 12, 2);
AppendEnd(&code);
return {"VectorMed3F32NanUsesMin3Path",
code,
{},
{0x40000000u, 0x40000000u, 0x40000000u},
{O::V_MOV_B32, O::V_MED3_F32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}
TestCase VectorFloatConversionOps() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovLiteral(&code, 0, 0xfffffffdu);
code.push_back(EncodeVop1(0x01, 1, InlineU32(7)));
AppendVMovLiteral(&code, 2, 0x40e00000u);
AppendVMovLiteral(&code, 3, 0xc0400000u);
AppendVMovLiteral(&code, 4, 0x3f800000u);
AppendVMovLiteral(&code, 5, 0x00003c00u);
AppendVMovLiteral(&code, 6, 0x40700000u);
AppendVMovLiteral(&code, 7, 0x44332211u);
AppendVMovLiteral(&code, 8, 0x40000000u);
AppendVMovLiteral(&code, 9, 0x40800000u);
AppendVMovLiteral(&code, 10, 0x40100000u);
AppendVMovLiteral(&code, 11, 0x40300000u);
code.push_back(EncodeVop1(0x01, 12, InlineU32(0)));
code.push_back(EncodeVop1(0x01, 37, InlineU32(15)));
code.push_back(EncodeVop1(0x05, 13, Vgpr(0)));
code.push_back(EncodeVop1(0x06, 14, Vgpr(1)));
code.push_back(EncodeVop1(0x07, 15, Vgpr(2)));
code.push_back(EncodeVop1(0x08, 16, Vgpr(3)));
code.push_back(EncodeVop1(0x0a, 17, Vgpr(4)));
code.push_back(EncodeVop1(0x0b, 18, Vgpr(5)));
code.push_back(EncodeVop1(0x0c, 42, Vgpr(11)));
code.push_back(EncodeVop1(0x0d, 19, Vgpr(6)));
code.push_back(EncodeVop1(0x11, 20, Vgpr(7)));
code.push_back(EncodeVop1(0x12, 21, Vgpr(7)));
code.push_back(EncodeVop1(0x13, 22, Vgpr(7)));
code.push_back(EncodeVop1(0x14, 23, Vgpr(7)));
code.push_back(EncodeVop1(0x2a, 24, Vgpr(8)));
code.push_back(EncodeVop1(0x20, 25, Vgpr(10)));
code.push_back(EncodeVop1(0x21, 26, Vgpr(11)));
code.push_back(EncodeVop1(0x22, 27, Vgpr(10)));
code.push_back(EncodeVop1(0x23, 28, Vgpr(10)));
code.push_back(EncodeVop1(0x24, 29, Vgpr(11)));
code.push_back(EncodeVop1(0x25, 30, Vgpr(4)));
code.push_back(EncodeVop1(0x27, 36, Vgpr(9)));
code.push_back(EncodeVop1(0x2e, 32, Vgpr(9)));
code.push_back(EncodeVop1(0x33, 33, Vgpr(9)));
code.push_back(EncodeVop1(0x35, 34, Vgpr(12)));
code.push_back(EncodeVop1(0x36, 35, Vgpr(12)));
code.push_back(EncodeVop1(0x0e, 38, Vgpr(37)));
code.push_back(EncodeVop1(0x2a, 39, 249));
code.push_back(EncodeVop1Sdwa(8, 6, 0, 6, 0, 0, 0, 0, 0, 1));
AppendVMovLiteral(&code, 40, 0xc0000000u);
AppendVop3(&code, 0x1aa, 41, Vgpr(40), 0, 0, 1, 0, false, 1);
AppendVMovLiteral(&code, 2, 0xc0003c00u); // low=1.0h, high=-2.0h
code.push_back(0x7e1016f9u);
code.push_back(0x00250602u);
const u32 results[] = {13, 14, 15, 16, 17, 18, 42, 19, 20, 21, 22, 23, 24, 25,
26, 27, 28, 29, 30, 36, 32, 33, 34, 35, 38, 39, 41, 8};
for (u32 i = 0; i < static_cast<u32>(std::size(results)); i++) {
AppendStoreVgpr(&code, results[i], i);
}
AppendEnd(&code);
return {"VectorFloatConversionOps",
code,
{},
{0xc0400000u, 0x40e00000u, 7, 0xfffffffdu, 0x00003c00u,
0x3f800000u, 3, 3, 0x41880000u, 0x42080000u,
0x424c0000u, 0x42880000u, 0x3f000000u, 0x3e800000u, 0x40000000u,
0x40400000u, 0x40000000u, 0x40000000u, 0x40000000u, 0x40000000u,
0x3f000000u, 0x40000000u, 0, 0x3f800000u, 0xbd800000u,
0x3f800000u, 0x3f800000u, 0x40000000u},
{O::V_MOV_B32, O::V_CVT_F32_I32, O::V_CVT_F32_U32,
O::V_CVT_U32_F32, O::V_CVT_I32_F32, O::V_CVT_F16_F32,
O::V_CVT_F32_F16, O::V_CVT_RPI_I32_F32, O::V_CVT_FLR_I32_F32,
O::V_CVT_OFF_F32_I4, O::V_CVT_F32_UBYTE0, O::V_CVT_F32_UBYTE1,
O::V_CVT_F32_UBYTE2, O::V_CVT_F32_UBYTE3, O::V_RCP_F32,
O::V_FRACT_F32, O::V_TRUNC_F32, O::V_CEIL_F32,
O::V_RNDNE_F32, O::V_FLOOR_F32, O::V_EXP_F32,
O::V_LOG_F32, O::V_RSQ_F32, O::V_SQRT_F32,
O::V_SIN_F32, O::V_COS_F32, O::BUFFER_STORE_DWORD,
O::S_ENDPGM}};
}
TestCase VectorFrexpF32Edges() {
using O = ShaderOpcode;
const std::vector<u32> inputs = {
0x00000000u, 0x80000000u, 0x3f000000u, 0xbf800000u, 0x40800000u,
0xc1000000u, 0x00800000u, 0x007fffffu, 0x00000001u, 0x7f800000u,
0x7fc00000u, 0xff800000u, 0x7fa00001u,
};
std::vector<u32> code;
for (u32 i = 0; i < static_cast<u32>(inputs.size()); i++) {
AppendVMovU32(&code, 31, i * sizeof(u32));
AppendBufferLoadDword(&code, i == 3u ? 23u : i, 31);
}
for (u32 i = 0; i < static_cast<u32>(inputs.size()); i++) {
const auto exponent_dst = i == 3u ? 19u : 32u + i;
const auto mantissa_dst = i == 3u ? 21u : 48u + i;
if (i == 3u) {
code.push_back(0x7e267ef9u); // captured v_frexp_exp_i32_f32 v19, abs(v23)
code.push_back(0x00260617u);
} else {
code.push_back(EncodeVop1(0x3f, exponent_dst, Vgpr(i)));
}
AppendStoreVgpr(&code, exponent_dst, i);
if (i == 3u) {
code.push_back(0x7e2a80f9u); // captured v_frexp_mant_f32 v21, abs(v23)
code.push_back(0x00260617u);
} else {
code.push_back(EncodeVop1(0x40, mantissa_dst, Vgpr(i)));
}
AppendStoreVgpr(&code, mantissa_dst, inputs.size() + i);
}
AppendEnd(&code);
TestCase test;
test.name = "VectorFrexpF32Edges";
test.code = std::move(code);
test.initial = inputs;
test.initial.resize(inputs.size() * 2u);
test.expected = {
0u, 0u, 0u, 1u, 3u,
4u, 0xffffff83u, 0xffffff82u, 0xffffff6cu, 0u,
0u, 0u, 0u, 0x00000000u, 0x80000000u,
0x3f000000u, 0x3f000000u, 0x3f000000u, 0xbf000000u, 0x3f000000u,
0x3f7ffffeu, 0x3f000000u, 0x7f800000u, 0x7fc00000u, 0xff800000u,
0x7fa00001u,
};
test.opcodes = {O::V_MOV_B32, O::BUFFER_LOAD_DWORD,
O::V_FREXP_EXP_I32_F32, O::V_FREXP_MANT_F32,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.decoded_counts = {{"V_FREXP_EXP_I32_F32", inputs.size()},
{"V_FREXP_MANT_F32", inputs.size()}};
test.required_spirv = {"FindUMsb", "OpBitFieldUExtract",
"OpShiftLeftLogical"};
return test;
}
TestCase CvtF32ToIntSaturatesNaNAndOutOfRange() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovLiteral(&code, 0, 0x7fc00000u);
AppendVMovLiteral(&code, 1, 0xbf800000u);
AppendVMovLiteral(&code, 2, 0x7f800000u);
AppendVMovLiteral(&code, 3, 0x4f800000u);
AppendVMovLiteral(&code, 4, 0xff800000u);
AppendVMovLiteral(&code, 5, 0x4f000000u);
code.push_back(EncodeVop1(0x07, 10, Vgpr(0)));
code.push_back(EncodeVop1(0x07, 11, Vgpr(1)));
code.push_back(EncodeVop1(0x07, 12, Vgpr(2)));
code.push_back(EncodeVop1(0x07, 13, Vgpr(3)));
code.push_back(EncodeVop1(0x08, 14, Vgpr(0)));
code.push_back(EncodeVop1(0x08, 15, Vgpr(2)));
code.push_back(EncodeVop1(0x08, 16, Vgpr(4)));
code.push_back(EncodeVop1(0x08, 17, Vgpr(5)));
for (u32 i = 0; i < 8; i++) {
AppendStoreVgpr(&code, 10 + i, i);
}
AppendEnd(&code);
return {"CvtF32ToIntSaturatesNaNAndOutOfRange",
code,
{},
{0, 0, 0xffffffffu, 0xffffffffu, 0, 0x7fffffffu, 0x80000000u,
0x7fffffffu},
{O::V_MOV_B32, O::V_CVT_U32_F32, O::V_CVT_I32_F32,
O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}
TestCase VectorSpecialF32FlushesDenormalInputs() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovLiteral(&code, 0, 0x00000001u);
code.push_back(EncodeVop1(0x27, 1, Vgpr(0)));
code.push_back(EncodeVop1(0x2a, 2, Vgpr(0)));
code.push_back(EncodeVop1(0x2e, 3, Vgpr(0)));
code.push_back(EncodeVop1(0x33, 4, Vgpr(0)));
AppendStoreVgpr(&code, 1, 0);
AppendStoreVgpr(&code, 2, 1);
AppendStoreVgpr(&code, 3, 2);
AppendStoreVgpr(&code, 4, 3);
AppendEnd(&code);
return {"VectorSpecialF32FlushesDenormalInputs",
code,
{},
{0xff800000u, 0x7f800000u, 0x7f800000u, 0x00000000u},
{O::V_MOV_B32, O::V_LOG_F32, O::V_RCP_F32, O::V_RSQ_F32,
O::V_SQRT_F32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}
TestCase VectorCompareInteger64Edges() {
using O = ShaderOpcode;
constexpr std::array<std::array<uint64_t, 2>, 10> pairs{{
{0, 0}, {0, 1}, {1, 0}, {0xffffffffull, 0x100000000ull},
{0x100000000ull, 0xffffffffull}, {0x8000000000000000ull, 0},
{0x7fffffffffffffffull, 0x8000000000000000ull},
{0xffffffffffffffffull, 0xffffffffffffffffull},
{0xffffffff80000000ull, 0xffffffff7fffffffull},
{0x1234567880000000ull, 0x123456787fffffffull}}};
TestCase test;
test.name = "VectorCompareInteger64Edges";
for (const auto& pair : pairs)
for (const auto bits : pair)
test.initial.insert(test.initial.end(), {u32(bits), u32(bits >> 32)});
test.expected = test.initial;
const auto store_mask = [&](u32 sgpr, bool value) {
AppendStoreSgprPair(&test.code, sgpr, static_cast<u32>(test.expected.size()));
test.expected.insert(test.expected.end(), {u32(value), 0});
};
for (u32 i = 0; i < pairs.size(); ++i) {
for (u32 word = 0; word < 4; ++word) {
AppendVMovU32(&test.code, 31, (i * 4 + word) * 4);
AppendBufferLoadDword(&test.code, 1 + word, 31);
}
test.code.push_back(EncodeVop1(0x02, 8, Vgpr(3)));
test.code.push_back(EncodeVop1(0x02, 9, Vgpr(4)));
const auto lhs = pairs[i][0], rhs = pairs[i][1];
const auto signed_lhs = std::bit_cast<int64_t>(lhs);
const auto signed_rhs = std::bit_cast<int64_t>(rhs);
for (const auto [opcode, value] : std::array<std::pair<u32, bool>, 11>{{
{0xa0, false}, {0xa1, signed_lhs < signed_rhs},
{0xa3, signed_lhs <= signed_rhs}, {0xa4, signed_lhs > signed_rhs},
{0xa5, lhs != rhs}, {0xa6, signed_lhs >= signed_rhs}, {0xa7, true},
{0xe0, false}, {0xe3, lhs <= rhs}, {0xe6, lhs >= rhs}, {0xe7, true}}}) {
test.code.push_back(EncodeVopc(opcode, Vgpr(1), 3));
store_mask(106, value);
AppendVop3(&test.code, opcode, 20, Vgpr(1), 8);
store_mask(20, value);
}
// A negative literal expands by signedness, including equality comparisons.
for (const auto [opcode, value] : std::array<std::pair<u32, bool>, 7>{{
{0xa1, -1 < signed_rhs}, {0xa3, -1 <= signed_rhs},
{0xa4, -1 > signed_rhs}, {0xa5, -1 != signed_rhs},
{0xa6, -1 >= signed_rhs},
{0xa2, -1 == signed_rhs}, {0xe3, 0xffffffffull <= rhs}}}) {
test.code.push_back(EncodeVopc(opcode, 255, 3));
test.code.push_back(0xffffffffu);
store_mask(106, value);
if (opcode == 0xa4 || opcode == 0xa6) {
test.code.push_back(EncodeSop1(0x04, 10, 126));
test.code.push_back(EncodeVopc(opcode + 0x10, 255, 3));
test.code.push_back(0xffffffffu);
test.code.push_back(EncodeSop1(0x04, 20, 126));
test.code.push_back(EncodeSop1(0x04, 126, 10));
store_mask(20, value);
}
}
for (const auto [opcode, value] : std::array<std::pair<u32, bool>, 3>{{
{0xa3, signed_lhs <= -1}, {0xa4, signed_lhs > -1},
{0xa6, signed_lhs >= -1}}}) {
for (const u32 source : {255u, 193u}) { // Literal and inline -1.
AppendVop3(&test.code, opcode, 20, Vgpr(1), source);
if (source == 255) test.code.push_back(0xffffffffu);
store_mask(20, value);
if (opcode == 0xa4 || opcode == 0xa6) {
test.code.push_back(EncodeSop1(0x04, 10, 126));
AppendVop3(&test.code, opcode + 0x10, 22, Vgpr(1), source);
if (source == 255) test.code.push_back(0xffffffffu);
test.code.push_back(EncodeSop1(0x04, 20, 126));
test.code.push_back(EncodeSop1(0x04, 126, 10));
store_mask(20, value);
}
}
}
test.code.push_back(EncodeSop1(0x04, 10, 126));
AppendVop3(&test.code, 0xb2, 126, 255, Vgpr(3)); // CMPX_EQ_I64 sign-extends -1.
test.code.push_back(0xffffffffu);
test.code.push_back(EncodeSop1(0x04, 20, 126));
test.code.push_back(EncodeSop1(0x04, 126, 10));
store_mask(20, -1 == signed_rhs);
// Captured 8395e43f382309df PC 0x20cc: s[2:3] = (s[0:1] != 0) & EXEC.
// Save the test buffer descriptor while its SGPRs hold the captured operands.
test.code.push_back(EncodeSop1(0x04, 32, 0));
test.code.push_back(EncodeSop1(0x04, 34, 2));
test.code.push_back(EncodeVop1(0x02, 0, Vgpr(1)));
test.code.push_back(EncodeVop1(0x02, 1, Vgpr(2)));
test.code.insert(test.code.end(), {0xd4a50002u, 0x00010000u});
test.code.push_back(EncodeSop1(0x04, 20, 2));
test.code.push_back(EncodeSop1(0x04, 0, 32));
test.code.push_back(EncodeSop1(0x04, 2, 34));
store_mask(20, lhs != 0);
}
AppendEnd(&test.code);
test.initial.resize(test.expected.size());
test.opcodes = {O::V_MOV_B32, O::BUFFER_LOAD_DWORD, O::V_READFIRSTLANE_B32,
O::V_CMP_F_I64, O::V_CMP_LT_I64, O::V_CMP_LE_I64, O::V_CMP_EQ_I64,
O::V_CMP_GT_I64, O::V_CMP_NE_I64, O::V_CMP_GE_I64,
O::V_CMP_T_I64, O::V_CMPX_EQ_I64, O::V_CMPX_GT_I64, O::V_CMPX_GE_I64,
O::V_CMP_F_U64, O::V_CMP_LE_U64, O::V_CMP_GE_U64, O::V_CMP_T_U64,
O::S_MOV_B64, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.required_spirv = {"OpSLessThan", "OpULessThanEqual", "OpUGreaterThanEqual"};
return test;
}
TestCase VectorCompareExecWaveMasks(u32 wave_size) {
using O = ShaderOpcode;
// Rows exercise signed 16-bit truncation, full-width integer ordering, and
// ordered/unordered FP64 comparisons (including NaN, signed zero, and subnormals).
constexpr std::array<std::array<std::array<uint64_t, 2>, 8>, 3> inputs{{
{{{0x1111ffff, 0x22220000}, {0x33338000, 0x44447fff},
{0x55550000, 0x66660000}, {0x77777fff, 0x88888000},
{0x99990001, 0xaaaa0002}, {0xbbbb0002, 0xcccc0001},
{0xddddffff, 0xeeeeffff}, {0xffff8001, 0x11118000}}},
{{{0, 0}, {0, 1}, {1, 0}, {0xffffffffull, 0x100000000ull},
{0x100000000ull, 0xffffffffull}, {0x8000000000000000ull, 0},
{0xffffffffffffffffull, 0xffffffffffffffffull},
{0x1234567880000000ull, 0x123456787fffffffull}}},
{{{0, 0x8000000000000000ull}, {1, 0}, {0x8000000000000001ull, 0},
{0x7ff8000000000001ull, 0}, {0, 0x7ff0000000000001ull},
{0xc000000000000000ull, 0xbff0000000000000ull},
{0x7ff0000000000000ull, 0x7ff0000000000000ull},
{0x3ff0000000000001ull, 0x3ff0000000000000ull}}}}};
struct Compare { u32 encoding; O opcode; u32 group; bool update_exec{true}; };
constexpr std::array comparisons{
Compare{0x99, O::V_CMPX_LT_I16, 0}, Compare{0x9a, O::V_CMPX_EQ_I16, 0},
Compare{0x9b, O::V_CMPX_LE_I16, 0}, Compare{0x9c, O::V_CMPX_GT_I16, 0},
Compare{0x9d, O::V_CMPX_NE_I16, 0}, Compare{0x9e, O::V_CMPX_GE_I16, 0},
Compare{0xb0, O::V_CMPX_F_I64, 1}, Compare{0xb1, O::V_CMPX_LT_I64, 1},
Compare{0xb2, O::V_CMPX_EQ_I64, 1}, Compare{0xb3, O::V_CMPX_LE_I64, 1},
Compare{0xb4, O::V_CMPX_GT_I64, 1}, Compare{0xb6, O::V_CMPX_GE_I64, 1},
Compare{0xa4, O::V_CMP_GT_I64, 1, false},
Compare{0xa6, O::V_CMP_GE_I64, 1, false},
Compare{0xb7, O::V_CMPX_T_I64, 1}, Compare{0xf0, O::V_CMPX_F_U64, 1},
Compare{0xf1, O::V_CMPX_LT_U64, 1}, Compare{0xf2, O::V_CMPX_EQ_U64, 1},
Compare{0xf3, O::V_CMPX_LE_U64, 1}, Compare{0xf4, O::V_CMPX_GT_U64, 1},
Compare{0xf6, O::V_CMPX_GE_U64, 1}, Compare{0xf7, O::V_CMPX_T_U64, 1},
Compare{0x30, O::V_CMPX_F_F64, 2}, Compare{0x3f, O::V_CMPX_TRU_F64, 2},
Compare{0x33, O::V_CMPX_LE_F64, 2}, Compare{0x36, O::V_CMPX_GE_F64, 2},
Compare{0x31, O::V_CMPX_LT_F64, 2}, Compare{0x32, O::V_CMPX_EQ_F64, 2},
Compare{0x34, O::V_CMPX_GT_F64, 2}, Compare{0x35, O::V_CMPX_LG_F64, 2},
Compare{0x37, O::V_CMPX_O_F64, 2}, Compare{0x38, O::V_CMPX_U_F64, 2},
Compare{0x39, O::V_CMPX_NGE_F64, 2}, Compare{0x3a, O::V_CMPX_NLG_F64, 2},
Compare{0x3b, O::V_CMPX_NGT_F64, 2}, Compare{0x3c, O::V_CMPX_NLE_F64, 2},
Compare{0x3d, O::V_CMPX_NEQ_F64, 2}, Compare{0x3e, O::V_CMPX_NLT_F64, 2}};
constexpr u32 low_exec = 0xeeeeeeeeu, high_exec = 0x77777777u;
constexpr u32 vcc_lo = 0x12345678u, vcc_hi = 0x89abcdefu;
TestCase test;
test.name = wave_size == 32 ? "VectorCompareExecWave32Masks" : "VectorCompareExecWave64Masks";
test.initial.assign(12 * wave_size, 0);
for (u32 group = 0; group < inputs.size(); ++group)
for (u32 lane = 0; lane < wave_size; ++lane)
for (u32 word = 0; word < 4; ++word)
test.initial[(group * 4 + word) * wave_size + lane] =
u32(inputs[group][lane % 8][word / 2] >> (32 * (word % 2)));
test.expected = test.initial;
auto& code = test.code;
code.push_back(EncodeSop1(0x04, 10, 126));
for (const auto compare : comparisons) {
for (u32 word = 0; word < 4; ++word) {
code.push_back(EncodeVop2(0x1a, 30, InlineU32(2), 0));
AppendVMovU32(&code, 31, (compare.group * 4 + word) * wave_size * 4);
code.push_back(EncodeVop2(0x25, 31, Vgpr(31), 30));
AppendBufferLoadDword(&code, 1 + word, 31);
}
std::array<u32, 2> expected_exec{};
for (u32 lane = 0; lane < wave_size; ++lane) {
const auto pair = inputs[compare.group][lane % 8];
const auto a = std::bit_cast<int16_t>(uint16_t(pair[0]));
const auto b = std::bit_cast<int16_t>(uint16_t(pair[1]));
const auto lhs = std::bit_cast<double>(pair[0]);
const auto rhs = std::bit_cast<double>(pair[1]);
bool result = false;
switch (compare.encoding) {
case 0x99: result = a < b; break;
case 0x9a: result = a == b; break;
case 0x9b: result = a <= b; break;
case 0x9c: result = a > b; break;
case 0x9d: result = a != b; break;
case 0x9e: result = a >= b; break;
case 0x30: case 0xb0: case 0xf0: result = false; break;
case 0xb1: result = std::bit_cast<int64_t>(pair[0]) < std::bit_cast<int64_t>(pair[1]); break;
case 0xb2: case 0xf2: result = pair[0] == pair[1]; break;
case 0xb3: result = std::bit_cast<int64_t>(pair[0]) <= std::bit_cast<int64_t>(pair[1]); break;
case 0xa4: case 0xb4: result = std::bit_cast<int64_t>(pair[0]) > std::bit_cast<int64_t>(pair[1]); break;
case 0xa6: case 0xb6: result = std::bit_cast<int64_t>(pair[0]) >= std::bit_cast<int64_t>(pair[1]); break;
case 0x3f: case 0xb7: case 0xf7: result = true; break;
case 0xf1: result = pair[0] < pair[1]; break;
case 0xf3: result = pair[0] <= pair[1]; break;
case 0xf4: result = pair[0] > pair[1]; break;
case 0xf6: result = pair[0] >= pair[1]; break;
case 0x33: result = lhs <= rhs; break;
case 0x36: result = lhs >= rhs; break;
case 0x31: result = lhs < rhs; break;
case 0x32: result = lhs == rhs; break;
case 0x34: result = lhs > rhs; break;
case 0x35: result = lhs < rhs || lhs > rhs; break;
case 0x37: result = !std::isnan(lhs) && !std::isnan(rhs); break;
case 0x38: result = std::isnan(lhs) || std::isnan(rhs); break;
// The N* predicates are the logical negation of their ordered counterpart,
// which is true for any comparison involving a NaN (IEEE unordered compares).
case 0x39: result = !(lhs >= rhs); break;
case 0x3a: result = !(lhs < rhs || lhs > rhs); break;
case 0x3b: result = !(lhs > rhs); break;
case 0x3c: result = !(lhs <= rhs); break;
case 0x3d: result = !(lhs == rhs); break;
case 0x3e: result = !(lhs < rhs); break;
}
if (result) expected_exec[lane / 32] |= 1u << (lane % 32);
}
expected_exec[0] &= low_exec;
expected_exec[1] &= high_exec;
for (const bool vop3 : {false, true}) {
AppendSMovLiteral(&code, 106, vcc_lo);
AppendSMovLiteral(&code, 107, vcc_hi);
AppendSMovLiteral(&code, 22, vcc_lo);
AppendSMovLiteral(&code, 23, vcc_hi);
AppendSMovLiteral(&code, 126, low_exec);
if (wave_size == 64) AppendSMovLiteral(&code, 127, high_exec);
if (vop3) AppendVop3(&code, compare.encoding, 22, Vgpr(1), Vgpr(3));
else code.push_back(EncodeVopc(compare.encoding, Vgpr(1), 3));
code.push_back(EncodeSMovB32(20, 126));
code.push_back(EncodeSMovB32(21, 127));
code.push_back(EncodeSop1(0x04, 126, 10));
const bool vcc_dest = !compare.update_exec && !vop3;
const bool sgpr_dest = !compare.update_exec && vop3;
// Wave32 writes only the low half of VCC or the selected SGPR pair.
const u32 mask_hi = wave_size == 64 ? expected_exec[1] : vcc_hi;
for (const auto [reg, value] : std::array<std::pair<u32, u32>, 6>{{
{20, compare.update_exec ? expected_exec[0] : low_exec},
{21, compare.update_exec ? expected_exec[1] : (wave_size == 64 ? high_exec : 0)},
{106, vcc_dest ? expected_exec[0] : vcc_lo},
{107, vcc_dest ? mask_hi : vcc_hi},
{22, sgpr_dest ? expected_exec[0] : vcc_lo},
{23, sgpr_dest ? mask_hi : vcc_hi}}}) {
AppendStoreSgprAtLaneDwordOffset(&code, reg, 0, u32(test.expected.size()));
test.expected.insert(test.expected.end(), wave_size, value);
}
}
test.opcodes.push_back(compare.opcode);
}
AppendEnd(&code);
test.initial.resize(test.expected.size());
test.compute_info.threads_num[0] = wave_size;
test.compute_info.threads_num[1] = test.compute_info.threads_num[2] = 1;
test.compute_info.thread_ids_num = 1;
test.compute_info.wave_size = wave_size;
test.has_compute_info = true;
test.opcodes.insert(test.opcodes.end(), {O::V_MOV_B32, O::S_MOV_B32, O::S_MOV_B64,
O::V_LSHLREV_B32, O::V_ADD_NC_U32, O::BUFFER_LOAD_DWORD,
O::BUFFER_STORE_DWORD, O::S_ENDPGM});
test.required_spirv = {"OpFOrdLessThanEqual", "OpFOrdGreaterThanEqual", "OpFOrdLessThan",
"OpFOrdGreaterThan", "OpFOrdEqual", "OpFOrdNotEqual",
"OpFUnordEqual", "OpFUnordNotEqual", "OpFUnordLessThan",
"OpFUnordGreaterThan", "OpFUnordLessThanEqual", "OpFUnordGreaterThanEqual"};
return test;
}
TestCase VectorCompareF64Edges() {
using O = ShaderOpcode;
constexpr std::array<std::array<uint64_t, 2>, 16> pairs{{
{0xc000000000000000ull, 0xbff0000000000000ull},
{0x8000000000000001ull, 0}, {0x3fc45f306dc9c882ull, 0x3fc45f306dc9c882ull},
{0, 0x8000000000000000ull}, {1, 0}, {1, 1}, {1, 2},
{0x11d04c7b00ull, 0}, {0x3ff0000000000000ull, 0x3ff0000000000001ull},
{0xbff0000000000000ull, 0xbff0000000000000ull},
{0x7ff0000000000000ull, 0x7ff0000000000000ull},
{0x7ff0000000000000ull, 0xfff0000000000000ull},
{0x7ff8000000000001ull, 0x7ff8000000000001ull},
{0x7ff0000000000001ull, 0}, {0, 0x7ff8000000000001ull},
{0x7fefffffffffffffull, 0x7fefffffffffffffull}}};
TestCase test;
test.name = "VectorCompareF64Edges";
for (const auto& pair : pairs)
for (const auto bits : pair)
test.initial.insert(test.initial.end(), {u32(bits), u32(bits >> 32)});
test.expected = test.initial;
const auto store_mask = [&](u32 sgpr, bool value) {
AppendStoreSgprPair(&test.code, sgpr, static_cast<u32>(test.expected.size()));
test.expected.insert(test.expected.end(), {u32(value), 0});
};
for (u32 i = 0; i < pairs.size(); ++i) {
for (u32 word = 0; word < 4; ++word) {
AppendVMovU32(&test.code, 31, (i * 4 + word) * 4);
AppendBufferLoadDword(&test.code, 1 + word, 31);
}
const auto lhs = std::bit_cast<double>(pairs[i][0]);
const auto rhs = std::bit_cast<double>(pairs[i][1]);
for (const bool vop3 : {false, true}) {
const auto a = vop3 ? -std::abs(lhs) : lhs;
const auto b = vop3 ? -std::abs(rhs) : rhs;
const bool ordered = !std::isnan(a) && !std::isnan(b);
// Predicates 0x20..0x2f; N* is the negation of its ordered counterpart.
const std::array results{false, a < b, a == b, a <= b, a > b,
a < b || a > b, a >= b, ordered, !ordered, !(a >= b),
!(a < b || a > b), !(a > b), !(a <= b), !(a == b), !(a < b), true};
for (u32 predicate = 0; predicate < results.size(); ++predicate) {
if (vop3)
AppendVop3(&test.code, 0x20 + predicate, 20, Vgpr(1), Vgpr(3),
0, 3, 0, false, 0, 3);
else
test.code.push_back(EncodeVopc(0x20 + predicate, Vgpr(1), 3));
store_mask(vop3 ? 20 : 106, results[predicate]);
}
}
test.code.push_back(EncodeVopc(0x23, 255, 3));
test.code.push_back(0x3ff00000u); // FP64 literals supply the high word.
store_mask(106, 1.0 <= rhs);
AppendVop3(&test.code, 0x23, 20, 248, Vgpr(3));
store_mask(20, std::bit_cast<double>(uint64_t{0x3fc45f306dc9c882ull}) <= rhs);
test.code.push_back(EncodeVop1(0x02, 100, Vgpr(3)));
test.code.push_back(EncodeVop1(0x02, 101, Vgpr(4)));
// Captured e6d76d24f59f8015 PC 0xe4: compare zero with s100:s101.
test.code.insert(test.code.end(), {0xd422006au, 0x0000c880u});
store_mask(106, 0.0 == rhs);
AppendVop3(&test.code, 0x23, 20, Vgpr(1), 100);
store_mask(20, lhs <= rhs);
}
AppendEnd(&test.code);
test.initial.resize(test.expected.size());
test.opcodes = {O::V_MOV_B32, O::BUFFER_LOAD_DWORD, O::V_READFIRSTLANE_B32,
O::V_CMP_F_F64, O::V_CMP_LT_F64, O::V_CMP_EQ_F64, O::V_CMP_LE_F64,
O::V_CMP_GT_F64, O::V_CMP_LG_F64, O::V_CMP_GE_F64, O::V_CMP_O_F64,
O::V_CMP_U_F64, O::V_CMP_NGE_F64, O::V_CMP_NLG_F64, O::V_CMP_NGT_F64,
O::V_CMP_NLE_F64, O::V_CMP_NEQ_F64, O::V_CMP_NLT_F64, O::V_CMP_TRU_F64,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.required_spirv = {"OpCapability Float64", "OpTypeFloat 64", "OpIsNan",
"OpFOrdEqual", "OpFOrdNotEqual", "OpFOrdLessThan",
"OpFOrdLessThanEqual", "OpFOrdGreaterThan", "OpFOrdGreaterThanEqual",
"OpFUnordEqual", "OpFUnordNotEqual", "OpFUnordLessThan",
"OpFUnordLessThanEqual", "OpFUnordGreaterThan", "OpFUnordGreaterThanEqual"};
return test;
}
TestCase VectorCompare64WaveMasks(u32 wave_size, bool integer_ne = false) {
using O = ShaderOpcode;
constexpr std::array<uint64_t, 4> values{0, 1, 0x8000000000000000ull,
0x7ff8000000000000ull};
constexpr u32 low_exec = 0xeeeeeeeeu, high_exec = 0x77777777u;
constexpr u32 vcc_hi = 0x89abcdefu;
constexpr u32 vcc_lo = 0x12345678u;
const u32 result_reg = integer_ne ? 20 : 106;
const u32 result_bits = integer_ne ? 0xeeeeeeeeu : 0x55555555u;
TestCase test;
test.name = integer_ne
? (wave_size == 32 ? "VectorCompareNeI64Wave32Masks" : "VectorCompareNeI64Wave64Masks")
: (wave_size == 32 ? "VectorCompareF64Wave32Masks" : "VectorCompareF64Wave64Masks");
test.initial.assign((integer_ne ? 8 : 6) * wave_size, 0);
for (u32 lane = 0; lane < wave_size; ++lane) {
test.initial[lane] = u32(values[lane % 4]);
test.initial[wave_size + lane] = u32(values[lane % 4] >> 32);
}
test.expected = test.initial;
for (u32 lane = 0; lane < wave_size; ++lane) {
test.expected[2 * wave_size + lane] = low_exec & result_bits;
test.expected[3 * wave_size + lane] = wave_size == 64 ? high_exec & result_bits : vcc_hi;
test.expected[4 * wave_size + lane] = low_exec;
test.expected[5 * wave_size + lane] = wave_size == 64 ? high_exec : 0;
if (integer_ne) {
test.expected[6 * wave_size + lane] = vcc_lo;
test.expected[7 * wave_size + lane] = vcc_hi;
}
}
auto& code = test.code;
code.push_back(EncodeVop2(0x1a, 30, InlineU32(2), 0));
AppendBufferLoadDword(&code, 1, 30);
AppendVMovU32(&code, 31, wave_size * 4);
code.push_back(EncodeVop2(0x25, 31, Vgpr(31), 30));
AppendBufferLoadDword(&code, 2, 31);
code.push_back(EncodeSop1(0x04, 10, 126));
AppendSMovLiteral(&code, result_reg + 1, vcc_hi);
if (integer_ne) {
AppendSMovLiteral(&code, 106, vcc_lo);
AppendSMovLiteral(&code, 107, vcc_hi);
}
AppendSMovLiteral(&code, 126, low_exec);
if (wave_size == 64) AppendSMovLiteral(&code, 127, high_exec);
AppendVop3(&code, integer_ne ? 0xa5 : 0x22, result_reg, InlineU32(0), Vgpr(1));
code.push_back(EncodeSMovB32(22, 126));
code.push_back(EncodeSMovB32(23, 127));
code.push_back(EncodeSop1(0x04, 126, 10));
for (const auto [reg, offset] : std::array<std::pair<u32, u32>, 4>{{
{result_reg, 2}, {result_reg + 1, 3}, {22, 4}, {23, 5}}})
AppendStoreSgprAtLaneDwordOffset(&code, reg, 0, offset * wave_size);
if (integer_ne) {
AppendStoreSgprAtLaneDwordOffset(&code, 106, 0, 6 * wave_size);
AppendStoreSgprAtLaneDwordOffset(&code, 107, 0, 7 * wave_size);
}
AppendEnd(&code);
test.compute_info.threads_num[0] = wave_size;
test.compute_info.threads_num[1] = test.compute_info.threads_num[2] = 1;
test.compute_info.thread_ids_num = 1;
test.compute_info.wave_size = wave_size;
test.has_compute_info = true;
test.opcodes = {O::V_MOV_B32, O::S_MOV_B32, O::S_MOV_B64, O::V_LSHLREV_B32,
O::V_ADD_NC_U32, O::BUFFER_LOAD_DWORD,
integer_ne ? O::V_CMP_NE_I64 : O::V_CMP_EQ_F64,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.required_spirv = {integer_ne ? "OpINotEqual" : "OpFOrdEqual"};
return test;
}
TestCase VectorF64CapturedScreenSpaceShadows() {
using O = ShaderOpcode;
TestCase test;
test.name = "VectorF64CapturedScreenSpaceShadows";
const auto store_pair = [&](u32 reg, double value) {
const auto bits = std::bit_cast<uint64_t>(value);
AppendStoreVgpr(&test.code, reg, static_cast<u32>(test.expected.size()));
AppendStoreVgpr(&test.code, reg + 1,
static_cast<u32>(test.expected.size() + 1));
test.expected.insert(test.expected.end(), {static_cast<u32>(bits),
static_cast<u32>(bits >> 32)});
};
for (const auto values :
{std::array<int32_t, 3>{63, 127, 101},
std::array<int32_t, 3>{16777217, -16777217, 123456789},
std::array<int32_t, 3>{INT32_MIN, INT32_MAX, -31}}) {
AppendVMovLiteral(&test.code, 7, static_cast<u32>(values[0]));
AppendVMovLiteral(&test.code, 1, static_cast<u32>(values[1]));
AppendVMovLiteral(&test.code, 9, static_cast<u32>(values[2]));
// Exact FP64 instruction words from b90e2024732c6111, PCs 0xc8 through
// 0x130.
test.code.insert(test.code.end(), {0x7e060907u, 0x7e0a0901u, 0x7e020909u});
store_pair(3, double(values[0]));
store_pair(5, double(values[1]));
store_pair(1, double(values[2]));
test.code.push_back(0x7e065f03u);
const double reciprocal = 1.0 / double(values[0]);
store_pair(3, reciprocal);
test.code.insert(test.code.end(), {0xd5650005u, 0x00020305u});
const double product = double(values[1]) * double(values[2]);
store_pair(5, product);
test.code.insert(test.code.end(), {0xd54c0001u, 0x84060705u});
const double result = std::fma(product, reciprocal, -double(values[2]));
store_pair(1, result);
test.code.push_back(0x7e041f01u);
AppendStoreVgpr(&test.code, 2, static_cast<u32>(test.expected.size()));
test.expected.push_back(std::bit_cast<u32>(static_cast<float>(result)));
}
AppendEnd(&test.code);
test.opcodes = {O::V_MOV_B32, O::V_CVT_F64_I32, O::V_RCP_F64,
O::V_MUL_F64, O::V_FMA_F64, O::V_CVT_F32_F64,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.required_spirv = {"OpCapability Float64",
"OpTypeFloat 64",
"OpFDiv",
"Fma",
"SignedZeroInfNanPreserve 64"};
test.ir_counts = {{"ConvertF64S32", 9},
{"FPRecip64", 3},
{"FPMul64", 3},
{"FPFma64", 3},
{"ConvertF32F64", 3}};
ShaderComputeInputInfo info{};
std::vector<u32> normal_key, other_key;
BuildStageStaticKey(info, normal_key);
info.float_mode = 0xc4;
BuildStageStaticKey(info, other_key);
Require(test.name, "pipeline identity",
normal_key != other_key && normal_key.size() == other_key.size(),
"FP64 rounding modes must not share a compiled shader key");
return test;
}
TestCase VectorFractF64CapturedAndEdges() {
using O = ShaderOpcode;
constexpr std::array<std::array<uint64_t, 2>, 17> cases{{
{0x400a000000000000ull, 0x3fd0000000000000ull}, // 3.25 -> .25
{0xbff4000000000000ull, 0x3fe8000000000000ull}, // -1.25 -> .75
{0x3ff0000000000001ull, 0x3cb0000000000000ull}, // 1 + 2^-52
{0xbff0000000000001ull, 0x3feffffffffffffeull},
{0x4320000000000001ull, 0x3fe0000000000000ull}, // 2^51 + .5
{0, 0}, {0x8000000000000000ull, 0},
{0x401c000000000000ull, 0}, {0xc01c000000000000ull, 0},
{1, 1}, {0x8000000000000001ull, 0x3ff0000000000000ull},
{0x000fffffffffffffull, 0x000fffffffffffffull},
{0x7fefffffffffffffull, 0},
{0x7ff0000000000000ull, 0x7ff8000000000000ull},
{0xfff0000000000000ull, 0x7ff8000000000000ull},
{0x7ff8123456789abcull, 0x7ff8000000000000ull},
{0x7ff0000000000001ull, 0x7ff8000000000000ull}}};
TestCase test;
test.name = "VectorFractF64CapturedAndEdges";
const auto store_pair = [&](u32 reg, uint64_t expected) {
const auto index = static_cast<u32>(test.expected.size());
AppendStoreVgpr(&test.code, reg, index);
AppendStoreVgpr(&test.code, reg + 1, index + 1);
test.expected.insert(test.expected.end(), {u32(expected), u32(expected >> 32)});
};
for (u32 i = 0; i < cases.size(); i++) {
const auto [source, expected] = cases[i];
test.initial.insert(test.initial.end(), {u32(source), u32(source >> 32)});
for (u32 word = 0; word < 2; word++) {
AppendVMovU32(&test.code, 30, (i * 2 + word) * 4);
AppendBufferLoadDword(&test.code, 4 + word, 30);
}
test.code.push_back(0x7e087d04u); // Captured v_fract_f64 v[4:5], v[4:5].
if ((source & 0x7ff0000000000000ull) == 0x7ff0000000000000ull) {
// Classify quiet NaNs without requiring a particular payload or sign.
AppendVMovU32(&test.code, 4, 0);
AppendVMovLiteral(&test.code, 6, 0x7ff80000u);
test.code.push_back(EncodeVop2(0x1b, 5, Vgpr(6), 5));
}
store_pair(4, expected);
}
AppendVMovU32(&test.code, 1, 0);
AppendVMovLiteral(&test.code, 2, 0xbff40000u);
AppendVop3(&test.code, 0x1be, 1, Vgpr(1), 0, 0, 1, 0, false, 0, 1);
store_pair(1, 0x3fe8000000000000ull); // Odd pair; ABS precedes NEG.
AppendVMovU32(&test.code, 4, 1);
AppendVMovLiteral(&test.code, 5, 0x3ff00000u);
test.code.push_back(EncodeSop1(0x04, 12, 126));
test.code.push_back(EncodeSop1(0x04, 126, InlineU32(0)));
test.code.push_back(0x7e087d04u);
test.code.push_back(EncodeSop1(0x04, 126, 12));
store_pair(4, 0x3ff0000000000001ull); // Both words survive inactive EXEC.
AppendEnd(&test.code);
test.initial.resize(test.expected.size());
test.opcodes = {O::V_MOV_B32, O::BUFFER_LOAD_DWORD, O::V_FRACT_F64,
O::V_AND_B32, O::S_MOV_B64, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.decoded_counts = {{"V_FRACT_F64 v4, v4", cases.size() + 1}};
test.required_spirv = {"OpCapability Float64", "OpTypeFloat 64", "Fract"};
return test;
}
TestCase VectorMinMaxF64CapturedAndEdges() {
using O = ShaderOpcode;
// MODE.IEEE=0: select the numeric operand, or the second NaN unchanged.
constexpr uint64_t positive_zero = 0, negative_zero = 0x8000000000000000ull;
constexpr uint64_t one = 0x3ff0000000000000ull, minus_two = 0xc000000000000000ull;
constexpr uint64_t qnan = 0x7ff8123456789abcull, snan = 0xfff0000000000042ull;
constexpr uint64_t infinity = 0x7ff0000000000000ull;
constexpr std::array<std::array<uint64_t, 4>, 20> cases{{
{one, minus_two, minus_two, one}, {minus_two, one, minus_two, one},
{one, one, one, one},
{positive_zero, negative_zero, negative_zero, positive_zero},
{negative_zero, positive_zero, negative_zero, positive_zero},
{negative_zero, negative_zero, negative_zero, negative_zero},
{positive_zero, positive_zero, positive_zero, positive_zero},
{qnan, one, one, one}, {one, qnan, one, one},
{snan, minus_two, minus_two, minus_two}, {minus_two, snan, minus_two, minus_two},
{qnan, snan, snan, snan}, {snan, qnan, qnan, qnan},
{infinity, one, one, infinity},
{infinity | negative_zero, one, infinity | negative_zero, one},
{0x3fe0000000000000ull, one, 0x3fe0000000000000ull, one},
{1, negative_zero | 1, negative_zero | 1, 1}, {1, 0, 0, 1},
{1, 2, 1, 2}, {2, 1, 1, 2}}};
TestCase test;
test.name = "VectorMinMaxF64CapturedAndEdges";
for (const auto &item : cases) {
for (u32 source = 0; source < 2; source++) {
test.initial.insert(test.initial.end(), {u32(item[source]), u32(item[source] >> 32)});
}
}
test.expected = test.initial;
const auto store_pair = [&](u32 reg, uint64_t bits) {
AppendStoreVgpr(&test.code, reg, static_cast<u32>(test.expected.size()));
AppendStoreVgpr(&test.code, reg + 1, static_cast<u32>(test.expected.size() + 1));
test.expected.insert(test.expected.end(), {u32(bits), u32(bits >> 32)});
};
for (u32 i = 0; i < cases.size(); i++) {
for (u32 word = 0; word < 4; word++) {
AppendVMovU32(&test.code, 30, (i * 4 + word) * 4);
AppendBufferLoadDword(&test.code, 2 + word, 30);
}
AppendVop3(&test.code, 0x166, 7, Vgpr(2), Vgpr(4));
store_pair(7, cases[i][2]);
AppendVop3(&test.code, 0x167, 9, Vgpr(2), Vgpr(4));
store_pair(9, cases[i][3]);
// CS daced64c72398e80 PCs 0x10c/0x11c clamp the pair to [0, 1].
test.code.insert(test.code.end(), {0xd5670006u, 0x00020480u,
0xd5660006u, 0x00020cf2u});
const auto source = std::bit_cast<double>(cases[i][0]);
uint64_t clamped = 0;
if (source > 0) clamped = source < 1 ? cases[i][0] : one;
store_pair(6, clamped);
}
AppendSMovLiteral(&test.code, 8, 0);
AppendSMovLiteral(&test.code, 9, u32(minus_two >> 32));
// Scalar source and odd destination pairs; ABS must precede NEG.
AppendVop3(&test.code, 0x166, 1, 8, 243, 0, 1, 0, false, 0, 1);
store_pair(1, minus_two);
AppendVop3(&test.code, 0x167, 1, 8, 243, 0, 1, 0, false, 0, 1);
store_pair(1, 0xbff0000000000000ull);
test.code.push_back(EncodeSop1(0x04, 12, 126));
test.code.push_back(EncodeSop1(0x04, 126, InlineU32(0)));
AppendVop3(&test.code, 0x166, 1, 242, 242);
AppendVop3(&test.code, 0x167, 1, 242, 242);
test.code.push_back(EncodeSop1(0x04, 126, 12));
store_pair(1, 0xbff0000000000000ull);
AppendEnd(&test.code);
test.initial.resize(test.expected.size());
test.opcodes = {O::V_MOV_B32, O::S_MOV_B32, O::S_MOV_B64, O::BUFFER_LOAD_DWORD,
O::V_MIN_F64, O::V_MAX_F64, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.required_spirv = {"OpCapability Float64", "OpTypeFloat 64",
"SignedZeroInfNanPreserve 64"};
return test;
}
TestCase VectorRoundF64EdgesAndModifiers() {
using O = ShaderOpcode;
constexpr std::array<uint64_t, 14> sources{
0x400a000000000000ull, 0xbff4000000000000ull,
0x3fd0000000000000ull, 0xbfd0000000000000ull,
0, 0x8000000000000000ull, 0x4330000000000001ull, 0xc330000000000001ull,
0x7ff0000000000000ull, 0xfff0000000000000ull,
0xfff8123456789abcull, 0x7ff0000000000001ull, 1, 0x8000000000000001ull};
TestCase test;
test.name = "VectorRoundF64EdgesAndModifiers";
for (const auto bits : sources) {
test.initial.insert(test.initial.end(), {u32(bits), u32(bits >> 32)});
}
test.expected = test.initial;
const auto rounded = [](u32 opcode, double value) {
switch (opcode) {
case 0x17: return std::trunc(value);
case 0x18: return std::ceil(value);
case 0x1a: return std::floor(value);
default: std::abort();
}
};
const auto store_pair = [&](u32 reg, uint64_t bits) {
AppendStoreVgpr(&test.code, reg, static_cast<u32>(test.expected.size()));
AppendStoreVgpr(&test.code, reg + 1, static_cast<u32>(test.expected.size() + 1));
test.expected.insert(test.expected.end(), {u32(bits), u32(bits >> 32)});
};
for (const u32 opcode : {0x17u, 0x18u, 0x1au}) {
for (u32 i = 0; i < sources.size(); i++) {
for (u32 word = 0; word < 2; word++) {
AppendVMovU32(&test.code, 30, (i * 2 + word) * 4);
AppendBufferLoadDword(&test.code, 2 + word, 30);
}
test.code.push_back(EncodeVop1(opcode, 5, Vgpr(2)));
AppendVop3(&test.code, 0x180 + opcode, 7, Vgpr(2), 0);
const auto value = std::bit_cast<double>(sources[i]);
for (const u32 reg : {5u, 7u}) {
if (std::isnan(value)) {
// IEEE=false does not promise a particular NaN payload or quiet bit.
test.code.push_back(EncodeVopc(0x22, Vgpr(reg), reg));
AppendStoreSgprPair(&test.code, 106, static_cast<u32>(test.expected.size()));
test.expected.insert(test.expected.end(), {0, 0});
} else {
store_pair(reg, std::bit_cast<uint64_t>(rounded(opcode, value)));
}
}
}
AppendSMovLiteral(&test.code, 8, 0);
AppendSMovLiteral(&test.code, 9, 0xbff40000u);
AppendVop3(&test.code, 0x180 + opcode, 1, 8, 0, 0, 1, 0, false, 0, 1);
const auto expected = std::bit_cast<uint64_t>(rounded(opcode, -1.25));
store_pair(1, expected);
test.code.push_back(EncodeSop1(0x04, 12, 126));
test.code.push_back(EncodeSop1(0x04, 126, InlineU32(0)));
test.code.push_back(EncodeVop1(opcode, 1, 242));
test.code.push_back(EncodeSop1(0x04, 126, 12));
store_pair(1, expected);
}
AppendEnd(&test.code);
test.initial.resize(test.expected.size());
test.opcodes = {O::V_MOV_B32, O::S_MOV_B32, O::S_MOV_B64, O::BUFFER_LOAD_DWORD,
O::V_TRUNC_F64, O::V_CEIL_F64, O::V_FLOOR_F64, O::V_CMP_EQ_F64,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.required_spirv = {"OpCapability Float64", "OpTypeFloat 64", "Trunc", "Ceil", "Floor"};
return test;
}
TestCase VectorFractF64CapturedChainRuntimeExec() {
using O = ShaderOpcode;
TestCase test;
test.name = "VectorFractF64CapturedChainRuntimeExec";
auto& code = test.code;
AppendSMovLiteral(&code, 60, 3);
AppendSMovLiteral(&code, 61, 3);
code.push_back(EncodeSop1(0x04, 126, 60)); // Lanes 0,1 and 32,33.
code.push_back(EncodeVop1(0x01, 20, Vgpr(0)));
code.push_back(EncodeVop2(0x1b, 30, InlineU32(1), 0));
code.push_back(EncodeVop2(0x1a, 30, InlineU32(4), 30));
AppendVMovU32(&code, 7, 192 * 4);
code.push_back(EncodeVop2(0x25, 30, Vgpr(7), 30));
constexpr u32 input_regs[] = {6, 4, 10, 11};
for (u32 word = 0; word < 4; ++word) {
code.push_back(EncodeMubuf0(0x0c, word * 4));
code.push_back(EncodeMubuf1(input_regs[word], 0, 30));
}
code.push_back(EncodeSmem0(0x08, 16, 0));
code.push_back(EncodeSmem1(200 * 4, 125));
constexpr u32 output_regs[] = {12, 22, 24};
for (const auto reg : output_regs) AppendVMovLiteral(&code, reg, 0xdeadbeefu);
code.push_back(EncodeVopc(0xc2, InlineU32(1), 6));
code.push_back(EncodeSop1(0x24, 8, 106));
code.push_back(EncodeSMovB32(106, 16));
AppendSMovLiteral(&code, 107, 0x3fea36a9u);
code.push_back(EncodeVop1(0x2a, 4, Vgpr(4))); // Runtime FP32 reciprocal with FP64 narrowing.
// Captured 4ee720fc4678fa19 PCs 0x1e8..0x20c, 0x23c..0x258, 0x260..0x270.
code.insert(code.end(), {0x7e04086au, 0xbeea03f4u, 0x7e082104u, 0xd54c0004u,
0x0410d502u, 0xbeea0b86u, 0xbeeb03ffu, 0x3fe3f9e1u,
0x7e087d04u, 0x7e181f04u});
code.push_back(EncodeVop1(0x01, 4, Vgpr(10))); // Second runtime image value.
code.insert(code.end(), {0x7e082104u, 0xd54c0004u, 0x0410d502u, 0xbeea0b87u,
0xbeeb03ffu, 0x3fc46bacu, 0x7e087d04u, 0x7e2c1f04u,
0x7e08210bu, 0xd54c0002u, 0x0410d502u, 0x7e047d02u,
0x7e301f02u});
code.push_back(EncodeSop1(0x04, 126, 8));
for (u32 component = 0; component < 3; ++component)
AppendStoreVgprAtLaneDwordOffset(&code, output_regs[component], 20, component * 64);
AppendEnd(&code);
test.initial.resize(192);
test.initial.insert(test.initial.end(), {1, 0x3e800000u, 0xbfa00000u, 0x40600000u,
0, 0x3f800000u, 0x40000000u, 0x40400000u, 3});
test.expected = test.initial;
constexpr u32 results[] = {0x3eea3fbcu, 0x3f1f6d21u, 0x3f7a860du};
for (u32 component = 0; component < 3; ++component) {
for (const u32 lane : {0u, 32u}) test.expected[component * 64 + lane] = results[component];
for (const u32 lane : {1u, 33u}) test.expected[component * 64 + lane] = 0xdeadbeefu;
}
test.compute_info.wave_size = 64;
test.compute_info.threads_num[0] = 64;
test.compute_info.threads_num[1] = test.compute_info.threads_num[2] = 1;
test.compute_info.thread_ids_num = 1;
test.has_compute_info = true;
test.required_spirv = {"OpCapability Float64", "OpFDiv", "Fma", "Fract"};
test.forbidden_spirv = {"RoundingModeRTE"};
test.ir_counts = {{"FPRecip32", 1}, {"ConvertF64S32", 1}, {"ConvertF64F32", 3},
{"FPFma64", 3}, {"FPFract64", 3}, {"ConvertF32F64", 3}};
test.opcodes = {O::S_MOV_B32, O::S_MOV_B64, O::V_MOV_B32, O::V_AND_B32,
O::V_LSHLREV_B32, O::V_ADD_NC_U32, O::BUFFER_LOAD_DWORD,
O::S_BUFFER_LOAD_DWORD, O::V_CMP_EQ_U32, O::S_AND_SAVEEXEC_B64,
O::S_BREV_B32, O::V_CVT_F64_I32, O::V_CVT_F64_F32, O::V_FMA_F64,
O::V_FRACT_F64, O::V_RCP_F32, O::V_CVT_F32_F64, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
return test;
}
TestCase VectorF64WideningConversions() {
using O = ShaderOpcode;
TestCase test;
test.name = "VectorF64WideningConversions";
const auto store_pair = [&](u32 reg, uint64_t bits) {
AppendStoreVgpr(&test.code, reg, static_cast<u32>(test.expected.size()));
AppendStoreVgpr(&test.code, reg + 1,
static_cast<u32>(test.expected.size() + 1));
test.expected.insert(test.expected.end(), {static_cast<u32>(bits),
static_cast<u32>(bits >> 32)});
};
for (const u32 bits :
{0u, 0x80000000u, 0x3f800001u, 0xbf800001u, 0x00800000u, 0x7f7fffffu,
0xff7fffffu, 0x7f800000u, 0xff800000u, 1u, 0x80000001u}) {
AppendSMovLiteral(&test.code, 8, bits);
test.code.push_back(0x7e002008u);
const auto source = (bits & 0x7f800000u) == 0 ? bits & 0x80000000u : bits;
store_pair(0, std::bit_cast<uint64_t>(
static_cast<double>(std::bit_cast<float>(source))));
}
for (const u32 bits : {0x7fc12345u, 0x7f812345u}) {
AppendSMovLiteral(&test.code, 8, bits);
test.code.push_back(0x7e002008u);
AppendVMovLiteral(&test.code, 3, 0x7ff80000u);
test.code.push_back(EncodeVop2(0x1b, 1, Vgpr(3), 1));
AppendStoreVgpr(&test.code, 1, static_cast<u32>(test.expected.size()));
test.expected.push_back(0x7ff80000u);
}
AppendVMovLiteral(&test.code, 1, std::bit_cast<u32>(-3.5f));
AppendVop3(&test.code, 0x190, 1, Vgpr(1), 0, 0, 1, 0, false, 0, 1);
store_pair(1, std::bit_cast<uint64_t>(-3.5));
for (const u32 value :
{0u, 1u, 16777217u, 0x7fffffffu, 0x80000000u, 0xffffffffu}) {
AppendVMovLiteral(&test.code, 1, value);
test.code.push_back(EncodeVop1(0x16, 1, Vgpr(1)));
store_pair(1, std::bit_cast<uint64_t>(static_cast<double>(value)));
}
AppendSMovLiteral(&test.code, 8, 0xffffffffu);
AppendVop3(&test.code, 0x196, 1, 8u, 0);
store_pair(1, std::bit_cast<uint64_t>(4294967295.0));
test.code.push_back(EncodeSop1(0x04, 12, 126));
test.code.push_back(EncodeSop1(0x04, 126, InlineU32(0)));
test.code.push_back(EncodeVop1(0x10, 1, InlineU32(0)));
test.code.push_back(EncodeVop1(0x16, 1, InlineU32(0)));
test.code.push_back(EncodeSop1(0x04, 126, 12));
store_pair(1, std::bit_cast<uint64_t>(4294967295.0));
AppendEnd(&test.code);
test.opcodes = {O::S_MOV_B32, O::S_MOV_B64, O::V_MOV_B32,
O::V_AND_B32, O::V_CVT_F64_F32, O::V_CVT_F64_U32,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.required_spirv = {"OpCapability Float64", "OpTypeFloat 64", "OpFConvert",
"OpConvertUToF"};
std::vector<u32> vertex_code;
AppendSMovLiteral(&vertex_code, 8, std::bit_cast<u32>(2.0f));
vertex_code.insert(vertex_code.end(),
{0x7e002008u, 0x7e005f00u, 0x7e401f00u});
vertex_code.push_back(EncodeExp0(0x0c, 0xf));
vertex_code.push_back(EncodeExp1(32, 32, 32, 32));
AppendEnd(&vertex_code);
ShaderVertexInputInfo vertex{};
ShaderRecompiler::CompileOptions options;
options.stage = ShaderType::Vertex;
options.input_info.vertex = &vertex;
auto translated = ShaderRecompiler::TranslateProgram(vertex_code, options);
auto result =
ShaderRecompiler::CompileProgram(std::move(translated), options, {});
ValidateSpirv(test.name, result.spirv);
Require(test.name, "captured vertex conversion chain",
result.ir_dump.find("ConvertF64F32") != std::string::npos &&
result.ir_dump.find("FPRecip64") != std::string::npos &&
result.ir_dump.find("ConvertF32F64") != std::string::npos,
"vertex compilation lost the captured widening/reciprocal/narrowing "
"sequence");
return test;
}
TestCase VectorF64ModesModifiersAndExec() {
using O = ShaderOpcode;
TestCase test;
test.name = "VectorF64ModesModifiersAndExec";
constexpr std::array<std::array<uint64_t, 2>, 8> add_pairs{{
{1, 1}, {1, 0x8000000000000001ull},
{0x8000000000000000ull, 0x8000000000000000ull},
{0x3ff0000000000000ull, 0x3ca0000000000000ull},
{0x3ff0000000000001ull, 0x3ca0000000000000ull},
{0x3ff0000000000000ull, 0xbff0000000000000ull},
{0x7ff0000000000000ull, 0x3ff0000000000000ull},
{0xfff0000000000000ull, 0xbff0000000000000ull}}};
for (const auto& pair : add_pairs)
for (const auto bits : pair)
test.initial.insert(test.initial.end(), {u32(bits), u32(bits >> 32)});
// Runtime F64 values immediately around even/odd F32 halfway points, both signs.
constexpr std::array<std::pair<uint64_t, u32>, 12> narrowing{{
{0x3ff000000fffffffull, 0x3f800000u}, {0x3ff0000010000000ull, 0x3f800000u},
{0x3ff0000010000001ull, 0x3f800001u}, {0x3ff000002fffffffull, 0x3f800001u},
{0x3ff0000030000000ull, 0x3f800002u}, {0x3ff0000030000001ull, 0x3f800002u},
{0xbff000000fffffffull, 0xbf800000u}, {0xbff0000010000000ull, 0xbf800000u},
{0xbff0000010000001ull, 0xbf800001u}, {0xbff000002fffffffull, 0xbf800001u},
{0xbff0000030000000ull, 0xbf800002u}, {0xbff0000030000001ull, 0xbf800002u}}};
for (const auto& item : narrowing)
test.initial.insert(test.initial.end(), {u32(item.first), u32(item.first >> 32)});
test.expected = test.initial;
const auto load_pair = [&](u32 reg, uint64_t bits) {
AppendVMovLiteral(&test.code, reg, static_cast<u32>(bits));
AppendVMovLiteral(&test.code, reg + 1, static_cast<u32>(bits >> 32));
};
const auto store_pair = [&](u32 reg, uint64_t bits) {
AppendStoreVgpr(&test.code, reg, static_cast<u32>(test.expected.size()));
AppendStoreVgpr(&test.code, reg + 1,
static_cast<u32>(test.expected.size() + 1));
test.expected.insert(test.expected.end(), {static_cast<u32>(bits),
static_cast<u32>(bits >> 32)});
};
const auto store_word = [&](u32 reg, u32 bits) {
AppendStoreVgpr(&test.code, reg, static_cast<u32>(test.expected.size()));
test.expected.push_back(bits);
};
for (const auto [source, expected] :
{std::pair<uint64_t, uint64_t>{0, 0x7ff0000000000000ull},
{0x8000000000000000ull, 0xfff0000000000000ull},
{0x7ff0000000000000ull, 0},
{0xfff0000000000000ull, 0x8000000000000000ull},
{0x0008000000000000ull, 0x7fe0000000000000ull},
{0x7fe0000000000000ull, 0x0008000000000000ull}}) {
load_pair(1, source);
test.code.push_back(EncodeVop1(0x2f, 1, Vgpr(1)));
store_pair(1, expected);
}
for (u32 i = 0; i < add_pairs.size(); ++i) {
const auto pair = add_pairs[i];
for (u32 word = 0; word < 4; ++word) {
AppendVMovU32(&test.code, 31, (i * 4 + word) * 4);
AppendBufferLoadDword(&test.code, 1 + word, 31);
}
AppendVop3(&test.code, 0x164, 1, Vgpr(1), Vgpr(3));
store_pair(1, std::bit_cast<uint64_t>(std::bit_cast<double>(pair[0]) +
std::bit_cast<double>(pair[1])));
}
// Scalar pair, exact inline 1/(2*pi), literal high word, and ABS before NEG.
AppendSMovLiteral(&test.code, 8, 0);
AppendSMovLiteral(&test.code, 9, 0xbff00000u);
AppendVop3(&test.code, 0x164, 1, 8, 244, 0, 1, 0, false, 0, 1);
store_pair(1, std::bit_cast<uint64_t>(1.0));
AppendVop3(&test.code, 0x164, 1, 255, 240);
test.code.push_back(0x40080000u);
store_pair(1, std::bit_cast<uint64_t>(3.5));
AppendVop3(&test.code, 0x164, 1, 248, InlineU32(0));
store_pair(1, 0x3fc45f306dc9c882ull);
for (const auto pair : std::array<std::array<uint64_t, 2>, 2>{{
{0x7ff0000000000000ull, 0xfff0000000000000ull},
{0x7ff0000000000001ull, 0}}}) {
load_pair(1, pair[0]);
load_pair(3, pair[1]);
AppendVop3(&test.code, 0x164, 1, Vgpr(1), Vgpr(3));
AppendVMovLiteral(&test.code, 3, 0x7ff80000u);
test.code.push_back(EncodeVop2(0x1b, 2, Vgpr(3), 2));
store_word(2, 0x7ff80000u);
}
// A fused result retains the residual that separate multiply/add would round
// to zero.
load_pair(1, std::bit_cast<uint64_t>(1.0 + 0x1p-27));
load_pair(3, std::bit_cast<uint64_t>(1.0 - 0x1p-27));
AppendVop3(&test.code, 0x14c, 1, Vgpr(1), Vgpr(3), 243u);
store_pair(1, std::bit_cast<uint64_t>(-0x1p-54));
// Unaligned pairs, SGPR pairs, inline floats, literal highword expansion, ABS
// then NEG.
test.code.push_back(EncodeSop1(0x03, 8, 255));
test.code.push_back(0u);
test.code.push_back(EncodeSop1(0x03, 9, 255));
test.code.push_back(0xbff00000u);
AppendVop3(&test.code, 0x165, 1, 8u, 244u, 0, 1, 0, false, 0, 1);
store_pair(1, std::bit_cast<uint64_t>(-2.0));
AppendVop3(&test.code, 0x165, 1, 255u, 242u);
test.code.push_back(0x40080000u);
store_pair(1, std::bit_cast<uint64_t>(3.0));
AppendVop3(&test.code, 0x165, 1, 248u, 242u);
store_pair(1, 0x3fc45f306dc9c882ull);
AppendVop3(&test.code, 0x184, 1, InlineU32(3), 0);
store_pair(1, std::bit_cast<uint64_t>(3.0));
AppendVop3(&test.code, 0x1af, 1, 244u, 0);
store_pair(1, std::bit_cast<uint64_t>(0.5));
// Preserve both destination words when EXEC is clear, restoring the original
// live lanes.
test.code.push_back(EncodeSop1(0x04, 12, 126));
test.code.push_back(EncodeSop1(0x04, 126, InlineU32(0)));
test.code.push_back(EncodeVop1(0x04, 1, InlineU32(9)));
AppendVop3(&test.code, 0x164, 1, 242, 242);
test.code.push_back(EncodeSop1(0x04, 126, 12));
store_pair(1, std::bit_cast<uint64_t>(0.5));
for (const auto [source, expected] :
{std::pair<uint64_t, u32>{0x47f0000000000000ull, 0x7f7fffffu},
{0xc7f0000000000000ull, 0xff7fffffu},
{0x7ff0000000000000ull, 0x7f800000u},
{0xfff0000000000000ull, 0xff800000u},
{0x3800000000000000ull, 0},
{0xb800000000000000ull, 0x80000000u},
{0x3810000000000000ull, 0x00800000u},
{0x3ff0000010000000ull, 0x3f800000u}}) {
load_pair(1, source);
AppendVop3(&test.code, 0x18f, 1, Vgpr(1), 0);
store_word(1, expected);
}
for (u32 i = 0; i < narrowing.size(); ++i) {
for (u32 word = 0; word < 2; ++word) {
AppendVMovU32(&test.code, 31, (add_pairs.size() * 4 + i * 2 + word) * 4);
AppendBufferLoadDword(&test.code, 1 + word, 31);
}
AppendVop3(&test.code, 0x18f, 1, Vgpr(1), 0);
store_word(1, narrowing[i].second);
}
load_pair(1, 0x7ff8123456789abcull);
test.code.push_back(EncodeVop1(0x2f, 1, Vgpr(1)));
AppendVMovLiteral(&test.code, 3, 0x7ff80000u);
test.code.push_back(EncodeVop2(0x1b, 2, Vgpr(3), 2));
store_word(2, 0x7ff80000u);
AppendEnd(&test.code);
test.opcodes = {O::V_MOV_B32, O::S_MOV_B32, O::S_MOV_B64,
O::V_CVT_F64_I32, O::V_RCP_F64, O::V_MUL_F64,
O::V_ADD_F64, O::V_FMA_F64, O::V_CVT_F32_F64, O::V_AND_B32,
O::BUFFER_LOAD_DWORD, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.initial.resize(test.expected.size());
test.required_spirv = {"OpFAdd"};
return test;
}
TestCase VectorRcpIflagF32IntegerReciprocal() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 0, 0);
AppendVMovU32(&code, 1, 1);
AppendVMovU32(&code, 2, 2);
AppendVMovLiteral(&code, 3, 0xffffffffu);
for (u32 i = 0; i < 4; i++) {
code.push_back(EncodeVop1(0x06, 4 + i, Vgpr(i)));
code.push_back(EncodeVop1(0x2b, 8 + i, Vgpr(4 + i)));
}
AppendVMovLiteral(&code, 12, 0xc0000000u);
AppendVop3(&code, 0x1ab, 13, Vgpr(12), 0, 0, 1, 0, false, 1);
for (u32 i = 0; i < 4; i++) {
AppendStoreVgpr(&code, 8 + i, i);
}
AppendStoreVgpr(&code, 13, 4);
AppendEnd(&code);
TestCase test;
test.name = "VectorRcpIflagF32IntegerReciprocal";
test.code = std::move(code);
test.expected = {0x7f800000u, 0x3f800000u, 0x3f000000u, 0x2f800000u,
0x3f800000u};
test.opcodes = {O::V_MOV_B32, O::V_CVT_F32_U32, O::V_RCP_IFLAG_F32,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.decoded_counts = {{"V_RCP_IFLAG_F32", 5}};
test.ir_counts = {{"FPRecipIFlag32", 5}};
test.required_spirv = {"OpConvertUToF", "OpFDiv"};
test.forbidden_spirv = {"8388608"};
return test;
}
TestCase VectorSinCosMaxFiniteSpecialCases() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovLiteral(&code, 0, 0xff7fffffu);
code.push_back(EncodeVop1(0x35, 1, Vgpr(0)));
code.push_back(EncodeVop1(0x36, 2, Vgpr(0)));
AppendStoreVgpr(&code, 1, 0);
AppendStoreVgpr(&code, 2, 1);
AppendEnd(&code);
return {"VectorSinCosMaxFiniteSpecialCases",
code,
{},
{0x00000000u, 0x3f800000u},
{O::V_MOV_B32, O::V_SIN_F32, O::V_COS_F32, O::BUFFER_STORE_DWORD,
O::S_ENDPGM}};
}
TestCase VectorCompareF32ExposureGuard() {
using O = ShaderOpcode;
TestCase test;
test.name = "VectorCompareF32ExposureGuard";
test.initial = {0x00000001u, 0x80000001u, 0x007fffffu, 0x807fffffu,
0x00000000u, 0x80000000u, 0x3f800000u, 0x40000000u};
test.expected = test.initial;
AppendVMovLiteral(&test.code, 1, 0x3f800000u);
for (u32 i = 0; i < test.initial.size(); i++) {
AppendVMovU32(&test.code, 31, i * sizeof(u32));
AppendBufferLoadDword(&test.code, 0, 31);
// Captured exposure guard: replace zero with 1.0 before taking its reciprocal.
test.code.push_back(EncodeVopc(0x02, InlineU32(0), 0));
test.code.push_back(EncodeVop2(0x01, 2, Vgpr(0), 1));
test.code.push_back(EncodeVop1(0x2a, 3, Vgpr(2)));
AppendStoreVgpr(&test.code, 2, static_cast<u32>(test.expected.size()));
AppendStoreVgpr(&test.code, 3, static_cast<u32>(test.expected.size() + 1));
test.expected.push_back(i == 7 ? 0x40000000u : 0x3f800000u);
test.expected.push_back(i == 7 ? 0x3f000000u : 0x3f800000u);
}
test.initial.resize(test.expected.size());
AppendEnd(&test.code);
test.opcodes = {O::BUFFER_LOAD_DWORD, O::V_CMP_EQ_F32, O::V_CNDMASK_B32,
O::V_RCP_F32, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
return test;
}
TestCase VectorCompareF32DenormalModes(u32 mode) {
using O = ShaderOpcode;
enum Relation : u32 { Less, Equal, Greater, Unordered };
struct Pair {
u32 lhs;
u32 rhs;
Relation relation;
};
constexpr Pair pairs[] = {
{0x00000001u, 0x00000000u, Equal},
{0x80000001u, 0x80000000u, Equal},
{0x00000000u, 0x00000001u, Equal},
{0x80000000u, 0x80000001u, Equal},
{0x007fffffu, 0x80000001u, Equal},
{0x00800000u, 0x007fffffu, Greater},
{0xbf800000u, 0x80000001u, Less},
{0xff800000u, 0xbf800000u, Less},
{0x7f800000u, 0x7f800000u, Equal},
{0x7fc12345u, 0x00000000u, Unordered},
{0x00000000u, 0x7fa12345u, Unordered},
{0x00000000u, 0x80000000u, Equal},
};
TestCase test;
constexpr const char* names[] = {"VectorCompareF32DenormalModeC0",
"VectorCompareF32DenormalModeE0"};
test.name = names[(mode >> 5u) & 1u];
test.compute_info.float_mode = static_cast<uint8_t>(mode);
test.has_compute_info = true;
for (const auto& pair : pairs) {
test.initial.insert(test.initial.end(), {pair.lhs, pair.rhs});
}
test.expected = test.initial;
AppendVMovU32(&test.code, 1, 1);
for (u32 i = 0; i < std::size(pairs); i++) {
AppendVMovU32(&test.code, 31, i * 2u * sizeof(u32));
AppendBufferLoadDword(&test.code, 2, 31);
AppendVMovU32(&test.code, 31, (i * 2u + 1u) * sizeof(u32));
AppendBufferLoadDword(&test.code, 3, 31);
// The sixteen architectural predicates select subsets of LT/EQ/GT/unordered.
for (u32 predicate = 0; predicate < 16; predicate++) {
test.code.push_back(EncodeVopc(predicate, Vgpr(2), 3));
test.code.push_back(EncodeVop2(0x01, 4, InlineU32(0), 1));
AppendStoreVgpr(&test.code, 4, static_cast<u32>(test.expected.size()));
test.expected.push_back((predicate >> pairs[i].relation) & 1u);
}
}
test.initial.resize(test.expected.size());
AppendEnd(&test.code);
test.opcodes = {O::BUFFER_LOAD_DWORD, O::V_CMP_EQ_F32, O::V_CMP_LE_F32,
O::V_CMP_GE_F32, O::V_CNDMASK_B32, O::BUFFER_STORE_DWORD,
O::S_ENDPGM};
return test;
}
TestCase VectorCompareOps() {
using O = ShaderOpcode;
std::vector<u32> code;
std::vector<u32> results;
std::vector<u32> expected;
u32 dst = 10;
code.push_back(EncodeVop1(0x01, 1, InlineU32(1)));
AppendVMovLiteral(&code, 2, 0x40000000u);
AppendVMovLiteral(&code, 3, 0x40400000u);
code.push_back(EncodeVop1(0x01, 4, InlineU32(1)));
AppendVMovLiteral(&code, 5, 0xffffffffu);
auto append_compare = [&](u32 opcode, u32 src0, u32 src1, bool value) {
if (dst == 31u) {
dst++;
}
code.push_back(EncodeVopc(opcode, src0, src1));
code.push_back(EncodeVop2(0x01, dst, InlineU32(0), 1));
results.push_back(dst);
expected.push_back(value ? 1u : 0u);
dst++;
};
append_compare(0x00, Vgpr(2), 3, false);
append_compare(0x01, Vgpr(2), 3, true);
append_compare(0x02, Vgpr(2), 3, false);
append_compare(0x03, Vgpr(2), 3, true);
append_compare(0x04, Vgpr(2), 3, false);
append_compare(0x05, Vgpr(2), 3, true);
append_compare(0x06, Vgpr(2), 3, false);
append_compare(0x07, Vgpr(2), 3, true);
append_compare(0x08, Vgpr(2), 3, false);
append_compare(0x09, Vgpr(2), 3, true);
append_compare(0x0a, Vgpr(2), 3, false);
append_compare(0x0b, Vgpr(2), 3, true);
append_compare(0x0c, Vgpr(2), 3, false);
append_compare(0x0d, Vgpr(2), 3, true);
append_compare(0x0e, Vgpr(2), 3, false);
append_compare(0x0f, Vgpr(2), 3, true);
append_compare(0x80, Vgpr(5), 4, false);
append_compare(0x81, Vgpr(5), 4, true);
append_compare(0x82, Vgpr(5), 4, false);
append_compare(0x83, Vgpr(5), 4, true);
append_compare(0x84, Vgpr(5), 4, false);
append_compare(0x85, Vgpr(5), 4, true);
append_compare(0x86, Vgpr(5), 4, false);
append_compare(0x87, Vgpr(5), 4, true);
append_compare(0xc0, Vgpr(4), 5, false);
append_compare(0xc1, Vgpr(4), 5, true);
append_compare(0xc2, Vgpr(4), 5, false);
append_compare(0xc3, Vgpr(4), 5, true);
append_compare(0xc4, Vgpr(4), 5, false);
append_compare(0xc5, Vgpr(4), 5, true);
append_compare(0xc6, Vgpr(4), 5, false);
append_compare(0xc7, Vgpr(4), 5, true);
code.push_back(0x7c1d00f9u);
code.push_back(0x86069201u);
code.push_back(EncodeVopc(0x84, 249, 4));
code.push_back(EncodeVopcSdwa(5, 19, 1));
code.push_back(EncodeVopc(0x01, 250, 3));
code.push_back(EncodeVop2Dpp(2));
AppendVMovLiteral(&code, 6, 0xc0000000u);
AppendVMovLiteral(&code, 7, 0x3f800000u);
AppendVop3(&code, 0x04, 20, Vgpr(6), Vgpr(7), 0, 1);
for (u32 i = 0; i < static_cast<u32>(results.size()); i++) {
AppendStoreVgpr(&code, results[i], i);
}
AppendStoreSgpr(&code, 18, static_cast<u32>(results.size()));
expected.push_back(1u);
AppendStoreSgpr(&code, 19, static_cast<u32>(results.size() + 1u));
expected.push_back(0u);
AppendStoreSgpr(&code, 106, static_cast<u32>(results.size() + 2u));
expected.push_back(1u);
AppendStoreSgpr(&code, 20, static_cast<u32>(results.size() + 3u));
expected.push_back(1u);
AppendEnd(&code);
return {"VectorCompareOps",
code,
{},
expected,
{O::V_MOV_B32, O::V_CNDMASK_B32, O::V_CMP_F_F32,
O::V_CMP_LT_F32, O::V_CMP_EQ_F32, O::V_CMP_LE_F32,
O::V_CMP_GT_F32, O::V_CMP_LG_F32, O::V_CMP_GE_F32,
O::V_CMP_O_F32, O::V_CMP_U_F32, O::V_CMP_NGE_F32,
O::V_CMP_NLG_F32, O::V_CMP_NGT_F32, O::V_CMP_NLE_F32,
O::V_CMP_NEQ_F32, O::V_CMP_NLT_F32, O::V_CMP_TRU_F32,
O::V_CMP_F_I32, O::V_CMP_LT_I32, O::V_CMP_EQ_I32,
O::V_CMP_LE_I32, O::V_CMP_GT_I32, O::V_CMP_NE_I32,
O::V_CMP_GE_I32, O::V_CMP_T_I32, O::V_CMP_F_U32,
O::V_CMP_LT_U32, O::V_CMP_EQ_U32, O::V_CMP_LE_U32,
O::V_CMP_GT_U32, O::V_CMP_NE_U32, O::V_CMP_GE_U32,
O::V_CMP_T_U32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}
TestCase VectorVop3CompareEqU64OnGpu() {
using O = ShaderOpcode;
std::vector<u32> code;
code.push_back(EncodeVop1(0x01, 1, InlineU32(1)));
code.push_back(EncodeSop1(0x04, 106, 126)); // s_mov_b64 vcc, exec
code.push_back(0xd4e2006au);
code.push_back(0x0000d47eu); // v_cmp_eq_u64 vcc, exec, vcc
code.push_back(EncodeVop2(0x01, 2, InlineU32(0), 1));
AppendStoreVgpr(&code, 2, 0);
AppendSMovLiteral(&code, 107, 1); // high-dword mismatch
code.push_back(0xd4e2006au);
code.push_back(0x0000d47eu); // v_cmp_eq_u64 vcc, exec, vcc
code.push_back(EncodeVop2(0x01, 3, InlineU32(0), 1));
AppendStoreVgpr(&code, 3, 1);
AppendEnd(&code);
TestCase test{"VectorVop3CompareEqU64OnGpu",
code,
{},
{1, 0},
{O::V_MOV_B32, O::S_MOV_B64, O::V_CMP_EQ_U64,
O::V_CNDMASK_B32, O::S_MOV_B32, O::BUFFER_STORE_DWORD,
O::S_ENDPGM}};
test.decoded_counts = {{"V_CMP_EQ_U64 vcc_lo, exec_lo, vcc_lo", 2}};
return test;
}
TestCase VectorVop3CompareNeU64OnGpu() {
using O = ShaderOpcode;
std::vector<u32> code;
code.push_back(EncodeVop1(0x01, 1, InlineU32(1)));
code.push_back(EncodeSop1(0x04, 106, 126)); // s_mov_b64 vcc, exec
code.push_back(0xd4e5006au);
code.push_back(0x0000d47eu); // v_cmp_ne_u64 vcc, exec, vcc
code.push_back(EncodeVop2(0x01, 2, InlineU32(0), 1));
AppendStoreVgpr(&code, 2, 0);
AppendSMovLiteral(&code, 106, 0);
AppendSMovLiteral(&code, 107, 0);
code.push_back(0xd4e5006au);
code.push_back(0x0000d47eu); // v_cmp_ne_u64 vcc, exec, vcc
code.push_back(EncodeVop2(0x01, 3, InlineU32(0), 1));
AppendStoreVgpr(&code, 3, 1);
AppendEnd(&code);
return {"VectorVop3CompareNeU64OnGpu",
code,
{},
{0, 1},
{O::V_MOV_B32, O::S_MOV_B64, O::V_CMP_NE_U64, O::V_CNDMASK_B32,
O::S_MOV_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}
TestCase VectorVopcCmpxNeU64CapturedExecMask() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 7, 0);
AppendVMovU32(&code, 8, 0);
AppendVMovU32(&code, 31, 0);
code.push_back(0x7dea0e80u); // v_cmpx_ne_u64 exec, 0, v[7:8]
AppendBufferStoreDword(&code, 8, 31);
code.push_back(EncodeSMovB32(126, InlineU32(1)));
AppendVMovU32(&code, 8, 1);
AppendVMovU32(&code, 31, 4);
code.push_back(0x7dea0e80u); // v_cmpx_ne_u64 exec, 0, v[7:8]
AppendBufferStoreDword(&code, 8, 31);
AppendVMovU32(&code, 2, 0);
AppendVMovU32(&code, 3, 0);
AppendVMovU32(&code, 31, 8);
code.push_back(0x7dea0480u); // v_cmpx_ne_u64 exec, 0, v[2:3]
AppendBufferStoreDword(&code, 3, 31);
code.push_back(EncodeSMovB32(126, InlineU32(1)));
AppendVMovU32(&code, 3, 1);
AppendVMovU32(&code, 31, 12);
code.push_back(0x7dea0480u); // v_cmpx_ne_u64 exec, 0, v[2:3]
AppendBufferStoreDword(&code, 3, 31);
AppendEnd(&code);
TestCase test{"VectorVopcCmpxNeU64CapturedExecMask",
code,
{0x11223344u, 0x55667788u, 0xaabbccddu, 0xeeff0011u},
{0x11223344u, 1u, 0xaabbccddu, 1u},
{O::V_MOV_B32, O::V_CMPX_NE_U64, O::BUFFER_STORE_DWORD,
O::S_MOV_B32, O::S_ENDPGM}};
test.decoded_counts = {{"V_CMPX_NE_U64 exec_lo, 0, v7", 2},
{"V_CMPX_NE_U64 exec_lo, 0, v2", 2}};
return test;
}
TestCase VectorVop3CmpxNeI64CapturedExecMask() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 7, 0);
AppendVMovU32(&code, 8, 0);
AppendVMovU32(&code, 31, 0);
code.push_back(0xd4b5007eu);
code.push_back(0x00020e80u); // v_cmpx_ne_i64 exec, 0, v[7:8]
AppendBufferStoreDword(&code, 8, 31);
code.push_back(EncodeSMovB32(126, InlineU32(1)));
AppendVMovU32(&code, 8, 1);
AppendVMovU32(&code, 31, 4);
code.push_back(0xd4b5007eu);
code.push_back(0x00020e80u); // v_cmpx_ne_i64 exec, 0, v[7:8]
AppendBufferStoreDword(&code, 8, 31);
AppendEnd(&code);
TestCase test{"VectorVop3CmpxNeI64CapturedExecMask",
code,
{0x11223344u, 0x55667788u},
{0x11223344u, 1u},
{O::V_MOV_B32, O::V_CMPX_NE_I64, O::BUFFER_STORE_DWORD,
O::S_MOV_B32, O::S_ENDPGM}};
test.decoded_counts = {{"V_CMPX_NE_I64 exec_lo, 0, v7", 2}};
return test;
}
TestCase VectorVop3CompareGtU64OnGpu() {
using O = ShaderOpcode;
std::vector<u32> code;
code.push_back(EncodeVop1(0x01, 1, InlineU32(1)));
AppendSMovLiteral(&code, 4, 0);
AppendSMovLiteral(&code, 5, 0x80000000u);
code.push_back(0xd4e4006au);
code.push_back(0x00010004u); // v_cmp_gt_u64 vcc, s[4:5], 0
code.push_back(EncodeVop2(0x01, 2, InlineU32(0), 1));
AppendStoreVgpr(&code, 2, 0);
code.push_back(0xd4e4006au);
code.push_back(0x0001006au); // v_cmp_gt_u64 vcc, vcc, 0
code.push_back(EncodeVop2(0x01, 3, InlineU32(0), 1));
AppendStoreVgpr(&code, 3, 1);
AppendSMovLiteral(&code, 106, 0);
AppendSMovLiteral(&code, 107, 0);
code.push_back(0xd4e4006au);
code.push_back(0x0001006au); // v_cmp_gt_u64 vcc, vcc, 0
code.push_back(EncodeVop2(0x01, 4, InlineU32(0), 1));
AppendStoreVgpr(&code, 4, 2);
AppendEnd(&code);
TestCase test{"VectorVop3CompareGtU64OnGpu",
code,
{},
{1, 1, 0},
{O::V_MOV_B32, O::S_MOV_B32, O::V_CMP_GT_U64, O::V_CNDMASK_B32,
O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
test.compute_info.threads_num[0] = 64;
test.compute_info.threads_num[1] = 1;
test.compute_info.threads_num[2] = 1;
test.compute_info.wave_size = 32;
test.compute_info.thread_ids_num = 1;
test.has_compute_info = true;
return test;
}
TestCase VectorVopcCompareLtU64OnGpu() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 0, 5);
AppendVMovU32(&code, 1, 0);
AppendVMovU32(&code, 2, 1);
AppendSMovLiteral(&code, 4, 4);
AppendSMovLiteral(&code, 5, 0);
code.push_back(EncodeVopc(0xe1, 4, 0));
code.push_back(EncodeVop2(0x01, 3, InlineU32(0), 2));
AppendStoreVgpr(&code, 3, 0);
AppendSMovLiteral(&code, 4, 6);
code.push_back(EncodeVopc(0xe1, 4, 0));
code.push_back(EncodeVop2(0x01, 4, InlineU32(0), 2));
AppendStoreVgpr(&code, 4, 1);
AppendEnd(&code);
TestCase test{"VectorVopcCompareLtU64OnGpu",
code,
{},
{1, 0},
{O::V_MOV_B32, O::S_MOV_B32, O::V_CMP_LT_U64,
O::V_CNDMASK_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
test.decoded_counts = {{"V_CMP_LT_U64 vcc_lo, s4, v0", 2}};
return test;
}
TestCase VectorVop3CompareEqI64OnGpu() {
using O = ShaderOpcode;
std::vector<u32> code;
code.push_back(EncodeVop1(0x01, 1, InlineU32(1)));
code.push_back(EncodeSop1(0x04, 4, 126)); // s_mov_b64 s[4:5], exec
code.push_back(0xd4a2006au);
code.push_back(0x0000087eu); // v_cmp_eq_i64 vcc, exec, s[4:5]
code.push_back(EncodeVop2(0x01, 2, InlineU32(0), 1));
AppendStoreVgpr(&code, 2, 0);
AppendSMovLiteral(&code, 4, 0);
code.push_back(0xd4a2006au);
code.push_back(0x0000087eu); // low dword mismatch
code.push_back(EncodeVop2(0x01, 3, InlineU32(0), 1));
AppendStoreVgpr(&code, 3, 1);
AppendEnd(&code);
return {"VectorVop3CompareEqI64OnGpu",
code,
{},
{1, 0},
{O::V_MOV_B32, O::S_MOV_B64, O::V_CMP_EQ_I64, O::V_CNDMASK_B32,
O::S_MOV_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}
TestCase VectorCompareClassF32() {
using O = ShaderOpcode;
std::vector<u32> code;
std::vector<u32> expected;
AppendVMovU32(&code, 1, 1);
u32 out = 0;
auto append_class = [&](u32 bits, u32 class_mask, bool match) {
AppendVMovLiteral(&code, 0, bits);
AppendVMovU32(&code, 2, class_mask);
code.push_back(EncodeVopc(0x88, Vgpr(0), 2));
code.push_back(EncodeVop2(0x01, 3, InlineU32(0), 1));
AppendStoreVgpr(&code, 3, out++);
expected.push_back(match ? 1u : 0u);
};
append_class(0x7fc00000u, 1u << 1u, true);
append_class(0xff800000u, 1u << 2u, true);
append_class(0xbf800000u, 1u << 3u, true);
append_class(0x80000000u, 1u << 5u, true);
append_class(0x00000000u, 1u << 6u, true);
append_class(0x40000000u, 1u << 8u, true);
append_class(0x7f800000u, 1u << 9u, true);
append_class(0x40000000u, 1u << 3u, false);
append_class(0x00000001u, 1u << 7u, true);
append_class(0x80000001u, 1u << 4u, true);
AppendVMovLiteral(&code, 0, 0x3f800000u);
AppendVMovU32(&code, 2, 1u << 3u);
AppendVop3(&code, 0x88, 20, Vgpr(0), Vgpr(2), 0, 0, 0, false, 0, 1);
AppendStoreSgprPair(&code, 20, out);
expected.push_back(1u);
expected.push_back(0u);
AppendEnd(&code);
return {"VectorCompareClassF32",
code,
{},
expected,
{O::V_MOV_B32, O::V_CMP_CLASS_F32, O::V_CNDMASK_B32,
O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}
TestCase VectorCompareClassF32ImmediateAndDynamicMasks() {
using O = ShaderOpcode;
// One input for each architectural class bit, in bit order.
constexpr std::array<u32, 10> values{
0x7f800001u, 0x7fc00001u, 0xff800000u, 0xbf800000u, 0x80000001u,
0x80000000u, 0x00000000u, 0x00000001u, 0x3f800000u, 0x7f800000u};
constexpr std::array<u32, 8> masks{
0u, 0x3ffu, 3u, 7u, 0xfffffc00u, 0xffffffffu, 0xfffffc03u, 0xfffffc07u};
TestCase test;
test.name = "VectorCompareClassF32ImmediateAndDynamicMasks";
test.initial.assign(values.begin(), values.end());
test.initial.insert(test.initial.end(), masks.begin(), masks.end());
test.expected = test.initial;
auto &code = test.code;
AppendVMovU32(&code, 1, 1);
for (u32 index = 0; index < values.size(); ++index) {
AppendVMovU32(&code, 30, index * 4u);
AppendBufferLoadDword(&code, 0, 30);
for (u32 mask_index = 0; mask_index < masks.size(); ++mask_index) {
for (const bool dynamic : {false, true}) {
if (dynamic) {
AppendVMovU32(&code, 30, (values.size() + mask_index) * 4u);
AppendBufferLoadDword(&code, 2, 30);
code.push_back(EncodeVopc(0x88, Vgpr(0), 2));
} else {
AppendVop3(&code, 0x88, 106, Vgpr(0), 255);
code.push_back(masks[mask_index]);
}
code.push_back(EncodeVop2(0x01, 3, InlineU32(0), 1));
AppendStoreVgpr(&code, 3, static_cast<u32>(test.expected.size()));
test.expected.push_back((masks[mask_index] & (1u << index)) != 0u);
}
}
}
AppendEnd(&code);
test.initial.resize(test.expected.size());
test.opcodes = {O::V_MOV_B32, O::BUFFER_LOAD_DWORD, O::V_CMP_CLASS_F32,
O::V_CNDMASK_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.required_spirv = {"OpIsNan"};
return test;
}
TestCase VectorVopcSdwaCmpxClassF32CapturedExecMask() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovLiteral(&code, 13, 0x7fc00000u); // Quiet NaN.
AppendVMovU32(&code, 3, 7);
AppendVMovU32(&code, 31, 0);
code.push_back(EncodeSMovB32(106, InlineU32(2))); // qNaN class mask.
code.insert(code.end(), {0x7d30d4f9u, 0x8606000du});
AppendBufferStoreDword(&code, 3, 31);
code.push_back(EncodeSMovB32(126, InlineU32(1))); // Restore lane-zero EXEC.
AppendVMovLiteral(&code, 13, 0x3f800000u); // Positive normal.
AppendVMovU32(&code, 3, 9);
AppendVMovU32(&code, 31, 4);
code.push_back(EncodeSMovB32(106, InlineU32(2))); // qNaN class mask.
code.insert(code.end(), {0x7d30d4f9u, 0x8606000du});
AppendBufferStoreDword(&code, 3, 31);
AppendEnd(&code);
TestCase test{
"VectorVopcSdwaCmpxClassF32CapturedExecMask",
code,
{0x11111111u, 0x22222222u},
{7u, 0x22222222u},
{O::S_MOV_B32, O::V_MOV_B32, O::V_CMPX_CLASS_F32,
O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
test.decoded_counts = {
{"V_CMPX_CLASS_F32 exec_lo, v13, vcc_lo", 2}};
return test;
}
TestCase VectorCmpClassF16(u32 wave_size) {
using O = ShaderOpcode;
constexpr std::array<u32, 10> values{
0x7c01u, 0x7e01u, 0xfc00u, 0xbc00u, 0x8001u,
0x8000u, 0x0000u, 0x0001u, 0x3c00u, 0x7c00u};
constexpr u32 guard = 0x87654321u;
TestCase test;
test.name = wave_size == 64 ? "VectorCmpClassF16Wave64" : "VectorCmpClassF16Wave32";
for (const auto value : values) {
test.initial.insert(test.initial.end(), {0xa55a0000u | value, (value << 16u) | 0x7e01u});
}
test.expected = test.initial;
auto &code = test.code;
const auto compare = [&](std::initializer_list<u32> words, u32 expected,
u32 dst = 106u, u32 exec = 1u) {
AppendSMovLiteral(&code, 106, 0x12345678u);
AppendSMovLiteral(&code, 107, guard);
AppendSMovLiteral(&code, dst + 1u, guard);
code.push_back(EncodeSopc(0x06, InlineU32(expected), InlineU32(0)));
code.insert(code.end(), words);
const u32 sources[] = {dst, dst + 1u, 126, 127, 253};
for (u32 i = 0; i < std::size(sources); ++i) {
code.push_back(EncodeSMovB32(30u + i, sources[i]));
}
code.push_back(EncodeSMovB32(126, InlineU32(1)));
const u32 results[] = {expected, wave_size == 32 ? guard : 0u, exec, 0u, 1u - expected};
for (u32 i = 0; i < std::size(results); ++i) {
AppendStoreSgpr(&code, 30u + i, static_cast<u32>(test.expected.size()));
test.expected.push_back(results[i]);
}
if (dst != 106u) {
AppendStoreSgpr(&code, 106, static_cast<u32>(test.expected.size()));
AppendStoreSgpr(&code, 107, static_cast<u32>(test.expected.size()) + 1u);
test.expected.insert(test.expected.end(), {0x12345678u, guard});
}
};
for (u32 index = 0; index < values.size(); ++index) {
AppendVMovU32(&code, 30, index * 8u);
AppendBufferLoadDword(&code, 0, 30);
AppendVMovU32(&code, 30, index * 8u + 4u);
AppendBufferLoadDword(&code, 2, 30);
for (u32 bit = 0; bit < 12; ++bit) {
AppendVMovU32(&code, 1, bit == 11 ? 0 : 1u << bit);
const u32 expected = bit == index ? 1u : 0u;
compare({EncodeVopc(0x8f, Vgpr(0), 1)}, expected);
compare({EncodeVop3Word0(0x8f, 20), EncodeVop3Word1(Vgpr(0), Vgpr(1))},
expected, 20);
}
// SDWA selects the high half, applies -abs, and writes an explicit SGPR pair.
compare({EncodeVopc(0x8f, 249, InlineU32(56)),
EncodeVopcSdwa(2, 20, 1, 5, 6, 0, 0, 1, 1, 0, 0, 0, 1)},
index >= 3 && index <= 8 ? 1u : 0u, 20);
compare({EncodeVop3Word0(0x8f, 20, 1),
EncodeVop3Word1(Vgpr(0), InlineU32(56), 0, 0, 1)},
index >= 3 && index <= 8 ? 1u : 0u, 20);
}
AppendVMovU32(&code, 1, 1u << 8u);
compare({EncodeVopc(0x8f, 242, 1)}, 1); // Inline 1.0 becomes half 0x3c00.
compare({EncodeVopc(0x8f, 255, 1), 0xdead3c00u}, 1); // Literal low half.
compare({EncodeVop3Word0(0x8f, 20), EncodeVop3Word1(242, 248)}, 1, 20);
// Inline INV_2PI supplies half bits 0x3118, including the positive-normal class.
code.push_back(EncodeSMovB32(126, InlineU32(0)));
compare({EncodeVopc(0x8f, 242, 1)}, 0, 106, 0); // Inactive lanes clear the result.
AppendEnd(&code);
test.initial.resize(test.expected.size());
test.opcodes = {O::V_MOV_B32, O::S_MOV_B32, O::S_CMP_EQ_U32, O::BUFFER_LOAD_DWORD,
O::V_CMP_CLASS_F16, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.decoded_counts = {{"V_CMP_CLASS_F16", 264u}};
test.compute_info.wave_size = wave_size;
test.has_compute_info = true;
return test;
}
TestCase VectorCmpxClassF16(u32 wave_size) {
using O = ShaderOpcode;
// One raw half value for each architectural class bit, in bit order.
constexpr std::array<u32, 10> values{
0x7c01u, 0x7e01u, 0xfc00u, 0xbc00u, 0x8001u,
0x8000u, 0x0000u, 0x0001u, 0x3c00u, 0x7c00u};
TestCase test;
test.name = wave_size == 64 ? "VectorCmpxClassF16Wave64" : "VectorCmpxClassF16Wave32";
for (const auto value : values) {
test.initial.insert(test.initial.end(), {0xa55a0000u | value, (value << 16u) | 0x7e01u});
}
test.expected = test.initial;
auto &code = test.code;
AppendSMovLiteral(&code, 106, 0x12345678u);
AppendSMovLiteral(&code, 107, 0x87654321u);
code.push_back(EncodeSMovB32(127, InlineU32(0)));
const auto compare = [&](std::initializer_list<u32> words, u32 expected) {
code.insert(code.end(), words);
code.push_back(EncodeSMovB32(20, 126));
code.push_back(EncodeSMovB32(21, 127));
code.push_back(EncodeSMovB32(126, InlineU32(1)));
AppendStoreSgpr(&code, 20, static_cast<u32>(test.expected.size()));
AppendStoreSgpr(&code, 21, static_cast<u32>(test.expected.size()) + 1u);
test.expected.insert(test.expected.end(), {expected, 0});
};
for (u32 index = 0; index < values.size(); ++index) {
AppendVMovU32(&code, 30, index * 8u);
AppendBufferLoadDword(&code, 0, 30);
AppendVMovU32(&code, 30, index * 8u + 4u);
AppendBufferLoadDword(&code, 2, 30);
for (u32 bit = 0; bit < 12; ++bit) {
AppendVMovU32(&code, 1, bit == 11 ? 0 : 1u << bit);
const u32 expected = bit == index ? 1 : 0;
if ((bit & 1u) == 0) {
compare({EncodeVopc(0x9f, Vgpr(0), 1)}, expected);
} else {
compare({EncodeVop3Word0(0x9f, 126), EncodeVop3Word1(Vgpr(0), Vgpr(1))}, expected);
}
}
const u32 finite = index >= 3 && index <= 8 ? 1 : 0;
compare({0x7d3f70f9u, 0x86360000u}, finite); // Captured -abs(v0), inline56.
compare({0x7d3f70f9u, 0x86350002u}, finite); // Captured -abs(v2.word1).
}
AppendVMovU32(&code, 1, 1u << 8u);
compare({EncodeVopc(0x9f, 242, 1)}, 1); // Inline1.0 encodes half0x3c00.
AppendVMovU32(&code, 1, 1u << 6u);
compare({EncodeVopc(0x9f, 249, 1),
EncodeVopcSdwa(242, 0, 0, 5, 6, 0, 0, 0, 0, 0, 0, 1)}, 1);
AppendVMovU32(&code, 1, 1u << 7u);
compare({EncodeVopc(0x9f, 249, 1),
EncodeVopcSdwa(242, 0, 0, 1, 6, 0, 0, 0, 0, 0, 0, 1)}, 1);
AppendVMovU32(&code, 30, 8u * 8u);
AppendBufferLoadDword(&code, 0, 30);
compare({EncodeVopc(0x9f, 249, 248),
EncodeVopcSdwa(0, 0, 0, 6, 6, 0, 0, 0, 0, 0, 0, 0, 1)}, 1);
// INV_2PI as a half class mask is0x3118, including positive normal bit8.
AppendVMovU32(&code, 1, 1u << 8u);
code.push_back(EncodeSMovB32(126, InlineU32(0)));
compare({EncodeVopc(0x9f, Vgpr(0), 1)}, 0);
AppendStoreSgpr(&code, 106, static_cast<u32>(test.expected.size()));
AppendStoreSgpr(&code, 107, static_cast<u32>(test.expected.size()) + 1u);
test.expected.insert(test.expected.end(), {0x12345678u, 0x87654321u});
AppendEnd(&code);
test.initial.resize(test.expected.size());
test.opcodes = {O::V_MOV_B32, O::S_MOV_B32, O::BUFFER_LOAD_DWORD,
O::V_CMPX_CLASS_F16, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.compute_info.wave_size = wave_size;
test.has_compute_info = true;
return test;
}
TestCase Wave32VccMasksPreserveOtherHalf() {
using O = ShaderOpcode;
TestCase test;
test.name = "Wave32VccMasksPreserveOtherHalf";
auto &code = test.code;
AppendSMovLiteral(&code, 106, 0x12345678u);
AppendSMovLiteral(&code, 107, 0x2468ace0u);
AppendVMovU32(&code, 1, 20);
u32 offset = 0;
const auto check_masks = [&](u32 low, u32 high) {
AppendStoreSgprAtLaneDwordOffset(&code, 106, 0, offset);
AppendStoreSgprAtLaneDwordOffset(&code, 107, 0, offset + 4);
AppendVop3(&code, 0x101, 2, InlineU32(10), Vgpr(1), 107);
code.push_back(EncodeVop2(0x01, 3, InlineU32(10), 1));
AppendStoreVgprAtLaneDwordOffset(&code, 2, 0, offset + 8);
AppendStoreVgprAtLaneDwordOffset(&code, 3, 0, offset + 12);
test.expected.insert(test.expected.end(), 4, low);
test.expected.insert(test.expected.end(), 4, high);
for (u32 mask : {high, low}) {
for (u32 lane = 0; lane < 4; ++lane) {
test.expected.push_back((mask & (1u << lane)) != 0 ? 20u : 10u);
}
}
offset += 16;
};
// PPSA10737 keeps a table offset in VCC_LO while comparing into VCC_HI.
AppendVop3(&code, 0xc2, 107, InlineU32(1), Vgpr(0));
check_masks(0x12345678u, 2u);
AppendVop3(&code, 0xc2, 106, InlineU32(2), Vgpr(0));
check_masks(4u, 2u);
code.push_back(EncodeSop2(0x10, 106, 106, 107)); // s_or_b32
check_masks(6u, 2u);
code.push_back(EncodeSop1(0x04, 106, 193u)); // Explicit s_mov_b64 writes both.
check_masks(0xffffffffu, 0xffffffffu);
AppendEnd(&code);
test.opcodes = {O::S_MOV_B32, O::S_MOV_B64, O::S_OR_B32, O::V_MOV_B32,
O::V_CMP_EQ_U32, O::V_CNDMASK_B32, O::V_LSHLREV_B32,
O::V_ADD_NC_U32, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.compute_info.threads_num[0] = 4;
test.compute_info.threads_num[1] = test.compute_info.threads_num[2] = 1;
test.compute_info.thread_ids_num = 1;
test.compute_info.wave_size = 32;
test.has_compute_info = true;
return test;
}
TestCase VectorCompareF16Ops() {
using O = ShaderOpcode;
std::vector<u32> code;
std::vector<u32> expected;
AppendVMovU32(&code, 1, 1);
AppendVMovLiteral(&code, 2, 0x40003c00u); // low=1.0h, high=2.0h
AppendVMovLiteral(&code, 3, 0x44004200u); // low=3.0h, high=4.0h
AppendVMovLiteral(&code, 4, 0x00007e00u); // low=qNaN
u32 dst = 10;
u32 out = 0;
auto append_compare = [&](u32 opcode, u32 src0, u32 src1, bool value) {
code.push_back(EncodeVopc(opcode, src0, src1));
code.push_back(EncodeVop2(0x01, dst, InlineU32(0), 1));
AppendStoreVgpr(&code, dst, out++);
expected.push_back(value ? 1u : 0u);
dst++;
};
append_compare(0xc9, Vgpr(2), 3, true);
append_compare(0xca, Vgpr(2), 2, true);
append_compare(0xcb, Vgpr(2), 3, true);
append_compare(0xcc, Vgpr(3), 2, true);
append_compare(0xcd, Vgpr(2), 3, true);
append_compare(0xce, Vgpr(3), 2, true);
append_compare(0xe9, Vgpr(2), 3, true);
append_compare(0xe9, Vgpr(3), 2, false);
append_compare(0xe9, Vgpr(2), 2, false);
append_compare(0xe9, Vgpr(4), 2, true);
append_compare(0xe9, Vgpr(2), 4, true);
append_compare(0xed, Vgpr(4), 2, true);
auto append_cmpx = [&](u32 opcode, u32 src0, u32 src1) {
code.push_back(EncodeSMovB32(126, InlineU32(1)));
code.push_back(EncodeVopc(opcode, src0, src1));
AppendStoreVgpr(&code, 1, out++);
expected.push_back(1u);
};
append_cmpx(0xd9, Vgpr(2), 3);
append_cmpx(0xda, Vgpr(2), 2);
append_cmpx(0xdb, Vgpr(2), 3);
append_cmpx(0xdc, Vgpr(3), 2);
append_cmpx(0xde, Vgpr(3), 2);
append_cmpx(0xfd, Vgpr(4), 2);
append_cmpx(0xfe, Vgpr(2), 2);
AppendEnd(&code);
return {"VectorCompareF16Ops",
code,
{},
expected,
{O::V_MOV_B32, O::S_MOV_B32, O::V_CMP_LT_F16, O::V_CMP_EQ_F16,
O::V_CMP_LE_F16, O::V_CMP_GT_F16, O::V_CMP_LG_F16, O::V_CMP_GE_F16,
O::V_CMP_NGE_F16, O::V_CMP_NEQ_F16, O::V_CMPX_LT_F16, O::V_CMPX_EQ_F16,
O::V_CMPX_LE_F16, O::V_CMPX_GT_F16, O::V_CMPX_GE_F16,
O::V_CMPX_NEQ_F16, O::V_CMPX_NLT_F16, O::V_CNDMASK_B32,
O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}
TestCase VectorCompareI16U16Ops() {
using O = ShaderOpcode;
std::vector<u32> code;
std::vector<u32> expected;
AppendVMovU32(&code, 1, 1);
AppendVMovLiteral(&code, 2, 0xabcd8000u);
AppendVMovLiteral(&code, 3, 0x00007fffu);
AppendVMovLiteral(&code, 4, 0x12348000u);
AppendVMovLiteral(&code, 5, 0xffff0001u);
AppendVMovLiteral(&code, 6, 0x00000001u);
u32 dst = 10;
u32 out = 0;
auto append_compare = [&](u32 opcode, u32 src0, u32 src1, bool value) {
code.push_back(EncodeVopc(opcode, src0, src1));
code.push_back(EncodeVop2(0x01, dst, InlineU32(0), 1));
AppendStoreVgpr(&code, dst, out++);
expected.push_back(value ? 1u : 0u);
dst++;
};
append_compare(0x8a, Vgpr(2), 4, true);
append_compare(0x8a, Vgpr(2), 3, false);
append_compare(0x8b, Vgpr(2), 3, true);
append_compare(0x8b, Vgpr(3), 2, false);
append_compare(0x8b, Vgpr(5), 6, true);
append_compare(0x8c, Vgpr(3), 2, true);
append_compare(0x8c, Vgpr(2), 3, false);
append_compare(0x8d, Vgpr(2), 3, true);
append_compare(0x8d, Vgpr(5), 6, false);
append_compare(0xaa, Vgpr(2), 4, true);
append_compare(0xaa, Vgpr(2), 3, false);
append_compare(0xab, Vgpr(3), 2, true);
append_compare(0xab, Vgpr(2), 3, false);
append_compare(0xac, Vgpr(2), 3, true);
append_compare(0xac, Vgpr(3), 2, false);
append_compare(0xad, Vgpr(2), 3, true);
append_compare(0xad, Vgpr(5), 6, false);
append_compare(0xae, Vgpr(2), 3, true);
append_compare(0xae, Vgpr(3), 2, false);
append_compare(0xae, Vgpr(5), 6, true);
AppendEnd(&code);
return {"VectorCompareI16U16Ops",
code,
{},
expected,
{O::V_MOV_B32, O::V_CMP_EQ_I16, O::V_CMP_LE_I16, O::V_CMP_GT_I16,
O::V_CMP_NE_I16, O::V_CMP_EQ_U16, O::V_CMP_LE_U16, O::V_CMP_GT_U16,
O::V_CMP_NE_U16, O::V_CMP_GE_U16, O::V_CNDMASK_B32,
O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}
TestCase Vop2SdwaCndmaskSourceModifier() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovLiteral(&code, 6, 0x00000000u);
AppendVMovLiteral(&code, 47, 0x12345678u);
AppendVMovLiteral(&code, 53, 0x3f800000u);
code.push_back(EncodeVopc(0xc7, Vgpr(6), 6)); // v_cmp_t_u32
code.push_back(0x025e6af9u);
code.push_back(0x16060635u);
AppendStoreVgpr(&code, 47, 0);
AppendEnd(&code);
return {"Vop2SdwaCndmaskSourceModifier",
code,
{},
{0xbf800000u},
{O::V_MOV_B32, O::V_CMP_T_U32, O::V_CNDMASK_B32,
O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}
TestCase Vop2SdwaCndmaskCapturedByte3Source() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovLiteral(&code, 0, 0x55667788u);
AppendBufferLoadDword(&code, 3, 30);
code.push_back(EncodeVopc(0xc0, Vgpr(0), 0)); // v_cmp_f_u32
code.push_back(0x023a00f9u);
code.push_back(0x06030603u);
AppendStoreVgpr(&code, 29, 1);
code.push_back(EncodeVopc(0xc7, Vgpr(0), 0)); // v_cmp_t_u32
code.push_back(0x023a00f9u);
code.push_back(0x06030603u);
AppendStoreVgpr(&code, 29, 2);
AppendEnd(&code);
TestCase test;
test.name = "Vop2SdwaCndmaskCapturedByte3Source";
test.code = code;
test.initial = {0xabcd1234u, 0, 0};
test.expected = {0xabcd1234u, 0x000000abu, 0x55667788u};
test.opcodes = {O::V_MOV_B32, O::BUFFER_LOAD_DWORD, O::V_CMP_F_U32,
O::V_CMP_T_U32, O::V_CNDMASK_B32, O::BUFFER_STORE_DWORD,
O::S_ENDPGM};
test.decoded_counts = {{"V_CNDMASK_B32 v29, v3.sdwa(sel=3,sext=0), v0", 2}};
test.required_spirv = {"OpBitFieldUExtract", "OpSelect"};
return test;
}
TestCase Vop3CndmaskUsesSgprMaskLaneBits() {
using O = ShaderOpcode;
std::vector<u32> code = {
EncodeVop1(0x01, 1, 0), EncodeVop2(0x1a, 1, InlineU32(2), 1),
EncodeVop2(0x25, 1, Vgpr(0), 1), EncodeVopc(0xc2, InlineU32(0), 0),
EncodeSop1(0x04, 4, 106),
};
AppendVop3(&code, 0x101, 2, InlineU32(10), InlineU32(20), 4);
code.push_back(EncodeVop2(0x1a, 3, InlineU32(2), 1));
AppendBufferStoreDword(&code, 2, 3);
AppendEnd(&code);
TestCase test;
test.name = "Vop3CndmaskUsesSgprMaskLaneBits";
test.code = code;
test.expected = {20, 10, 10, 10, 20, 10, 10, 10};
test.opcodes = {O::V_MOV_B32, O::V_LSHLREV_B32, O::V_ADD_NC_U32,
O::V_CMP_EQ_U32, O::S_MOV_B64, O::V_CNDMASK_B32,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.compute_info.threads_num[0] = 4;
test.compute_info.threads_num[1] = 1;
test.compute_info.threads_num[2] = 1;
test.compute_info.group_id[0] = true;
test.compute_info.thread_ids_num = 1;
test.compute_info.workgroup_register = 0;
test.has_compute_info = true;
test.dispatch_x = 2;
return test;
}
TestCase Vop3CndmaskAllowsDataSourceModifier() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovLiteral(&code, 7, 0x3f800000u);
AppendSMovLiteral(&code, 34, 1);
AppendSMovLiteral(&code, 35, 0);
code.push_back(0xd5010004u);
code.push_back(0x408a0f07u);
AppendStoreVgpr(&code, 4, 0);
AppendEnd(&code);
return {"Vop3CndmaskAllowsDataSourceModifier",
code,
{},
{0xbf800000u},
{O::V_MOV_B32, O::S_MOV_B32, O::V_CNDMASK_B32, O::BUFFER_STORE_DWORD,
O::S_ENDPGM}};
}
TestCase VectorCompareExecOps() {
using O = ShaderOpcode;
std::vector<u32> code;
code.push_back(EncodeVop1(0x01, 1, InlineU32(1)));
AppendVMovLiteral(&code, 2, 0x40000000u);
AppendVMovLiteral(&code, 3, 0x40400000u);
code.push_back(EncodeVop1(0x01, 4, InlineU32(1)));
AppendVMovLiteral(&code, 5, 0xffffffffu);
u32 out = 0;
auto append_cmpx = [&](u32 opcode, u32 src0, u32 src1) {
code.push_back(EncodeSMovB32(126, InlineU32(1)));
code.push_back(EncodeVopc(opcode, src0, src1));
AppendStoreVgpr(&code, 1, out++);
};
append_cmpx(0x11, Vgpr(2), 3);
append_cmpx(0x12, Vgpr(3), 3);
append_cmpx(0x13, Vgpr(2), 3);
append_cmpx(0x14, Vgpr(3), 2);
append_cmpx(0x15, Vgpr(2), 3);
append_cmpx(0x16, Vgpr(3), 3);
append_cmpx(0x19, Vgpr(2), 3);
append_cmpx(0x1a, Vgpr(3), 3);
append_cmpx(0x1b, Vgpr(2), 3);
append_cmpx(0x1c, Vgpr(3), 2);
append_cmpx(0x1d, Vgpr(2), 3);
append_cmpx(0x1e, Vgpr(3), 3);
append_cmpx(0x91, Vgpr(5), 4);
append_cmpx(0x92, Vgpr(4), 4);
append_cmpx(0x93, Vgpr(5), 4);
append_cmpx(0x94, Vgpr(4), 5);
append_cmpx(0x95, Vgpr(5), 4);
append_cmpx(0x96, Vgpr(4), 4);
append_cmpx(0xd1, Vgpr(4), 5);
append_cmpx(0xd2, Vgpr(4), 4);
append_cmpx(0xd3, Vgpr(4), 5);
append_cmpx(0xd4, Vgpr(5), 4);
append_cmpx(0xd5, Vgpr(4), 5);
append_cmpx(0xd6, Vgpr(4), 4);
AppendEnd(&code);
return {"VectorCompareExecOps",
code,
{},
std::vector<u32>(out, 1),
{O::S_MOV_B32, O::V_MOV_B32, O::V_CMPX_LT_F32,
O::V_CMPX_EQ_F32, O::V_CMPX_LE_F32, O::V_CMPX_GT_F32,
O::V_CMPX_LG_F32, O::V_CMPX_GE_F32, O::V_CMPX_NGE_F32,
O::V_CMPX_NLG_F32, O::V_CMPX_NGT_F32, O::V_CMPX_NLE_F32,
O::V_CMPX_NEQ_F32, O::V_CMPX_NLT_F32, O::V_CMPX_LT_I32,
O::V_CMPX_EQ_I32, O::V_CMPX_LE_I32, O::V_CMPX_GT_I32,
O::V_CMPX_NE_I32, O::V_CMPX_GE_I32, O::V_CMPX_LT_U32,
O::V_CMPX_EQ_U32, O::V_CMPX_LE_U32, O::V_CMPX_GT_U32,
O::V_CMPX_NE_U32, O::V_CMPX_GE_U32, O::BUFFER_STORE_DWORD,
O::S_ENDPGM}};
}
TestCase VectorVopcCmpxOrderedCapturedExecMask() {
using O = ShaderOpcode;
struct CompareCase {
u32 lhs;
u32 rhs;
u32 initial_exec;
u32 expected_exec;
bool captured;
};
constexpr std::array<CompareCase, 8> cases{{
{0x3f800000u, 0x3f800000u, 1u, 1u, true}, // Finite value.
{0x7fc00000u, 0x7fc00000u, 1u, 0u, true}, // Quiet NaN.
{0x7f800001u, 0x7f800001u, 1u, 0u, true}, // Signaling NaN.
{0x7f800000u, 0x7f800000u, 1u, 1u, true}, // Infinity is ordered.
{0x80000000u, 0x80000000u, 1u, 1u, true}, // Signed zero is ordered.
{0x3f800000u, 0x3f800000u, 0u, 0u, true}, // Inactive lane.
{0x3f800000u, 0x7fc00000u, 1u, 0u, false}, // NaN in SRC1.
{0x7fc00000u, 0x3f800000u, 1u, 0u, false}, // NaN in SRC0.
}};
constexpr u32 vcc_lo = 0x13579bdfu;
constexpr u32 vcc_hi = 0x2468ace0u;
TestCase test;
test.name = "VectorVopcCmpxOrderedCapturedExecMask";
auto &code = test.code;
for (const auto &entry : cases) {
test.initial.push_back(entry.lhs);
test.initial.push_back(entry.rhs);
}
test.expected = test.initial;
for (u32 i = 0; i < cases.size(); ++i) {
const auto &entry = cases[i];
AppendVMovU32(&code, 30, i * 8u);
AppendBufferLoadDword(&code, 1, 30);
AppendVMovU32(&code, 30, i * 8u + 4u);
AppendBufferLoadDword(&code, 2, 30);
AppendSMovLiteral(&code, 106, vcc_lo);
AppendSMovLiteral(&code, 107, vcc_hi);
code.push_back(EncodeSMovB32(126, InlineU32(entry.initial_exec)));
code.push_back(entry.captured ? 0x7c2e0301u
: EncodeVopc(0x17u, Vgpr(1), 2));
code.push_back(EncodeSMovB32(20, 126));
code.push_back(EncodeSMovB32(21, 127));
code.push_back(EncodeSMovB32(22, 106));
code.push_back(EncodeSMovB32(23, 107));
code.push_back(EncodeSMovB32(126, InlineU32(1)));
const u32 out = static_cast<u32>(cases.size()) * 2u + i * 4u;
AppendStoreSgprPair(&code, 20, out);
AppendStoreSgprPair(&code, 22, out + 2u);
test.expected.insert(test.expected.end(),
{entry.expected_exec, 0u, vcc_lo, vcc_hi});
}
test.initial.resize(test.expected.size());
AppendEnd(&code);
test.opcodes = {O::V_MOV_B32, O::S_MOV_B32, O::BUFFER_LOAD_DWORD,
O::V_CMPX_O_F32, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.decoded_counts = {{"V_CMPX_O_F32 exec_lo, v1, v1", 6u},
{"V_CMPX_O_F32 exec_lo, v1, v2", 2u}};
test.compute_info.wave_size = 32;
test.has_compute_info = true;
return test;
}
TestCase VectorVop3FloatCompareNegSourceModifier() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovLiteral(&code, 0, 0x3f800000u);
AppendVMovU32(&code, 1, 0);
AppendVop3(&code, 0x01, 20, Vgpr(0), Vgpr(1), 0, 0, 0, false, 0,
0x1); // -1.0 < 0.0
AppendStoreSgprPair(&code, 20, 0);
AppendEnd(&code);
return {"VectorVop3FloatCompareNegSourceModifier",
code,
{},
{1, 0},
{O::V_MOV_B32, O::V_CMP_LT_F32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}
TestCase VectorVop3CmpxWritesExecMask() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 0, 2);
AppendVMovU32(&code, 1, 1);
AppendVMovU32(&code, 2, 0);
AppendVMovU32(&code, 30, 0);
AppendVop3(&code, 0xd1, 5, Vgpr(0), Vgpr(1)); // v_cmpx_lt_u32, false
AppendVMovU32(&code, 2, 7);
AppendBufferStoreDword(&code, 2, 30);
AppendEnd(&code);
return {"VectorVop3CmpxWritesExecMask",
code,
{0},
{0},
{O::V_MOV_B32, O::V_CMPX_LT_U32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}
TestCase VectorVopcSdwaCmpxWritesExecMask() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 0, 2);
AppendVMovU32(&code, 1, 1);
AppendVMovU32(&code, 2, 0);
AppendVMovU32(&code, 30, 0);
code.push_back(EncodeVopc(0xd1, 249, 1)); // v_cmpx_lt_u32.sdwa, false
code.push_back(EncodeVopcSdwa(0));
AppendVMovU32(&code, 2, 7);
AppendBufferStoreDword(&code, 2, 30);
AppendEnd(&code);
return {"VectorVopcSdwaCmpxWritesExecMask",
code,
{0},
{0},
{O::V_MOV_B32, O::V_CMPX_LT_U32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}
TestCase VectorVopcCmpxGtU16CapturedSdwaExecMask() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 2, 0x00008000u);
AppendVMovU32(&code, 3, 7);
AppendVMovU32(&code, 31, 0);
code.insert(code.end(), {0x7d7900f9u, 0x86060002u});
AppendBufferStoreDword(&code, 3, 31);
AppendVMovU32(&code, 2, 0x00010000u);
AppendVMovU32(&code, 3, 9);
AppendVMovU32(&code, 31, 4);
code.insert(code.end(), {0x7d7900f9u, 0x86060002u});
AppendBufferStoreDword(&code, 3, 31);
AppendEnd(&code);
TestCase test{"VectorVopcCmpxGtU16CapturedSdwaExecMask",
code,
{0x11111111u, 0x22222222u},
{7, 0x22222222u},
{O::V_MOV_B32, O::V_CMPX_GT_U16, O::BUFFER_STORE_DWORD,
O::S_ENDPGM}};
test.decoded_counts = {{"V_CMPX_GT_U16", 2}};
return test;
}
TestCase VectorVopcCmpxLtU16CapturedSdwaExecMask() {
using O = ShaderOpcode;
struct CompareCase {
u32 lhs;
u32 rhs;
u32 incoming_exec;
u32 expected_exec;
u32 encoding = 0; // Captured SDWA, compact, VOP3, WORD_1 src0/src1.
};
const std::array<CompareCase, 12> cases{{
{0xffff0000u, 0x12340001u, 1, 1}, // Ignore high halves.
{0xabcd0001u, 0x12340001u, 1, 0}, // Strict comparison: equal is false.
{0x0000ffffu, 0x12340001u, 1, 0}, // Unsigned, not signed -1.
{0x00007fffu, 0x12348000u, 1, 1},
{0x00008000u, 0x12347fffu, 1, 0},
{0x12340000u, 0x56780000u, 1, 0},
{0x0000fffeu, 0x1234ffffu, 1, 1},
{0x00000000u, 0x12340001u, 0, 0}, // CMPX cannot reactivate an inactive lane.
{0x00008000u, 0x0000ffffu, 1, 1, 1},
{0x0000ffffu, 0x00008000u, 1, 0, 2},
{0x0001ffffu, 0x12340002u, 1, 1, 3},
{0x00000001u, 0x00020000u, 1, 1, 4},
}};
constexpr u32 vcc_hi = 0x89abcdefu;
TestCase test;
test.name = "VectorVopcCmpxLtU16CapturedSdwaExecMask";
for (const auto &entry : cases) {
test.initial.push_back(entry.lhs);
}
test.expected = test.initial;
auto &code = test.code;
for (u32 i = 0; i < cases.size(); ++i) {
const auto &entry = cases[i];
AppendVMovU32(&code, 30, i * 4u);
AppendBufferLoadDword(&code, 0, 30); // Runtime input prevents constant folding.
AppendVMovU32(&code, 1, entry.rhs);
AppendSMovLiteral(&code, 106, entry.rhs);
AppendSMovLiteral(&code, 107, vcc_hi);
code.push_back(EncodeSMovB32(126, InlineU32(entry.incoming_exec)));
switch (entry.encoding) {
case 1: code.push_back(0x7d720300u); break; // compact v0, v1
case 2: code.insert(code.end(), {0xd4b9007eu, 0x00020300u}); break;
case 3: code.insert(code.end(), {0x7d72d4f9u, 0x86050000u}); break;
case 4: code.insert(code.end(), {0x7d72d4f9u, 0x85060000u}); break;
default: code.insert(code.end(), {0x7d72d4f9u, 0x86060000u}); break;
}
code.push_back(EncodeSMovB32(20, 126)); // Snapshot EXEC before restoring it.
code.push_back(EncodeSMovB32(21, 127));
code.push_back(EncodeSMovB32(126, InlineU32(1)));
const u32 out = static_cast<u32>(cases.size()) + i * 4u;
AppendStoreSgprPair(&code, 20, out);
AppendStoreSgprPair(&code, 106, out + 2u); // CMPX must not overwrite either VCC half.
test.expected.insert(test.expected.end(),
{entry.expected_exec, 0u, entry.rhs, vcc_hi});
}
AppendEnd(&code);
test.opcodes = {O::V_MOV_B32, O::S_MOV_B32, O::BUFFER_LOAD_DWORD,
O::V_CMPX_LT_U16, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.decoded_counts = {{"V_CMPX_LT_U16", cases.size()}};
test.required_spirv = {"OpULessThan"};
return test;
}
TestCase VectorVopcCmpxLtI16CapturedSdwaExecMask() {
using O = ShaderOpcode;
enum Encoding { Captured, Compact, Vop3, HighLhs, HighRhs, SdwaOperands };
struct CompareCase {
u32 lhs;
u32 rhs;
u32 expected_exec;
Encoding encoding = Captured;
u32 incoming_exec = 1;
u32 src0_sel = 6;
u32 src1_sel = 6;
u32 src0_sext = 0;
u32 src1_sext = 0;
};
const std::array<CompareCase, 30> cases{{
{0xffff000eu, 15u, 1}, // Ignore the high half: 14 < 15.
{0x8000000fu, 15u, 0}, // Equality is false despite a negative high half.
{0x00000010u, 15u, 0},
{0x7fff0000u, 15u, 1},
{0x0000ffffu, 15u, 1}, // Signed -1, not unsigned 65535.
{0x00008000u, 15u, 1}, // Minimum signed halfword.
{0x00007fffu, 15u, 0}, // Maximum signed halfword.
{0x0000fff0u, 15u, 1},
{0x0000ffffu, 15u, 0, Captured, 0}, // Cannot reactivate an inactive lane.
{0xffff000fu, 15u, 0, Captured, 0},
{0x1234ffffu, 0xabcd0000u, 1, Compact}, // -1 < 0.
{0xffff0000u, 0x0000ffffu, 0, Compact}, // 0 < -1 is false.
{0xabcd8000u, 0x12347fffu, 1, Vop3},
{0x12347fffu, 0xabcd8000u, 0, Vop3},
{0xffff0010u, 15u, 1, HighLhs}, // Select -1 in WORD_1.
{0x0010ffffu, 15u, 0, HighLhs}, // Select 16, not the negative low half.
{0x1234fffeu, 0xffff8000u, 1, HighRhs}, // -2 < selected -1.
{0x1234ffffu, 0xabcd0000u, 1, SdwaOperands, 1, 4, 4, 1, 1},
{0x12345680u, 0u, 1, SdwaOperands, 1, 0, 6, 1, 0}, // BYTE_0 SEXT: -128.
{0x1234567fu, 0u, 0, SdwaOperands, 1, 0, 6, 1, 0}, // +127.
{0x12345680u, 0u, 0, SdwaOperands, 1, 0, 6, 0, 0}, // Without SEXT: +128.
{0x12348000u, 0u, 1, SdwaOperands, 1, 1, 6, 1, 0},
{0x12803456u, 0u, 1, SdwaOperands, 1, 2, 6, 1, 0},
{0x80345678u, 0u, 1, SdwaOperands, 1, 3, 6, 1, 0},
{0x0000ffffu, 0x12345680u, 0, SdwaOperands, 1, 6, 0, 0, 1}, // -1 < -128 is false.
{0x0000ff7fu, 0x12345680u, 1, SdwaOperands, 1, 6, 0, 0, 1}, // -129 < -128.
{0x0000ffffu, 0x12347f00u, 1, SdwaOperands, 1, 6, 1, 0, 1},
{0u, 0x12803456u, 0, SdwaOperands, 1, 6, 2, 0, 1},
{0u, 0x80345678u, 0, SdwaOperands, 1, 6, 3, 0, 1},
{0xff000001u, 0x000100ffu, 1, SdwaOperands, 1, 3, 2, 1, 1},
}};
constexpr u32 vcc_lo = 0x76543210u;
constexpr u32 vcc_hi = 0x89abcdefu;
TestCase test;
test.name = "VectorVopcCmpxLtI16CapturedSdwaExecMask";
for (const auto &entry : cases) {
test.initial.push_back(entry.lhs);
}
test.expected = test.initial;
auto &code = test.code;
for (u32 i = 0; i < cases.size(); ++i) {
const auto &entry = cases[i];
AppendVMovU32(&code, 30, i * 4u);
AppendBufferLoadDword(&code, 1, 30); // Runtime input prevents constant folding.
AppendVMovU32(&code, 2, entry.rhs);
AppendSMovLiteral(&code, 106, vcc_lo);
AppendSMovLiteral(&code, 107, vcc_hi);
code.push_back(EncodeSMovB32(126, InlineU32(entry.incoming_exec)));
switch (entry.encoding) {
case Compact: code.push_back(0x7d320501u); break;
case Vop3: code.insert(code.end(), {0xd499007eu, 0x00020501u}); break;
case HighLhs: code.insert(code.end(), {0x7d331ef9u, 0x86050001u}); break;
case HighRhs:
code.push_back(EncodeVopc(0x99u, 249u, 2u));
code.push_back(EncodeVopcSdwa(1u, 0u, 0u, 6u, 5u));
break;
case SdwaOperands:
code.push_back(EncodeVopc(0x99u, 249u, 2u));
code.push_back(EncodeVopcSdwa(1u, 0u, 0u, entry.src0_sel, entry.src1_sel,
entry.src0_sext, entry.src1_sext));
break;
default: code.insert(code.end(), {0x7d331ef9u, 0x86060001u}); break;
}
code.push_back(EncodeSMovB32(20, 126)); // Snapshot EXEC before restoring it.
code.push_back(EncodeSMovB32(21, 127));
code.push_back(EncodeSMovB32(126, InlineU32(1)));
const u32 out = static_cast<u32>(cases.size()) + i * 4u;
AppendStoreSgprPair(&code, 20, out);
AppendStoreSgprPair(&code, 106, out + 2u); // CMPX preserves both VCC halves.
test.expected.insert(test.expected.end(),
{entry.expected_exec, 0u, vcc_lo, vcc_hi});
}
AppendEnd(&code);
test.opcodes = {O::V_MOV_B32, O::S_MOV_B32, O::BUFFER_LOAD_DWORD,
O::V_CMPX_LT_I16, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.decoded_counts = {{"V_CMPX_LT_I16", cases.size()}};
test.required_spirv = {"OpSLessThan", "OpBitFieldSExtract"};
return test;
}
TestCase VectorVopcCmpxLtI16WaveMasks(u32 wave_size) {
using O = ShaderOpcode;
constexpr u32 marker = 0xfeedabcdu;
constexpr u32 vcc_lo = 0x01234567u;
constexpr u32 vcc_hi = 0x89abcdefu;
constexpr u32 expected_exec_lo = 0xaaaaaaaau;
const u32 expected_exec_hi = wave_size == 64u ? 0x33333333u : 0u;
TestCase test;
test.name = wave_size == 32u ? "VectorVopcCmpxLtI16Wave32Masks"
: "VectorVopcCmpxLtI16Wave64Masks";
test.initial.assign(6u * wave_size, 0xdeadbeefu);
constexpr u32 inputs[] = {0xffff000eu, 0x0000ffffu, 0x8000000fu, 0x12348000u};
for (u32 lane = 0; lane < wave_size; ++lane) {
test.initial[lane] = inputs[lane % 4u];
}
test.expected = test.initial;
for (u32 lane = 0; lane < wave_size; ++lane) {
test.expected[wave_size + lane] = expected_exec_lo;
test.expected[2u * wave_size + lane] = expected_exec_hi;
test.expected[3u * wave_size + lane] = vcc_lo;
test.expected[4u * wave_size + lane] = vcc_hi;
const u32 mask = lane < 32u ? expected_exec_lo : expected_exec_hi;
if ((mask & (1u << (lane % 32u))) != 0u) {
test.expected[5u * wave_size + lane] = marker;
}
}
auto &code = test.code;
code.push_back(EncodeVop2(0x1a, 30, InlineU32(2), 0));
AppendBufferLoadDword(&code, 1, 30);
AppendVMovLiteral(&code, 4, marker);
code.push_back(EncodeSop1(0x04, 10, 126)); // Save the full incoming EXEC.
AppendSMovLiteral(&code, 106, vcc_lo);
AppendSMovLiteral(&code, 107, vcc_hi);
AppendSMovLiteral(&code, 126, 0xeeeeeeeeu);
AppendSMovLiteral(&code, 127, 0x77777777u);
code.insert(code.end(), {0x7d331ef9u, 0x86060001u});
code.push_back(EncodeSMovB32(20, 126));
code.push_back(EncodeSMovB32(21, 127));
AppendStoreVgprAtLaneDwordOffset(&code, 4, 0, 5u * wave_size);
code.push_back(EncodeSop1(0x04, 126, 10)); // Restore lanes to inspect the raw masks.
AppendStoreSgprAtLaneDwordOffset(&code, 20, 0, wave_size);
AppendStoreSgprAtLaneDwordOffset(&code, 21, 0, 2u * wave_size);
AppendStoreSgprAtLaneDwordOffset(&code, 106, 0, 3u * wave_size);
AppendStoreSgprAtLaneDwordOffset(&code, 107, 0, 4u * wave_size);
AppendEnd(&code);
test.compute_info.threads_num[0] = wave_size;
test.compute_info.threads_num[1] = test.compute_info.threads_num[2] = 1;
test.compute_info.thread_ids_num = 1;
test.compute_info.wave_size = wave_size;
test.has_compute_info = true;
test.opcodes = {O::V_MOV_B32, O::S_MOV_B32, O::S_MOV_B64, O::V_LSHLREV_B32,
O::V_ADD_NC_U32, O::BUFFER_LOAD_DWORD, O::V_CMPX_LT_I16,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.required_spirv = {"OpSLessThan", "OpBitFieldSExtract"};
return test;
}
TestCase VectorVopcCmpxU16SdwaCompactVop3ExecMask(bool not_equal) {
using O = ShaderOpcode;
struct CompareCase {
u32 lhs;
u32 rhs;
u32 incoming_exec;
u32 expected_exec;
u32 encoding = 0; // SDWA, compact, VOP3, sign-extended words/bytes, literal.
};
const std::array<CompareCase, 21> cases{{
{0x12340000u, 0x56780000u, 1, 1}, // Equal low half; ignore high half.
{0xffff0001u, 0xabcd0001u, 1, 1},
{0x00000000u, 0x12340000u, 1, 1},
{0xaaaa5555u, 0xbbbb5555u, 1, 1},
{0x00000000u, 0x00000001u, 1, 0},
{0x0000ffffu, 0x00000000u, 1, 0},
{0x7fff8000u, 0x00018000u, 1, 1}, // Equal low halves despite different high halves.
{0x00000000u, 0x00010000u, 1, 1},
{0x12340001u, 0x12340002u, 1, 0, 1},
{0x00008000u, 0x0000ffffu, 1, 0, 2},
{0x0001ffffu, 0x12340001u, 1, 0, 3}, // Sign-extended low halves differ.
{0x00000001u, 0x00010001u, 0, 0}, // CMPX cannot reactivate an inactive lane.
{0x12345680u, 0x0000ff80u, 1, 1, 4}, // SEXT BYTE_0 is unsigned U16 0xff80.
{0x123456ffu, 0x0000ffffu, 1, 1, 4},
{0x0000ff80u, 0x12348000u, 1, 1, 5}, // SEXT BYTE_1 on src1.
{0x1234ffffu, 0x5678ffffu, 1, 1, 6}, // SEXT WORD_0 versus DWORD low half.
{0xabcd0001u, 1u, 1, 1, 7}, // Literal source, equal low halves.
{0xabcd0000u, 1u, 1, 0, 7},
{0x00000000u, 1u, 0, 0}, // Neither EQ nor NE may reactivate an inactive lane.
{0x00010000u, 0xffff0001u, 1, 1, 8}, // Select the high word of src0.
{0xabcd0002u, 0x0002ffffu, 1, 1, 9}, // Select the high word of src1.
}};
const u32 opcode = not_equal ? 0xbdu : 0xbau;
constexpr u32 vcc_hi = 0x89abcdefu;
TestCase test;
test.name = not_equal ? "VectorVopcCmpxNeU16SdwaCompactVop3ExecMask"
: "VectorVopcCmpxEqU16SdwaCompactVop3ExecMask";
for (const auto &entry : cases) {
test.initial.push_back(entry.lhs);
}
test.expected = test.initial;
auto &code = test.code;
for (u32 i = 0; i < cases.size(); ++i) {
const auto &entry = cases[i];
AppendVMovU32(&code, 30, i * 4u);
AppendBufferLoadDword(&code, 0, 30); // Runtime input prevents constant folding.
AppendVMovU32(&code, 1, entry.rhs);
AppendSMovLiteral(&code, 106, entry.rhs);
AppendSMovLiteral(&code, 107, vcc_hi);
code.push_back(EncodeSMovB32(126, InlineU32(entry.incoming_exec)));
switch (entry.encoding) {
case 1: code.push_back(EncodeVopc(opcode, Vgpr(0), 1u)); break;
case 2: AppendVop3(&code, opcode, 126u, Vgpr(0), Vgpr(1)); break;
case 3:
code.push_back(EncodeVopc(opcode, 249u, 1u));
code.push_back(EncodeVopcSdwa(0u, 0u, 0u, 6u, 6u, 1u, 1u));
break;
case 4:
code.push_back(EncodeVopc(opcode, 249u, 1u));
code.push_back(EncodeVopcSdwa(0u, 0u, 0u, 0u, 6u, 1u, 0u));
break;
case 5:
code.push_back(EncodeVopc(opcode, 249u, 1u));
code.push_back(EncodeVopcSdwa(0u, 0u, 0u, 6u, 1u, 0u, 1u));
break;
case 6:
code.push_back(EncodeVopc(opcode, 249u, 1u));
code.push_back(EncodeVopcSdwa(0u, 0u, 0u, 4u, 6u, 1u, 0u));
break;
case 7:
code.push_back(EncodeVopc(opcode, 255u, 0u));
code.push_back(entry.rhs);
break;
case 8:
case 9:
code.push_back(EncodeVopc(opcode, 249u, 1u));
code.push_back(EncodeVopcSdwa(0u, 0u, 0u, entry.encoding == 8u ? 5u : 6u,
entry.encoding == 9u ? 5u : 6u));
break;
default:
code.push_back(EncodeVopc(opcode, 249u, 1u));
code.push_back(EncodeVopcSdwa(0u));
break;
}
code.push_back(EncodeSMovB32(20, 126)); // Snapshot EXEC before restoring it.
code.push_back(EncodeSMovB32(21, 127));
code.push_back(EncodeSMovB32(126, InlineU32(1u)));
const u32 out = static_cast<u32>(cases.size()) + i * 4u;
AppendStoreSgprPair(&code, 20, out);
AppendStoreSgprPair(&code, 106, out + 2u); // CMPX must not overwrite either VCC half.
const u32 expected_exec = not_equal ? entry.incoming_exec & (entry.expected_exec ^ 1u)
: entry.expected_exec;
test.expected.insert(test.expected.end(), {expected_exec, 0u, entry.rhs, vcc_hi});
}
AppendEnd(&code);
test.opcodes = {O::V_MOV_B32, O::S_MOV_B32, O::BUFFER_LOAD_DWORD,
not_equal ? O::V_CMPX_NE_U16 : O::V_CMPX_EQ_U16,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.decoded_counts = {{not_equal ? "V_CMPX_NE_U16" : "V_CMPX_EQ_U16", cases.size()}};
test.required_spirv = {not_equal ? "OpINotEqual" : "OpIEqual"};
return test;
}
TestCase VectorVopcCmpxEqU16SdwaCompactVop3ExecMask() {
return VectorVopcCmpxU16SdwaCompactVop3ExecMask(false);
}
TestCase VectorVopcCmpxNeU16SdwaCompactVop3ExecMask() {
return VectorVopcCmpxU16SdwaCompactVop3ExecMask(true);
}
TestCase VectorVopcCmpNgtF16CapturedSdwaAndEdges() {
using O = ShaderOpcode;
enum Encoding { Captured, Compact, Vop3NegAbs, HighWord, ScalarDst, SdwaNegAbs };
struct CompareCase {
u32 lhs;
u32 rhs;
u32 expected;
Encoding encoding = Captured;
u32 exec = 1;
};
const std::array<CompareCase, 17> cases{{
{0x7e003c00u, 0, 0}, // +1; ignore NaN in the upper half.
{0x3c00bc00u, 0, 1}, // -1
{0x3c000000u, 0, 1}, // +0
{0x3c008000u, 0, 1}, // -0
{0x3c007e00u, 0, 1}, // Quiet NaN: NGT differs from ordered LE.
{0x3c007c01u, 0, 1}, // Signaling NaN
{0x00007c00u, 0, 0}, // +infinity
{0x0000fc00u, 0, 1}, // -infinity
{0x0000bc00u, 0, 0, Captured, 0}, // Inactive lane contributes no VCC bit.
{0x00003c00u, 0x00007e00u, 1, Compact}, // NaN in src1.
{0x00004000u, 0x00003c00u, 0, Compact}, // +2 > +1
{0x0000bc00u, 0, 1, Vop3NegAbs}, // -abs(-1)
{0x00003c00u, 0, 1, Vop3NegAbs}, // -abs(+1)
{0xbc004000u, 0x3c00c000u, 1, HighWord}, // Select both upper halves.
{0x00004000u, 0x00003c00u, 0, ScalarDst},
{0x00003c00u, 0x00004000u, 1, ScalarDst},
{0x0000bc00u, 0, 1, SdwaNegAbs},
}};
constexpr u32 vcc_sentinel = 0x89abcdefu;
TestCase test;
test.name = "VectorVopcCmpNgtF16CapturedSdwaAndEdges";
for (const auto &entry : cases) {
test.initial.insert(test.initial.end(), {entry.lhs, entry.rhs});
}
test.expected = test.initial;
auto &code = test.code;
for (u32 i = 0; i < cases.size(); ++i) {
const auto &entry = cases[i];
AppendVMovU32(&code, 30, i * 8u);
AppendBufferLoadDword(&code, 21, 30); // Runtime inputs prevent constant folding.
AppendVMovU32(&code, 30, i * 8u + 4u);
AppendBufferLoadDword(&code, 1, 30);
AppendSMovLiteral(&code, 106, vcc_sentinel);
AppendSMovLiteral(&code, 107, vcc_sentinel);
code.push_back(EncodeSMovB32(126, InlineU32(entry.exec)));
switch (entry.encoding) {
case Captured: code.insert(code.end(), {0x7dd700f9u, 0x86060015u}); break;
case Compact: code.push_back(EncodeVopc(0xeb, Vgpr(21), 1)); break;
case Vop3NegAbs:
AppendVop3(&code, 0xeb, 106, Vgpr(21), Vgpr(1), 0, 1, 0, false, 0, 1);
break;
case HighWord:
code.push_back(EncodeVopc(0xeb, 249, 1));
code.push_back(EncodeVopcSdwa(21, 0, 0, 5, 5));
break;
case ScalarDst:
code.push_back(EncodeVopc(0xeb, 249, 1));
code.push_back(EncodeVopcSdwa(21, 22, 1));
break;
case SdwaNegAbs:
code.push_back(EncodeVopc(0xeb, 249, 1));
code.push_back(EncodeVopcSdwa(21, 0, 0, 6, 6, 0, 0, 1, 1));
break;
}
code.push_back(EncodeSMovB32(20, entry.encoding == ScalarDst ? 22 : 106));
code.push_back(EncodeSMovB32(21, 126)); // CMP must preserve EXEC even if false.
code.push_back(EncodeSMovB32(126, InlineU32(1)));
const u32 out = static_cast<u32>(test.initial.size()) + i * 4u;
AppendStoreSgprPair(&code, 20, out);
AppendStoreSgprPair(&code, 106, out + 2u);
test.expected.insert(test.expected.end(),
{entry.expected, entry.exec,
entry.encoding == ScalarDst ? vcc_sentinel : entry.expected,
vcc_sentinel});
}
AppendEnd(&code);
test.opcodes = {O::V_MOV_B32, O::S_MOV_B32, O::BUFFER_LOAD_DWORD,
O::V_CMP_NGT_F16, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.decoded_counts = {{"V_CMP_NGT_F16", cases.size()}};
test.required_spirv = {"OpFUnordLessThanEqual"};
test.compute_info.wave_size = 32;
test.has_compute_info = true;
return test;
}
TestCase VectorVopcCmpNltF16CapturedSdwaAndEdges() {
using O = ShaderOpcode;
enum Encoding { Captured, Compact, Vop3NegAbs, SdwaNegAbs };
struct CompareCase {
u32 packed; // Captured instruction compares low half against high half.
u32 expected;
Encoding encoding = Captured;
u32 exec = 1;
};
const std::array<CompareCase, 16> cases{{
{0x40003c00u, 0}, // +1 < +2
{0x3c004000u, 1}, // +2 > +1
{0x3c003c00u, 1}, // Equal
{0x80000000u, 1}, // +0 versus -0
{0x00008000u, 1}, // -0 versus +0
{0x3c007e00u, 1}, // Quiet NaN in src0: unordered GE, not ordered GE.
{0x7e003c00u, 1}, // Quiet NaN in src1
{0x3c007c01u, 1}, // Signaling NaN in src0
{0x7c013c00u, 1}, // Signaling NaN in src1
{0xfc007c00u, 1}, // +infinity versus -infinity
{0x7c00fc00u, 0}, // -infinity versus +infinity
{0x00003c00u, 0, Captured, 0}, // Inactive lane contributes no mask bit.
{0x40003c00u, 0, Compact},
{0x7e003c00u, 1, Compact},
{0x0000bc00u, 0, Vop3NegAbs}, // -abs(-1) < 0
{0x0000bc00u, 0, SdwaNegAbs},
}};
constexpr u32 sentinel = 0x89abcdefu;
TestCase test;
test.name = "VectorVopcCmpNltF16CapturedSdwaAndEdges";
for (const auto &entry : cases) {
test.initial.push_back(entry.packed);
}
test.expected = test.initial;
auto &code = test.code;
code.push_back(EncodeSMovB32(22, 2)); // Preserve the input buffer descriptor.
code.push_back(EncodeSMovB32(23, 3));
for (u32 i = 0; i < cases.size(); ++i) {
const auto &entry = cases[i];
AppendVMovU32(&code, 30, i * 4u);
AppendBufferLoadDword(&code, 3, 30); // Runtime input prevents constant folding.
AppendVMovU32(&code, 1, entry.packed >> 16u);
for (u32 reg : {3u, 106u, 107u}) {
AppendSMovLiteral(&code, reg, sentinel);
}
code.push_back(EncodeSMovB32(126, InlineU32(entry.exec)));
switch (entry.encoding) {
case Captured: code.insert(code.end(), {0x7ddc06f9u, 0x05068203u}); break;
case Compact: code.push_back(EncodeVopc(0xee, Vgpr(3), 1)); break;
case Vop3NegAbs:
AppendVop3(&code, 0xee, 2, Vgpr(3), Vgpr(1), 0, 1, 0, false, 0, 1);
break;
case SdwaNegAbs:
code.insert(code.end(), {0x7ddc06f9u, 0x05368203u});
break;
}
code.push_back(EncodeSMovB32(20, entry.encoding == Compact ? 106 : 2));
code.push_back(EncodeSMovB32(21, 3)); // Wave32 must preserve adjacent SGPR.
code.push_back(EncodeSMovB32(24, 126));
code.push_back(EncodeSMovB32(25, 127));
code.push_back(EncodeSMovB32(126, InlineU32(1)));
code.push_back(EncodeSMovB32(2, 22));
code.push_back(EncodeSMovB32(3, 23));
const u32 out = static_cast<u32>(cases.size()) + i * 6u;
AppendStoreSgprPair(&code, 20, out);
AppendStoreSgprPair(&code, 24, out + 2u);
AppendStoreSgprPair(&code, 106, out + 4u);
test.expected.insert(test.expected.end(),
{entry.expected, sentinel, entry.exec, 0u,
entry.encoding == Compact ? entry.expected : sentinel,
sentinel});
}
AppendEnd(&code);
test.opcodes = {O::V_MOV_B32, O::S_MOV_B32, O::BUFFER_LOAD_DWORD,
O::V_CMP_NLT_F16, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.decoded_counts = {{"V_CMP_NLT_F16", cases.size()}};
test.required_spirv = {"OpFUnordGreaterThanEqual"};
test.compute_info.wave_size = 32;
test.has_compute_info = true;
return test;
}
TestCase VectorVopcCmpxNgtF16CapturedSdwaExecMask() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovLiteral(&code, 31, 0x7e003c00u);
AppendVMovU32(&code, 3, 7);
AppendVMovU32(&code, 30, 0);
code.insert(code.end(), {0x7df700f9u, 0x8605001fu});
AppendBufferStoreDword(&code, 3, 30);
AppendVMovLiteral(&code, 31, 0x3c007e00u);
AppendVMovU32(&code, 3, 9);
AppendVMovU32(&code, 30, 4);
code.insert(code.end(), {0x7df700f9u, 0x8605001fu});
AppendBufferStoreDword(&code, 3, 30);
AppendEnd(&code);
TestCase test{"VectorVopcCmpxNgtF16CapturedSdwaExecMask",
code,
{0x11111111u, 0x22222222u},
{7, 0x22222222u},
{O::V_MOV_B32, O::V_CMPX_NGT_F16, O::BUFFER_STORE_DWORD,
O::S_ENDPGM}};
test.decoded_counts = {{"V_CMPX_NGT_F16", 2}};
return test;
}
TestCase VectorVopcCmpxNleF16CapturedSdwaExecMask() {
using O = ShaderOpcode;
struct CompareCase { u32 value, expected, exec = 1; bool vop3 = false; };
const std::array<CompareCase, 10> cases{{
{0x3c007e00u, 1}, {0xbc003c00u, 0}, // High half +1/-1; opposite low halves.
{0x00003c00u, 0}, {0x80003c00u, 0}, // Signed zeros.
{0x7e003c00u, 1}, {0x7c013c00u, 1}, // Quiet/signaling NaN are unordered.
{0x7c003c00u, 1}, {0xfc003c00u, 0}, // Infinities.
{0x3c000000u, 0, 0}, // Inactive lane cannot enter EXEC.
{0x0000bc00u, 1, 1, true}, // VOP3 negates -1 before comparison.
}};
constexpr u32 sentinel = 0x89abcdefu;
TestCase test;
test.name = "VectorVopcCmpxNleF16CapturedSdwaExecMask";
for (const auto &entry : cases) test.initial.push_back(entry.value);
test.expected = test.initial;
auto &code = test.code;
for (u32 i = 0; i < cases.size(); ++i) {
const auto &entry = cases[i];
AppendVMovU32(&code, 30, i * 4u);
AppendBufferLoadDword(&code, 5, 30);
AppendSMovLiteral(&code, 106, sentinel);
AppendSMovLiteral(&code, 107, sentinel);
code.push_back(EncodeSMovB32(126, InlineU32(entry.exec)));
if (entry.vop3) {
AppendVop3(&code, 0xfc, 106, Vgpr(5), InlineU32(0), 0, 0, 0, false, 0, 1);
} else {
code.insert(code.end(), {0x7df900f9u, 0x86050005u});
}
code.push_back(EncodeSMovB32(8, 126));
code.push_back(EncodeSMovB32(126, InlineU32(1)));
const u32 out = test.expected.size();
code.push_back(EncodeVop1(0x01, 0, 8));
AppendStoreVgpr(&code, 0, out);
AppendStoreSgprPair(&code, 106, out + 1u);
test.expected.insert(test.expected.end(), {entry.expected, sentinel, sentinel});
}
AppendEnd(&code);
test.opcodes = {O::V_MOV_B32, O::S_MOV_B32, O::BUFFER_LOAD_DWORD,
O::V_CMPX_NLE_F16, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.decoded_counts = {{"V_CMPX_NLE_F16", cases.size()}};
test.required_spirv = {"OpFUnordGreaterThan"};
return test;
}
TestCase VectorCompareInvertedMaskSelect() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovLiteral(&code, 0, 0x12345678u);
code.push_back(EncodeVopc(0xc7, Vgpr(0), 0)); // v_cmp_t_u32 v0, v0
code.push_back(EncodeSop1(0x08, 106, 106)); // s_not_b64 vcc, vcc
code.push_back(EncodeVop2(0x01, 1, Vgpr(0), 128)); // v_cndmask_b32 v1, v0, 0
AppendBufferStoreDword(&code, 1, 30);
AppendEnd(&code);
return {"VectorCompareInvertedMaskSelect",
code,
{},
{0x12345678u},
{O::V_MOV_B32, O::V_CMP_T_U32, O::S_NOT_B64, O::V_CNDMASK_B32,
O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}
TestCase BranchSelect() {
using O = ShaderOpcode;
std::vector<u32> code = {
EncodeSMovB32(0, InlineU32(0)),
EncodeSopc(0x06, 0, InlineU32(0)),
EncodeSopp(0x05, 2),
EncodeSMovB32(1, InlineU32(1)),
EncodeSopp(0x02, 1),
EncodeSMovB32(1, InlineU32(7)),
EncodeSop2(0x0a, 2, 1, InlineU32(2)),
EncodeVop1(0x01, 0, 2),
};
AppendBufferStoreDword(&code, 0, 30);
AppendEnd(&code);
return {"BranchSelect",
code,
{},
{7},
{O::S_MOV_B32, O::S_CMP_EQ_U32, O::S_CBRANCH_SCC1, O::S_BRANCH,
O::S_CSELECT_B32, O::V_MOV_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}
TestCase SimpleLoop() {
using O = ShaderOpcode;
std::vector<u32> code = {
EncodeSMovB32(0, InlineU32(0)),
EncodeSopc(0x0a, 0, InlineU32(4)),
EncodeSopp(0x04, 2),
EncodeSop2(0x00, 0, 0, InlineU32(1)),
EncodeSopp(0x02, 0xfffcu),
EncodeVop1(0x01, 0, 0),
};
AppendBufferStoreDword(&code, 0, 30);
AppendEnd(&code);
return {"SimpleLoop",
code,
{},
{4},
{O::S_MOV_B32, O::S_CMP_LT_U32, O::S_CBRANCH_SCC0, O::S_ADD_U32,
O::S_BRANCH, O::V_MOV_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}
TestCase SharedReturnKeepsSelectedValues() {
using O = ShaderOpcode;
std::vector<u32> code = {
EncodeVop1(0x01, 5, 4),
EncodeVop2(0x1a, 5, InlineU32(6), 5),
EncodeVop2(0x25, 5, Vgpr(0), 5),
EncodeSMovB32(8, InlineU32(10)),
EncodeSopc(0x06, 4, InlineU32(0)),
0,
EncodeSMovB32(8, InlineU32(20)),
EncodeSopc(0x06, 4, InlineU32(1)),
0,
EncodeSMovB32(8, InlineU32(30)),
};
AppendStoreSgprAtLaneDwordOffset(&code, 8, 5, 0);
AppendEnd(&code);
const auto shared_return = static_cast<u32>(code.size());
AppendStoreSgprAtLaneDwordOffset(&code, 8, 5, 0);
AppendEnd(&code);
code[5] = EncodeSopp(0x05, shared_return - 6u);
code[8] = EncodeSopp(0x05, shared_return - 9u);
TestCase test;
test.name = "SharedReturnKeepsSelectedValues";
test.code = code;
for (u32 group = 0; group < 3; group++) {
test.expected.insert(test.expected.end(), 64, (group + 1u) * 10u);
}
test.opcodes = {O::V_MOV_B32, O::V_LSHLREV_B32, O::V_ADD_NC_U32,
O::S_MOV_B32, O::S_CMP_EQ_U32, O::S_CBRANCH_SCC1,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.required_spirv = {"OpSelectionMerge"};
test.forbidden_spirv = {"OpSwitch"};
test.compute_info.threads_num[0] = 64;
test.compute_info.threads_num[1] = 1;
test.compute_info.threads_num[2] = 1;
test.compute_info.group_id[0] = true;
test.compute_info.thread_ids_num = 1;
test.compute_info.workgroup_register = 4;
test.has_compute_info = true;
test.dispatch_x = 3;
return test;
}
TestCase SiblingSharedExitKeepsCapturedConditions() {
using O = ShaderOpcode;
std::vector<u32> code = {
EncodeVop1(0x01, 5, 4),
EncodeVop2(0x1a, 5, InlineU32(6), 5),
EncodeVop2(0x25, 5, Vgpr(0), 5), // group * 64 + lane
EncodeSop2(0x0e, 6, 4, InlineU32(1)),
0, // choose left or right
EncodeSMovB32(8, InlineU32(20)),
EncodeSop2(0x0e, 6, 4, InlineU32(2)),
0, // left: skip nested exit
EncodeSop2(0x0e, 6, 4, InlineU32(4)),
0, // nested left: shared exit
0, // left -> normal return
EncodeSMovB32(8, InlineU32(30)),
EncodeSop2(0x0e, 6, 4, InlineU32(8)),
0, // first right shared exit
EncodeSMovB32(8, InlineU32(40)),
EncodeSop2(0x0e, 6, 4, InlineU32(16)),
0, // second right shared exit
EncodeSMovB32(8, InlineU32(50)),
EncodeSopc(0x06, InlineU32(0), InlineU32(1)), // overwrite SCC
};
AppendStoreSgprAtLaneDwordOffset(&code, 8, 5, 0);
AppendEnd(&code);
const auto shared_exit = static_cast<u32>(code.size());
code.push_back(EncodeSop2(0x00, 8, 8, InlineU32(1))); // overwrite SCC again
AppendStoreSgprAtLaneDwordOffset(&code, 8, 5, 0);
AppendEnd(&code);
code[4] = EncodeSopp(0x04, 11u - 5u);
code[7] = EncodeSopp(0x04, 10u - 8u);
code[9] = EncodeSopp(0x04, shared_exit - 10u);
code[10] = EncodeSopp(0x02, 17u - 11u);
code[13] = EncodeSopp(0x04, shared_exit - 14u);
code[16] = EncodeSopp(0x04, shared_exit - 17u);
TestCase test;
test.name = "SiblingSharedExitKeepsCapturedConditions";
test.code = std::move(code);
for (u32 group = 0; group < 32; ++group) {
u32 result = 50;
if (group & 1u) {
if ((group & 2u) && !(group & 4u)) result = 21;
} else if (!(group & 8u)) {
result = 31;
} else if (!(group & 16u)) {
result = 41;
}
test.expected.insert(test.expected.end(), 64, result);
}
test.opcodes = {O::V_MOV_B32, O::V_LSHLREV_B32, O::V_ADD_NC_U32,
O::S_AND_B32, O::S_MOV_B32, O::S_CMP_EQ_U32, O::S_ADD_U32,
O::S_CBRANCH_SCC0, O::S_BRANCH, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.required_spirv = {"OpSelectionMerge"};
test.forbidden_spirv = {"OpSwitch"};
test.compute_info.wave_size = 64;
test.compute_info.threads_num[0] = 64;
test.compute_info.threads_num[1] = 1;
test.compute_info.threads_num[2] = 1;
test.compute_info.group_id[0] = true;
test.compute_info.thread_ids_num = 1;
test.compute_info.workgroup_register = 4;
test.has_compute_info = true;
test.dispatch_x = 32;
return test;
}
TestCase BranchVccnzUsesWaveMask() {
using O = ShaderOpcode;
std::vector<u32> code = {
EncodeVop1(0x01, 1, 0),
EncodeVop2(0x1a, 1, InlineU32(2), 1),
EncodeVop2(0x25, 1, Vgpr(0), 1),
EncodeVop2(0x1a, 3, InlineU32(2), 1),
EncodeVopc(0xc2, InlineU32(0), 0),
EncodeSopp(0x07, 2),
EncodeVop1(0x01, 2, InlineU32(11)),
EncodeSopp(0x02, 1),
};
AppendVMovU32(&code, 2, 42);
AppendBufferStoreDword(&code, 2, 3);
AppendEnd(&code);
TestCase test;
test.name = "BranchVccnzUsesWaveMask";
test.code = code;
test.expected = std::vector<u32>(8, 42);
test.opcodes = {O::V_MOV_B32, O::V_LSHLREV_B32, O::V_ADD_NC_U32,
O::V_CMP_EQ_U32, O::S_CBRANCH_VCCNZ, O::S_BRANCH,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.compute_info.threads_num[0] = 4;
test.compute_info.threads_num[1] = 1;
test.compute_info.threads_num[2] = 1;
test.compute_info.group_id[0] = true;
test.compute_info.thread_ids_num = 1;
test.compute_info.workgroup_register = 0;
test.has_compute_info = true;
test.dispatch_x = 2;
return test;
}
TestCase ScalarBranchRestoresInactiveLanes(u32 wave_size, u32 threads, bool upper_nonzero = false) {
using O = ShaderOpcode;
TestCase test;
if (upper_nonzero) {
test.name = wave_size == 32 ? "ScalarBranchRestoresLastLaneWave32"
: "ScalarBranchRestoresUpperHalfWave64";
} else if (threads < wave_size) {
test.name = wave_size == 32 ? "ScalarBranchRestoresPartialWave32"
: "ScalarBranchRestoresPartialWave64";
} else {
test.name = wave_size == 32 ? "ScalarBranchRestoresInactiveWave32"
: "ScalarBranchRestoresInactiveWave64";
}
auto &code = test.code;
code = {EncodeVop1(0x01, 1, InlineU32(32)),
EncodeVop2(0x25, 1, Vgpr(0), 1),
EncodeSop1(0x04, 8, 126), // Save all participating lanes.
EncodeSMovB32(20, InlineU32(0))};
AppendSMovLiteral(&code, 126, upper_nonzero ? (wave_size == 32 ? 0x80000000u : 0u) : 1u);
AppendSMovLiteral(&code, 127, upper_nonzero && wave_size == 64 ? 0x80000000u : 0u);
if (upper_nonzero) {
code.push_back(EncodeSopp(0x09, 1)); // Last lane makes the whole-wave EXEC nonzero.
code.push_back(EncodeSopp(0x02, 3));
} else {
code.push_back(EncodeSopp(0x08, 3)); // EXEC is nonzero for the whole wave.
}
code.push_back(EncodeSop1(0x04, 126, 8));
// The branch restores inactive lanes before reading the last lane's value.
AppendVop3(&code, 0x360, 20, Vgpr(1), InlineU32(threads - 1));
code.push_back(EncodeSop1(0x04, 126, 8));
AppendStoreSgprAtLaneDwordOffset(&code, 20, 0, 0);
AppendEnd(&code);
test.initial.assign(threads, 0);
test.expected.assign(threads, 32 + threads - 1);
test.opcodes = {O::V_MOV_B32, O::V_ADD_NC_U32, O::S_MOV_B32, O::S_MOV_B64,
upper_nonzero ? O::S_CBRANCH_EXECNZ : O::S_CBRANCH_EXECZ,
O::V_READLANE_B32, O::V_LSHLREV_B32,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.required_spirv = {upper_nonzero ? "OpGroupNonUniformAny" : "OpGroupNonUniformAll"};
test.compute_info.wave_size = wave_size;
test.compute_info.threads_num[0] = threads;
test.compute_info.thread_ids_num = 1;
test.has_compute_info = true;
return test;
}
TestCase ScalarMemRealtimeCapturedPlaceholder() {
using O = ShaderOpcode;
namespace D = ShaderRecompiler::Decoder;
std::vector<u32> code = {0xf4940300u, 0xfa000000u};
D::Instruction decoded;
D::DecodeInstruction(code, 0, decoded);
Require("ScalarMemRealtimeCapturedPlaceholder", "decode",
decoded.opcode == O::S_MEMREALTIME && decoded.word_count == 2 &&
decoded.dst.kind == D::OperandKind::Sgpr && decoded.dst.reg == 12 &&
decoded.data_dwords == 2 && decoded.src_count == 0 &&
decoded.src0.kind == D::OperandKind::Unknown &&
decoded.src1.kind == D::OperandKind::Unknown,
"captured clock instruction must write s12:s13 without memory operands");
AppendStoreSgprPair(&code, 12, 0);
code.insert(code.end(), {0xf4940300u, 0xfa000000u});
AppendStoreSgprPair(&code, 12, 2);
AppendEnd(&code);
return {"ScalarMemRealtimeCapturedPlaceholder",
code,
{},
{UINT32_MAX, UINT32_MAX, UINT32_MAX, UINT32_MAX},
{O::S_MEMREALTIME, O::V_MOV_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}
TestCase ScalarMemoryLoadVariants() {
using O = ShaderOpcode;
struct Load {
u32 opcode;
u32 dst;
u32 byte_offset;
u32 dwords;
};
const Load s_loads[] = {
{0x00, 20, 0, 1}, {0x01, 22, 4, 2}, {0x02, 24, 12, 4},
{0x03, 28, 28, 8}, {0x04, 36, 60, 16},
};
const Load s_buffer_loads[] = {
{0x08, 52, 0, 1}, {0x09, 54, 4, 2}, {0x0a, 56, 12, 4},
{0x0b, 60, 28, 8}, {0x0c, 68, 60, 16},
};
std::vector<u32> code;
code.push_back(EncodeSMovB32(0, InlineU32(0)));
for (const auto &load : s_loads) {
AppendSmemLoadOpcode(&code, load.opcode, load.dst, load.byte_offset);
}
for (const auto &load : s_buffer_loads) {
AppendSmemLoadOpcode(&code, load.opcode, load.dst, load.byte_offset);
}
u32 out = 0;
for (const auto &load : s_loads) {
for (u32 i = 0; i < load.dwords; i++) {
AppendStoreSgpr(&code, load.dst + i, out++);
}
}
for (const auto &load : s_buffer_loads) {
for (u32 i = 0; i < load.dwords; i++) {
AppendStoreSgpr(&code, load.dst + i, out++);
}
}
AppendEnd(&code);
std::vector<u32> initial;
for (u32 i = 0; i < 31u; i++) {
initial.push_back(0x10000000u + i);
}
std::vector<u32> expected = initial;
expected.insert(expected.end(), initial.begin(), initial.end());
return {"ScalarMemoryLoadVariants",
code,
initial,
expected,
{O::S_MOV_B32, O::S_LOAD_DWORD, O::S_LOAD_DWORDX2, O::S_LOAD_DWORDX4,
O::S_LOAD_DWORDX8, O::S_LOAD_DWORDX16, O::S_BUFFER_LOAD_DWORD,
O::S_BUFFER_LOAD_DWORDX2, O::S_BUFFER_LOAD_DWORDX4,
O::S_BUFFER_LOAD_DWORDX8, O::S_BUFFER_LOAD_DWORDX16, O::V_MOV_B32,
O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}
TestCase ScalarBufferOffsetAlignmentAndCarry() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendSMovLiteral(&code, 12, 0xfffffffcu);
code.push_back(EncodeSop2(0x00, 12, 12, 4));
code.push_back(EncodeSmem0(0x08, 20));
code.push_back(EncodeSmem1(4, 12));
AppendSMovLiteral(&code, 12, 3u);
code.push_back(EncodeSop2(0x00, 12, 12, 4));
code.push_back(EncodeSmem0(0x08, 21));
code.push_back(EncodeSmem1(1, 12));
AppendStoreSgpr(&code, 20, 0);
AppendStoreSgpr(&code, 21, 1);
AppendEnd(&code);
TestCase test{"ScalarBufferOffsetAlignmentAndCarry", code,
{0x11111111u, 0x22222222u}, {0u, 0x11111111u},
{O::S_MOV_B32, O::S_ADD_U32, O::S_BUFFER_LOAD_DWORD,
O::V_MOV_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
test.forbidden_spirv = {"flattened_srt"};
test.compute_info.threads_num[0] = 1;
test.compute_info.threads_num[1] = 1;
test.compute_info.threads_num[2] = 1;
test.compute_info.group_id[0] = true;
test.compute_info.workgroup_register = 4;
test.has_compute_info = true;
return test;
}
TestCase ScalarLoadSignedImmediateOffsetAddsSoffset() {
using O = ShaderOpcode;
std::vector<u32> code;
code.push_back(EncodeSMovB32(0, InlineU32(8)));
code.push_back(EncodeSMovB32(2, InlineU32(0)));
code.push_back(EncodeSMovB32(3, InlineU32(0)));
code.push_back(EncodeSmem0(0x00, 1, 1));
code.push_back(EncodeSmem1(0x1ffffcu, 0));
AppendStoreSgpr(&code, 1, 0);
AppendEnd(&code);
return {"ScalarLoadSignedImmediateOffsetAddsSoffset",
code,
{0x11111111u, 0x22222222u},
{0x22222222u, 0x22222222u},
{O::S_MOV_B32, O::S_LOAD_DWORD, O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}
TestCase BufferLoadStore() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendBufferLoadDword(&code, 0, 30);
code.push_back(EncodeVop1(0x01, 31, InlineU32(4)));
AppendBufferStoreDword(&code, 0, 31);
AppendEnd(&code);
return {
"BufferLoadStore",
code,
{0x11223344u, 0},
{0x11223344u, 0x11223344u},
{O::BUFFER_LOAD_DWORD, O::V_MOV_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}
TestCase BufferLoadDwordOffenIdxenUsesVaddrPlusOneOffset() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 20, 0);
AppendVMovU32(&code, 21, 8);
code.push_back(EncodeMubuf0(0x0cu, 0, true, true));
code.push_back(EncodeMubuf1(0, 0, 20));
AppendStoreVgpr(&code, 0, 0);
AppendEnd(&code);
return {
"BufferLoadDwordOffenIdxenUsesVaddrPlusOneOffset",
code,
{0x11111111u, 0x22222222u, 0x33333333u},
{0x33333333u, 0x22222222u, 0x33333333u},
{O::V_MOV_B32, O::BUFFER_LOAD_DWORD, O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}
TestCase BufferStoreDwordOffenIdxenUsesVaddrPlusOneOffset() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 20, 0);
AppendVMovU32(&code, 21, 12);
AppendVMovLiteral(&code, 0, 0xabcdef01u);
code.push_back(EncodeMubuf0(0x1cu, 0, true, true));
code.push_back(EncodeMubuf1(0, 0, 20));
AppendEnd(&code);
return {"BufferStoreDwordOffenIdxenUsesVaddrPlusOneOffset",
code,
{0x11111111u, 0x22222222u, 0x33333333u, 0x44444444u},
{0x11111111u, 0x22222222u, 0x33333333u, 0xabcdef01u},
{O::V_MOV_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}
TestCase BufferLoadDwordNoAddressFlagsIgnoresVaddr() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 20, 8);
code.push_back(EncodeMubuf0(0x0cu, 0, false, false));
code.push_back(EncodeMubuf1(0, 0, 20));
AppendStoreVgpr(&code, 0, 3);
AppendEnd(&code);
return {
"BufferLoadDwordNoAddressFlagsIgnoresVaddr",
code,
{0x11111111u, 0x22222222u, 0x33333333u, 0},
{0x11111111u, 0x22222222u, 0x33333333u, 0x11111111u},
{O::V_MOV_B32, O::BUFFER_LOAD_DWORD, O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}
TestCase BufferLoadDwordIdxenUsesDescriptorStride() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 20, 2);
code.push_back(EncodeMubuf0(0x0cu, 0, true, false));
code.push_back(EncodeMubuf1(0, 0, 20));
AppendStoreVgpr(&code, 0, 0);
AppendEnd(&code);
TestCase test;
test.name = "BufferLoadDwordIdxenUsesDescriptorStride";
test.code = code;
test.initial = {0x11111111u, 0x22222222u, 0x33333333u,
0x44444444u, 0x55555555u, 0x66666666u,
0x77777777u, 0x88888888u, 0x99aabbccu};
test.expected = {0x99aabbccu, 0x22222222u, 0x33333333u,
0x44444444u, 0x55555555u, 0x66666666u,
0x77777777u, 0x88888888u, 0x99aabbccu};
test.opcodes = {O::V_MOV_B32, O::BUFFER_LOAD_DWORD, O::BUFFER_STORE_DWORD,
O::S_ENDPGM};
test.user_data = MakeStructuredStorageBufferData(16, 3);
test.has_user_data = true;
return test;
}
TestCase BufferStoreDwordIdxenUsesDescriptorStride() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 20, 2);
AppendVMovLiteral(&code, 0, 0xabcdef01u);
code.push_back(EncodeMubuf0(0x1cu, 0, true, false));
code.push_back(EncodeMubuf1(0, 0, 20));
AppendEnd(&code);
TestCase test;
test.name = "BufferStoreDwordIdxenUsesDescriptorStride";
test.code = code;
test.initial = {0x11111111u, 0x22222222u, 0x33333333u,
0x44444444u, 0x55555555u, 0x66666666u,
0x77777777u, 0x88888888u, 0x99aabbccu};
test.expected = {0x11111111u, 0x22222222u, 0x33333333u,
0x44444444u, 0x55555555u, 0x66666666u,
0x77777777u, 0x88888888u, 0xabcdef01u};
test.opcodes = {O::V_MOV_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.user_data = MakeStructuredStorageBufferData(16, 3);
test.has_user_data = true;
return test;
}
TestCase BufferStoreDwordAppliesHostOffset() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovLiteral(&code, 0, 0xabcdef01u);
code.push_back(EncodeMubuf0(0x1cu, 2, false, false));
code.push_back(EncodeMubuf1(0, 0, 20));
AppendEnd(&code);
TestCase test;
test.name = "BufferStoreDwordAppliesHostOffset";
test.code = code;
test.initial = {0x11111111u, 0x22222222u, 0x33333333u, 0};
test.expected = {0x11111111u, 0xabcdef01u, 0x33333333u, 0};
test.storage_buffer_range_bytes = 8;
test.storage_buffer_offsets = {2};
test.opcodes = {O::V_MOV_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.user_data = MakeStructuredStorageBufferData(0, 8);
test.has_user_data = true;
return test;
}
TestCase BufferOffsetsUsePackedWords(bool storage) {
using O = ShaderOpcode;
constexpr std::array offsets{0u, 12u, 20u, 28u, 36u};
TestCase test;
test.name = storage ? "BufferOffsetsPackedWordsStorage" : "BufferOffsetsPackedWordsPush";
test.initial.resize(10);
test.expected = test.initial;
for (u32 i = 0; i < offsets.size(); ++i) {
const auto base = i * 4u;
AppendSMovLiteral(&test.code, base, 0x1000u + base * 0x1000u);
AppendSMovLiteral(&test.code, base + 1u, 4u << 16u);
AppendSMovLiteral(&test.code, base + 2u, 1u);
AppendSMovLiteral(&test.code, base + 3u, 1u << 24u);
const auto value = 0x12345678u + i;
AppendVMovLiteral(&test.code, i, value);
test.code.push_back(EncodeMubuf0(0x1cu, 0, false, false));
test.code.push_back(EncodeMubuf1(i, i, 20));
test.expected[offsets[i] / 4u] = value;
}
AppendEnd(&test.code);
test.storage_buffer_range_bytes = 4;
test.storage_buffer_offsets.assign(offsets.begin(), offsets.end());
test.expand_shader_data_storage = storage;
test.opcodes = {O::S_MOV_B32, O::V_MOV_B32, O::BUFFER_STORE_DWORD,
O::S_ENDPGM};
return test;
}
TestCase BufferLoadVariants() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 20, 0);
AppendBufferLoadOpcode(&code, 0x08, 0, 20);
AppendVMovU32(&code, 20, 2);
AppendBufferLoadOpcode(&code, 0x09, 1, 20);
AppendVMovU32(&code, 20, 0);
AppendBufferLoadOpcode(&code, 0x0a, 2, 20);
AppendVMovU32(&code, 20, 2);
AppendBufferLoadOpcode(&code, 0x0b, 3, 20);
AppendVMovU32(&code, 20, 8);
AppendBufferLoadOpcode(&code, 0x0d, 4, 20);
AppendBufferLoadOpcode(&code, 0x0f, 6, 20);
AppendBufferLoadOpcode(&code, 0x0e, 9, 20);
for (u32 i = 0; i < 13u; i++) {
AppendStoreVgpr(&code, i, i);
}
AppendEnd(&code);
return {"BufferLoadVariants",
code,
{0x80ff7f01u, 0x00008001u, 0x11223344u, 0x55667788u, 0x99aabbccu,
0xddeeff00u},
{0x01u, 0xffffffffu, 0x7f01u, 0xffff80ffu, 0x11223344u, 0x55667788u,
0x11223344u, 0x55667788u, 0x99aabbccu, 0x11223344u, 0x55667788u,
0x99aabbccu, 0xddeeff00u},
{O::BUFFER_LOAD_UBYTE, O::BUFFER_LOAD_SBYTE, O::BUFFER_LOAD_USHORT,
O::BUFFER_LOAD_SSHORT, O::BUFFER_LOAD_DWORDX2,
O::BUFFER_LOAD_DWORDX3, O::BUFFER_LOAD_DWORDX4, O::V_MOV_B32,
O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}
TestCase BufferLoadD16Captured(ShaderOpcode opcode, u32 wave_size = 32) {
const bool high = opcode == ShaderOpcode::BUFFER_LOAD_SHORT_D16_HI;
const bool byte = opcode == ShaderOpcode::BUFFER_LOAD_UBYTE_D16;
TestCase test;
test.name = byte ? (wave_size == 64 ? "BufferLoadUbyteD16Captured64" : "BufferLoadUbyteD16Captured32")
: high ? "BufferLoadShortD16HiCaptured" : "BufferLoadShortD16Captured";
auto& code = test.code;
const u32 data_reg = byte ? 0u : high ? 20u : 1u;
const u32 index_reg = byte ? 3u : high ? 21u : 15u;
const std::array<u32, 2> load = byte ? std::array{0xe080e000u, 0x80040003u}
: high ? std::array{0xe0942006u, 0x80021415u}
: std::array{0xe090200eu, 0x8000010fu};
AppendVMovU32(&code, index_reg, 1);
AppendVMovLiteral(&code, data_reg, 0xabcd1234u);
// Load both halves of v20 through s8:s11.
if (high) code.insert(code.end(), {0xe090200eu, 0x80021415u});
code.insert(code.end(), load.begin(), load.end());
AppendStoreVgpr(&code, data_reg, 0);
AppendVMovLiteral(&code, data_reg, 0xdeaf4321u);
code.push_back(EncodeSop1(0x04, 32, 126));
code.push_back(EncodeSop1(0x04, 126, InlineU32(0)));
code.insert(code.end(), load.begin(), load.end());
code.push_back(EncodeSop1(0x04, 126, 32));
AppendStoreVgpr(&code, data_reg, 1);
AppendVMovU32(&code, index_reg, 2);
AppendVMovLiteral(&code, data_reg, 0xbeef5678u);
code.insert(code.end(), load.begin(), load.end());
AppendStoreVgpr(&code, data_reg, 2);
if (byte) {
AppendVMovU32(&code, index_reg, 1);
for (u32 offset = 1; offset <= 3; ++offset) {
AppendVMovLiteral(&code, data_reg, 0xa1b2ffffu);
code.insert(code.end(), {load[0] | offset, load[1]});
AppendStoreVgpr(&code, data_reg, 2 + offset);
}
code.insert(code.end(), {load[0], load[1] | (index_reg << 8u)});
AppendStoreVgpr(&code, index_reg, 6);
}
AppendEnd(&code);
const u32 stride = high ? 16u : 32u;
test.initial.assign(high ? 12u : 20u, 0xa5a5a5a5u);
test.initial[byte ? 8u : high ? 5u : 11u] = byte ? 0xff007f80u : 0x8001cafeu;
if (high) test.initial[7] = 0xcafefaceu;
test.expected = test.initial;
test.expected[0] = byte ? 0xabcd0080u : high ? 0x8001cafeu : 0xabcd8001u;
test.expected[1] = 0xdeaf4321u;
test.expected[2] = high ? 0x00005678u : 0xbeef0000u;
test.user_data = MakeStructuredStorageBufferData(stride, 2);
if (high) std::copy_n(test.user_data.begin(), 4, test.user_data.begin() + 8);
if (byte) {
std::copy_n(test.user_data.begin(), 4, test.user_data.begin() + 16);
test.expected[3] = 0xa1b2007fu;
test.expected[4] = 0xa1b20000u;
test.expected[5] = 0xa1b200ffu;
test.expected[6] = 0x80u;
test.compute_info.threads_num[0] = test.compute_info.threads_num[1] =
test.compute_info.threads_num[2] = 1;
test.compute_info.wave_size = static_cast<uint8_t>(wave_size);
test.has_compute_info = true;
test.required_spirv = {"Volatile"};
}
test.user_data[50] = static_cast<u32>(test.initial.size() * sizeof(u32));
test.user_data[51] = 3u << 28u;
test.has_user_data = true;
test.storage_buffer_range_bytes = stride * 2u;
test.opcodes = {ShaderOpcode::V_MOV_B32, ShaderOpcode::S_MOV_B64,
opcode, ShaderOpcode::BUFFER_STORE_DWORD, ShaderOpcode::S_ENDPGM};
if (high) test.opcodes.push_back(ShaderOpcode::BUFFER_LOAD_SHORT_D16);
test.decoded_counts = {{byte ? "BUFFER_LOAD_UBYTE_D16 v0.sdwa(sel=4,sext=0)"
: high ? "BUFFER_LOAD_SHORT_D16_HI v20.sdwa(sel=5,sext=0)"
: "BUFFER_LOAD_SHORT_D16 v1.sdwa(sel=4,sext=0)", byte ? 6u : 3u}};
return test;
}
TestCase BufferSubwordLoadsAtHostOffset(u32 bytes, u32 component, bool sign) {
using O = ShaderOpcode;
TestCase test;
test.name = bytes == 1 ? (sign ? "BufferSbyteHostOffsetTail" : "BufferUbyteHostOffsetTail")
: sign ? "BufferSshortHostOffsetTail"
: component == 0 ? "BufferUshortHostOffsetFirst"
: component == 2 ? "BufferUshortHostOffsetCarry"
: "BufferUshortHostOffsetTail";
// The captured stride-six index buffer uses USHORT IDXEN offsets 4, 2, 0.
test.code = {EncodeMubuf0((bytes == 1 ? 0x08u : 0x0au) + sign,
component, true, false),
EncodeMubuf1(1, 0, 0),
EncodeMubuf0(0x1cu, 0, true, false), EncodeMubuf1(1, 12, 0)};
AppendEnd(&test.code);
test.initial = {0x800112abu, 0xfedc2345u, 0x80007fffu, 0xfaceffffu,
0xccccccccu, 0xccccccccu, 0xccccccccu};
test.expected = test.initial;
for (u32 record = 0; record < 2; ++record) {
u32 value = 0;
std::memcpy(&value, reinterpret_cast<const uint8_t *>(test.initial.data()) +
bytes + record * 6 + component, bytes);
if (sign && (value & (1u << (bytes * 8u - 1u))) != 0) {
value |= UINT32_MAX << (bytes * 8u);
}
test.expected[4 + record] = value;
}
test.expected[6] = 0;
test.storage_buffer_range_bytes = 12;
test.storage_buffer_offsets = {bytes, 16};
test.user_data = MakeStructuredStorageBufferData(6, 2);
test.user_data[0] = 0x1000u + bytes;
test.user_data[48] = 0x2000u;
test.user_data[49] = 4u << 16u;
test.user_data[50] = 3;
test.user_data[51] = 0x5204;
test.has_user_data = true;
test.compute_info.thread_ids_num = 1;
test.compute_info.threads_num[0] = 3;
test.compute_info.threads_num[1] = test.compute_info.threads_num[2] = 1;
test.has_compute_info = true;
test.opcodes = {bytes == 1 ? (sign ? O::BUFFER_LOAD_SBYTE : O::BUFFER_LOAD_UBYTE)
: (sign ? O::BUFFER_LOAD_SSHORT : O::BUFFER_LOAD_USHORT),
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
return test;
}
TestCase BufferSubwordHostOffsetOverflow() {
auto test = BufferSubwordLoadsAtHostOffset(2, 0, false);
test.name = "BufferSubwordHostOffsetOverflow";
test.code.clear();
AppendVMovLiteral(&test.code, 20, UINT32_MAX);
test.code.insert(test.code.end(),
{EncodeMubuf0(0x0au), EncodeMubuf1(1, 0, 20),
EncodeMubuf0(0x1cu, 0, true, false), EncodeMubuf1(1, 12, 0)});
AppendVMovLiteral(&test.code, 1, 0xbeef);
test.code.insert(test.code.end(), {EncodeMubuf0(0x1au), EncodeMubuf1(1, 0, 20)});
AppendEnd(&test.code);
test.expected[4] = test.expected[5] = 0;
test.opcodes.push_back(ShaderOpcode::V_MOV_B32);
test.opcodes.push_back(ShaderOpcode::BUFFER_STORE_SHORT);
return test;
}
TestCase BufferSubwordStoresAtHostOffset(u32 bytes) {
using O = ShaderOpcode;
TestCase test;
test.name = bytes == 1 ? "BufferByteStoreHostOffsetTail" : "BufferShortStoreHostOffsetTail";
AppendVMovLiteral(&test.code, 1, bytes == 1 ? 0xa5u : 0xcafeu);
test.code.push_back(EncodeMubuf0(bytes == 1 ? 0x18u : 0x1au,
6u - bytes, true, false));
test.code.push_back(EncodeMubuf1(1, 0, 0));
AppendEnd(&test.code);
test.initial = {0x800112abu, 0xfedc2345u, 0x80007fffu, 0xfaceffffu,
0x13579bdfu, 0x2468ace0u};
test.expected = bytes == 1
? std::vector<u32>{0x800112abu, 0xfea52345u, 0x80007fffu, 0xfaceffa5u,
0x13579bdfu, 0x2468ace0u}
: std::vector<u32>{0x800112abu, 0xcafe2345u, 0x80007fffu, 0xfacecafeu,
0x13579bdfu, 0x2468ace0u};
test.storage_buffer_range_bytes = 12;
test.storage_buffer_offsets = {bytes};
test.user_data = MakeStructuredStorageBufferData(6, 2);
test.user_data[0] = 0x1000u + bytes;
test.has_user_data = true;
test.compute_info.thread_ids_num = 1;
test.compute_info.threads_num[0] = 3;
test.compute_info.threads_num[1] = test.compute_info.threads_num[2] = 1;
test.has_compute_info = true;
test.opcodes = {O::V_MOV_B32, bytes == 1 ? O::BUFFER_STORE_BYTE : O::BUFFER_STORE_SHORT,
O::S_ENDPGM};
return test;
}
TestCase BufferLoadDwordx4SnapshotsOverlappingAddress() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 21, 0);
AppendVMovU32(&code, 22, 0);
code.push_back(EncodeMubuf0(0x0eu, 0, true, true));
code.push_back(EncodeMubuf1(21, 0, 21));
for (u32 i = 0; i < 4; i++) {
AppendStoreVgpr(&code, 21 + i, 4 + i);
}
AppendEnd(&code);
TestCase test;
test.name = "BufferLoadDwordx4SnapshotsOverlappingAddress";
test.code = std::move(code);
test.initial = {0x11111111u, 0x22222222u, 0x33333333u, 0x44444444u,
0, 0, 0, 0};
test.expected = {0x11111111u, 0x22222222u, 0x33333333u, 0x44444444u,
0x11111111u, 0x22222222u, 0x33333333u, 0x44444444u};
test.opcodes = {O::V_MOV_B32, O::BUFFER_LOAD_DWORDX4, O::BUFFER_STORE_DWORD,
O::S_ENDPGM};
test.user_data = MakeStructuredStorageBufferData(16, 2);
test.has_user_data = true;
return test;
}
TestCase BufferLoadDwordx2SnapshotsOverlappingAddress() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 21, 0);
AppendVMovU32(&code, 22, 0);
code.push_back(EncodeMubuf0(0x0du, 0, true, true));
code.push_back(EncodeMubuf1(21, 0, 21));
for (u32 i = 0; i < 2; i++) {
AppendStoreVgpr(&code, 21 + i, 2 + i);
}
AppendEnd(&code);
TestCase test;
test.name = "BufferLoadDwordx2SnapshotsOverlappingAddress";
test.code = std::move(code);
test.initial = {0x11111111u, 0x22222222u, 0, 0};
test.expected = {0x11111111u, 0x22222222u, 0x11111111u, 0x22222222u};
test.opcodes = {O::V_MOV_B32, O::BUFFER_LOAD_DWORDX2, O::BUFFER_STORE_DWORD,
O::S_ENDPGM};
test.user_data = MakeStructuredStorageBufferData(8, 2);
test.has_user_data = true;
return test;
}
TestCase BufferLoadDwordx3SnapshotsOverlappingAddress() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 21, 0);
AppendVMovU32(&code, 22, 0);
code.push_back(EncodeMubuf0(0x0fu, 0, true, true));
code.push_back(EncodeMubuf1(21, 0, 21));
for (u32 i = 0; i < 3; i++) {
AppendStoreVgpr(&code, 21 + i, 3 + i);
}
AppendEnd(&code);
TestCase test;
test.name = "BufferLoadDwordx3SnapshotsOverlappingAddress";
test.code = std::move(code);
test.initial = {0x11111111u, 0x22222222u, 0x33333333u, 0, 0, 0};
test.expected = {0x11111111u, 0x22222222u, 0x33333333u,
0x11111111u, 0x22222222u, 0x33333333u};
test.opcodes = {O::V_MOV_B32, O::BUFFER_LOAD_DWORDX3, O::BUFFER_STORE_DWORD,
O::S_ENDPGM};
test.user_data = MakeStructuredStorageBufferData(12, 2);
test.has_user_data = true;
return test;
}
TestCase BufferLoadDwordx4ZeroesOnlyOutOfBoundsTail() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 20, 8);
AppendBufferLoadOpcode(&code, 0x0e, 0, 20);
for (u32 i = 0; i < 4; i++) {
AppendStoreVgpr(&code, i, i);
}
AppendEnd(&code);
TestCase test;
test.name = "BufferLoadDwordx4ZeroesOnlyOutOfBoundsTail";
test.code = std::move(code);
test.initial = {0x11111111u, 0x22222222u, 0x33333333u, 0x44444444u};
test.expected = {0x33333333u, 0x44444444u, 0u, 0u};
test.storage_buffer_range_bytes = 16;
test.opcodes = {O::V_MOV_B32, O::BUFFER_LOAD_DWORDX4, O::BUFFER_STORE_DWORD,
O::S_ENDPGM};
return test;
}
TestCase BufferLoadsGpuSelectedDescriptors(u32 width, u32 wave = 32u) {
const bool scalar = width == 1u;
const bool xyz = width == 3u;
using O = ShaderOpcode;
constexpr uint64_t GuestBase = 0x0000000110000000ull;
struct DescriptorCase {
u32 stride, records, mode, soffset;
bool swizzle, bound;
std::array<u32, 6> expected;
};
const DescriptorCase cases[] = {
{24, 2, 0, 0, false, true, {7, 8, 9, 10, 11, 12}},
// PPSA01417 selects XYZ vertices from a table of stride-16 descriptors.
{16, 2, 0, 0, false, true, {5, 6, 7, 8, 0, 0}},
{12, 2, 0, 0, false, true, {4, 5, 6, 0, 0, 0}},
{12, 2, 1, 0, false, true, {4, 5, 6, 7, 8, 9}},
{12, 1, 2, 0, false, true, {4, 5, 6, 7, 8, 9}},
{12, 0, 2, 0, false, true, {}},
// OOB3 checks offset + soffset, independently of index * stride.
{12, 20, 3, 4, false, true, {5, 6, 7, 8, 0, 0}},
{12, 2, 0, 0, false, false, {}},
// RDNA2 swizzling uses four-byte elements; soffset is added afterward.
{12, 2, 0, 4, true, true, {3, 4, 5, 0, 0, 0}},
{12, 8, 3, 4, true, true, {3, 4, 5, 0, 0, 0}},
{12, 4, 3, 4, true, true, {}},
};
// OFFEN byte offset 5 aligns to DWORD; swizzled offsets use eight-index groups.
constexpr u32 scalar_expected[] = {2, 2, 2, 2, 2, 0, 3, 0, 10, 10, 0};
TestCase test;
test.name = scalar ? "BufferLoadDwordGpuSelectedDescriptors"
: xyz ? "BufferLoadDwordx3GpuSelectedDescriptors"
: "BufferLoadsGpuSelectedDescriptors";
test.initial.resize(2048);
test.compute_info.threads_num[0] = wave;
test.compute_info.threads_num[1] = test.compute_info.threads_num[2] = 1u;
test.compute_info.thread_ids_num = 1;
test.compute_info.wave_size = wave;
test.has_compute_info = true;
test.code.push_back(EncodeVop1(0x01, 25, Vgpr(0)));
AppendSMovLiteral(&test.code, 126, 1u);
AppendSMovLiteral(&test.code, 127, 0u);
for (u32 i = 0; i < std::size(cases); ++i) {
const auto &input = cases[i];
const u32 data_offset = 4096 + i * 128;
// 120-byte table entries with a descriptor at byte 8, as in PPSA04677.
const std::array<u32, 4> descriptor{
static_cast<u32>(GuestBase + data_offset),
(input.stride << 16u) | (input.swizzle ? 1u << 31u : 0u) | 1u,
input.records, (input.bound ? 0x5204u : 0x204u) | (input.mode << 28u)};
std::copy(descriptor.begin(), descriptor.end(), test.initial.begin() + 130 + i * 30);
for (u32 word = 0; word < 32; ++word) {
test.initial[data_offset / 4 + word] = i * 100 + word + 1;
}
// Reverse the table order through a GPU load and readfirstlane, not host constants.
const u32 selected = static_cast<u32>(std::size(cases)) - 1 - i;
test.initial[64 + i] = selected;
AppendVMovU32(&test.code, 30, (64 + i) * 4);
AppendBufferLoadDword(&test.code, 0, 30);
test.code.push_back(EncodeVop1(0x02, 20, Vgpr(0)));
test.code.push_back(EncodeSop2(0x26, 20, 20, 255)); // s_mul_i32 s20, s20, 120
test.code.push_back(120);
test.code.push_back(EncodeSmem0(0x0a, 8, 0));
test.code.push_back(EncodeSmem1(520, 20));
AppendSMovLiteral(&test.code, 22, cases[selected].soffset);
AppendVMovU32(&test.code, 21, scalar ? 5u : 1u);
test.code.push_back(EncodeMubuf0(scalar ? 0x0c : xyz ? 0x0f : 0x0e, 0, !scalar, scalar));
test.code.push_back(EncodeMubuf1(0, 2, 21, 22));
if (!scalar) {
test.code.push_back(EncodeMubuf0(xyz ? 0x0f : 0x0d, xyz ? 12 : 16, true, false));
test.code.push_back(EncodeMubuf1(xyz ? 3 : 4, 2, 21, 22));
}
const u32 outputs = scalar ? 1u : 6u;
for (u32 component = 0; component < outputs; ++component) {
AppendStoreVgpr(&test.code, component, i * outputs + component);
const u32 expected = scalar ? scalar_expected[selected] : cases[selected].expected[component];
test.expected.push_back(expected == 0 ? 0 : selected * 100 + expected);
}
}
constexpr u32 payload = 6144u, output = 8192u, canary = 0xa5a5a5a5u;
const u32 row = std::size(cases);
const std::array<u32, 4> descriptor{u32(GuestBase + payload), (32u << 16u) | 1u,
wave, 0x5204u | (1u << 23u)};
std::copy(descriptor.begin(), descriptor.end(), test.initial.begin() + 130u + row * 30u);
test.initial[64u + row] = row;
for (u32 lane = 0; lane < wave; ++lane)
for (u32 word = 0; word < 8u; ++word)
test.initial[payload / 4u + lane * 8u + word] = 0x90000000u | (lane << 8u) | word;
test.initial.resize((output + wave * 32u) / 4u, canary);
const auto matrix_results = std::move(test.expected);
test.expected = test.initial;
std::copy(matrix_results.begin(), matrix_results.end(), test.expected.begin());
AppendVMovU32(&test.code, 30, (64u + row) * 4u);
AppendBufferLoadDword(&test.code, 0, 30);
test.code.push_back(EncodeVop1(0x02, 20, Vgpr(0)));
test.code.insert(test.code.end(), {EncodeSop2(0x26, 20, 20, 255), 120u,
EncodeSmem0(0x0a, 8, 0), EncodeSmem1(520, 20)});
AppendSMovLiteral(&test.code, 126, UINT32_MAX);
AppendSMovLiteral(&test.code, 127, wave == 64u ? UINT32_MAX : 0u);
test.code.insert(test.code.end(), {EncodeVop2(0x1a, 24, InlineU32(5), 25),
EncodeVop2(0x25, 24, 255, 24), output});
const u32 opcode = scalar ? 0x0cu : xyz ? 0x0fu : 0x0eu;
for (u32 component = width; component < 4u; ++component)
AppendVMovLiteral(&test.code, 4u + component, canary);
AppendVMovU32(&test.code, 21, 0u);
AppendVMovLiteral(&test.code, 22, UINT32_MAX - 2u);
// U32 byte offset + immediate wraps to four before ADD_TID addressing and bounds.
test.code.insert(test.code.end(), {EncodeMubuf0(opcode, 7u, true, true), EncodeMubuf1(4, 2, 21),
EncodeMubuf0(0x1e, 0u, false, true), EncodeMubuf1(4, 12, 24)});
for (u32 component = 0; component < 4u; ++component)
AppendVMovLiteral(&test.code, 4u + component, canary);
AppendVMovLiteral(&test.code, 21, UINT32_MAX);
AppendVMovU32(&test.code, 22, 0u);
AppendSMovLiteral(&test.code, 126, 0x55555555u);
AppendSMovLiteral(&test.code, 127, wave == 64u ? 0xaaaaaaaau : 0u);
test.code.insert(test.code.end(), {EncodeMubuf0(opcode, 4u, true, true, true), EncodeMubuf1(4, 2, 21)});
AppendSMovLiteral(&test.code, 126, 0u);
AppendSMovLiteral(&test.code, 127, 0u);
test.code.insert(test.code.end(), {EncodeMubuf0(opcode, 0u, true, true, true), EncodeMubuf1(4, 2, 21)});
AppendSMovLiteral(&test.code, 126, UINT32_MAX);
AppendSMovLiteral(&test.code, 127, wave == 64u ? UINT32_MAX : 0u);
test.code.insert(test.code.end(), {EncodeMubuf0(0x1e, 16u, false, true), EncodeMubuf1(4, 12, 24)});
for (u32 lane = 0; lane < wave; ++lane) {
const bool active = (lane & 1u) == (lane < 32u ? 0u : 1u);
for (u32 component = 0; component < width; ++component) {
test.expected[output / 4u + lane * 8u + component] =
0x90000000u | (lane << 8u) | (component + 1u);
test.expected[output / 4u + lane * 8u + 4u + component] = !active ? canary :
lane == 0u ? 0u : 0x90000000u | ((lane - 1u) << 8u) | (component + 1u);
}
}
AppendEnd(&test.code);
test.bda_mappings = {{GuestBase, 0}};
test.opcodes = {O::V_MOV_B32, O::S_MOV_B32, O::BUFFER_LOAD_DWORD,
O::V_READFIRSTLANE_B32, O::S_MUL_I32, O::S_BUFFER_LOAD_DWORDX4,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
if (scalar) {
test.decoded_counts = {{"BUFFER_LOAD_DWORD ", 2u * std::size(cases) + 4u}};
} else if (xyz) {
test.opcodes.push_back(O::BUFFER_LOAD_DWORDX3);
} else {
test.opcodes.insert(test.opcodes.end(), {O::BUFFER_LOAD_DWORDX4, O::BUFFER_LOAD_DWORDX2});
}
test.opcodes.insert(test.opcodes.end(), {O::V_LSHLREV_B32, O::V_ADD_NC_U32, O::BUFFER_STORE_DWORDX4});
test.required_spirv = {"OpConvertUToPtr", "PhysicalStorageBuffer", "Volatile"};
return test;
}
TestCase BufferLoadsGpuSelectedDescriptors() {
return BufferLoadsGpuSelectedDescriptors(4u);
}
TestCase BufferLoadDwordx3GpuSelectedDescriptors() {
return BufferLoadsGpuSelectedDescriptors(3u);
}
TestCase BufferLoadDwordGpuSelectedDescriptors() {
return BufferLoadsGpuSelectedDescriptors(1u);
}
TestCase BufferLoadGpuSelectedDescriptors(u32 wave, ShaderOpcode load) {
using O = ShaderOpcode;
const bool formatted = load == O::BUFFER_LOAD_FORMAT_X;
const bool byte = load == O::BUFFER_LOAD_UBYTE;
using F = Prospero::BufferFormat;
constexpr uint64_t GuestBase = 0x0000000110000000ull;
struct DescriptorCase {
F format;
u32 selector, stride, records, mode, index, offset, soffset;
bool swizzle;
u32 expected;
bool zero_payload = false;
bool unmapped = false;
};
std::vector<DescriptorCase> cases;
if (formatted) {
cases = {
{F::k8UInt, 4, 1, 4, 3, 1, 0, 0, false, 0x7f},
{F::k8UInt, 4, 0, 4, 3, 0, 3, 0, false, 0x80},
{F::k16UInt, 4, 2, 4, 0, 1, 0, 0, false, 0x80ff},
{F::k32UInt, 4, 4, 2, 0, 1, 0, 0, false, 0x8001ff00},
{F::k32_32_32_32Float, 7, 16, 1, 0, 0, 0, 0, false, 0x3fc00000},
{F::k8SInt, 4, 0, 16, 3, 0, 3, 0, false, 0xffffff80},
{F::k16SInt, 4, 0, 16, 3, 0, 2, 0, false, 0xffff80ff},
{F::k8UNorm, 4, 0, 16, 3, 0, 2, 0, false, 0x3f800000},
{F::k8SNorm, 4, 0, 16, 3, 0, 3, 0, false, 0xbf800000},
{F::k8UScaled, 4, 0, 16, 3, 0, 1, 0, false, 0x42fe0000},
{F::k8SScaled, 4, 0, 16, 3, 0, 3, 0, false, 0xc3000000},
{F::k16Float, 4, 0, 16, 3, 0, 8, 0, false, 0x3f800000},
{F::k11_11_10UInt, 5, 0, 16, 3, 0, 0, 0, false, 0x7ef},
{F::k10_10_10_2UInt, 7, 0, 16, 3, 0, 0, 0, false, 2},
{F::k8_8_8_8UInt, 6, 0, 16, 3, 0, 0, 0, false, 0xff},
// Align the whole format first, then advance components from one swizzled base.
{F::k16UInt, 4, 0, 16, 3, 0, 1, 0, false, 0x7f01},
{F::k16UInt, 4, 0, 4, 3, 0, 3, 0, false, 0},
{F::k32_32Float, 5, 8, 2, 0, 1, 0, 4, true, 0x3fc00000},
// FORMAT_X still transfers a complete RGBA32 record for range checking.
{F::k32_32_32_32Float, 4, 0, 16, 3, 0, 4, 0, false, 0},
{F::k32_32_32_32Float, 4, 8, 1, 0, 0, 0, 0, false, 0},
{F::k32_32_32_32Float, 1, 0, 16, 3, 0, 4, 0, false, 0x3f800000},
{F::k32UInt, 1, 4, 0, 0, 0, 0, 0, false, 1},
{F::k32UInt, 0, 4, 1, 0, 0, 0, 0, false, 0},
{F::kInvalid, 1, 4, 1, 0, 0, 0, 0, false, 0},
{F::k32UInt, 4, 4, 0, 2, 0, 0, 0, false, 0},
{F::k32UInt, 4, 0, 4, 3, 0, 0, 8, false, 0},
{F::k8_8UInt, 5, 0, 16, 3, 0, 2, 0, false, 0x80},
{F::k16_16UInt, 5, 0, 16, 3, 0, 1, 0, false, 0x80ff},
{F::k16_16_16_16UInt, 7, 0, 16, 3, 0, 0, 0, false, 0x8001},
{F::k16_16_16_16Float, 6, 0, 16, 3, 0, 4, 0, false, 0x3f800000},
{F::k32_32_32UInt, 7, 0, 16, 3, 0, 0, 0, false, 0x80ff7f01},
{F::k32_32_32UInt, 6, 0, 16, 3, 0, 0, 0, false, 0x40003c00},
{F::k11_11_10Float, 6, 0, 16, 3, 0, 0, 0, false, 0x400c0000},
{F::k10_11_11Float, 4, 0, 16, 3, 0, 0, 0, false, 0x44040000},
// Whole-record OOB and constant/reserved selectors must not touch this absent BDA page.
{F::k32_32_32_32UInt, 4, 4, 1, 0, 0, 0, 0, false, 0, false, true},
{F::k32UInt, 0, 4, 1, 0, 0, 0, 0, false, 0, false, true},
{F::k32UInt, 1, 4, 1, 0, 0, 0, 0, false, 1, false, true},
{F::k32Float, 1, 4, 1, 0, 0, 0, 0, false, 0x3f800000, false, true},
{F::k32UInt, 2, 4, 1, 0, 0, 0, 0, false, 0, false, true},
{F::k32UInt, 3, 4, 1, 0, 0, 0, 0, false, 0, false, true},
{static_cast<F>(127), 4, 4, 1, 0, 0, 0, 0, false, 0, false, true},
};
} else if (!byte) {
cases = {
{F::k32Float, 1, 0, 16, 3, 0, 0, 0, false, 0x7f01},
{F::k8UInt, 7, 0, 16, 3, 0, 1, 0, false, 0x7f01},
{F::k8UInt, 4, 0, 16, 3, 0, 2, 0, false, 0x80ff},
{F::k8UInt, 4, 0, 16, 3, 0, 3, 0, false, 0x80ff},
{F::k8UInt, 4, 0, 4, 3, 0, 2, 0, false, 0x80ff},
{F::k8UInt, 4, 0, 4, 3, 0, 3, 0, false, 0},
{F::k8UInt, 4, 4, 2, 0, 1, 2, 0, false, 0x8001},
{F::k8UInt, 4, 4, 2, 0, 1, 3, 0, false, 0x8001},
{F::k8UInt, 4, 4, 1, 1, 1, 0, 0, false, 0},
{F::k8UInt, 4, 4, 1, 1, 0, 8, 0, false, 0x3c00},
{F::k8UInt, 4, 4, 1, 2, 1, 2, 0, false, 0x8001},
{F::k8UInt, 4, 4, 0, 2, 0, 0, 0, false, 0, false, true},
{F::k8UInt, 4, 0, 16, 3, 0, 0, 2, false, 0x80ff},
{F::k8UInt, 4, 0, 4, 3, 0, 0, 8, false, 0, false, true},
{F::kInvalid, 4, 4, 1, 0, 0, 0, 0, false, 0, false, true},
{F::k8UInt, 4, 0, 16, 3, 0, 15, 0, false, 0, false, true},
{F::k8UInt, 4, 8, 2, 0, 1, 2, 4, true, 0x4000},
};
} else {
cases = {
{F::k32Float, 1, 0, 16, 3, 0, 0, 0, false, 0x01},
{F::k8UInt, 7, 0, 16, 3, 0, 1, 0, false, 0x7f},
{F::k8UInt, 4, 0, 16, 3, 0, 2, 0, false, 0xff},
{F::k8UInt, 4, 0, 4, 3, 0, 3, 0, false, 0x80},
{F::k8UInt, 4, 0, 4, 3, 0, 4, 0, false, 0, false, true},
{F::k8UInt, 4, 4, 2, 0, 1, 3, 0, false, 0x80},
{F::k8UInt, 4, 0, 16, 3, 0, 0, 2, false, 0xff},
{F::k8UInt, 4, 8, 2, 0, 1, 3, 4, true, 0x40},
};
}
// Visit every geometry entry and every selector. Zero payloads have exact zero
// results in all numeric classes; selector ONE depends only on integer vs float.
for (u32 format = 0; formatted && format <= static_cast<u32>(F::k32_32_32_32Float); ++format) {
const auto info = ShaderRecompiler::Format::GetFormatInfo(static_cast<F>(format));
using T = ShaderRecompiler::Format::ComponentType;
for (u32 selector = 0; selector < 8u; ++selector) {
const u32 one = info.type == T::Uint || info.type == T::Sint ? 1u : 0x3f800000u;
cases.push_back({static_cast<F>(format), selector, 16, 1, 0, 0, 0, 0, false,
selector == 1u && info.type != T::Unknown ? one : 0u, true});
}
}
TestCase test;
test.name = formatted ? (wave == 32 ? "BufferLoadFormatXGpuSelectedDescriptors32"
: "BufferLoadFormatXGpuSelectedDescriptors64")
: byte ? (wave == 32 ? "BufferLoadUbyteGpuSelectedDescriptors32"
: "BufferLoadUbyteGpuSelectedDescriptors64")
: (wave == 32 ? "BufferLoadUshortGpuSelectedDescriptors32"
: "BufferLoadUshortGpuSelectedDescriptors64");
const auto align256 = [](u32 bytes) { return (bytes + 255u) & ~255u; };
const auto selection_base = align256(static_cast<u32>(cases.size()) * 4u);
const auto descriptor_base = align256(selection_base + static_cast<u32>(cases.size()) * 4u);
const auto payload_base = align256(descriptor_base + static_cast<u32>(cases.size()) * 32u);
test.initial.resize((payload_base + cases.size() * 64u) / 4u);
constexpr std::array payload{0x80ff7f01u, 0x8001ff00u, 0x40003c00u, 0x3fc00000u};
for (u32 i = 0; i < std::size(cases); ++i) {
const auto &input = cases[i];
const u32 data_offset = payload_base + i * 64;
const uint64_t data_address = GuestBase + (input.unmapped ? 0x100000u : data_offset);
const std::array<u32, 8> row{
static_cast<u32>(data_address),
1u | (input.stride << 16u) | (input.swizzle ? 1u << 31u : 0u),
input.records, input.selector | (static_cast<u32>(input.format) << 12u) |
(input.mode << 28u),
input.index, input.offset, input.soffset, 0};
std::copy(row.begin(), row.end(), test.initial.begin() + descriptor_base / 4u + i * 8);
if (!input.zero_payload)
std::copy(payload.begin(), payload.end(), test.initial.begin() + data_offset / 4);
// Read the selection on the GPU, as GTA's descriptor waterfall does.
test.initial[selection_base / 4u + i] = std::size(cases) - 1 - i;
test.expected.push_back(cases[std::size(cases) - 1 - i].expected);
}
auto &code = test.code;
AppendSMovLiteral(&code, 28, 0);
AppendSMovLiteral(&code, 29, static_cast<u32>(cases.size()));
const auto loop = static_cast<u32>(code.size());
code.push_back(EncodeVop1(0x01, 30, 28));
code.push_back(EncodeVop2(0x1a, 30, InlineU32(2), 30));
code.push_back(EncodeMubuf0(0x0c, selection_base));
code.push_back(EncodeMubuf1(2, 0, 30));
code.push_back(EncodeVop1(0x02, 20, Vgpr(2)));
code.push_back(EncodeSop2(0x1e, 20, 20, InlineU32(5)));
code.push_back(EncodeSmem0(0x0b, 8, 0));
code.push_back(EncodeSmem1(descriptor_base, 20));
code.push_back(EncodeVop1(0x01, 21, 12));
code.push_back(EncodeVop1(0x01, 22, 13));
code.push_back(EncodeMubuf0(formatted ? 0x00 : byte ? 0x08 : 0x0a, 0, true, true, !formatted));
code.push_back(EncodeMubuf1(4, 2, 21, 14));
AppendBufferStoreDword(&code, 4, 30);
code.push_back(EncodeSop2(0x00, 28, 28, InlineU32(1)));
code.push_back(EncodeSopc(0x0a, 28, 29));
code.push_back(EncodeSopp(0x05, loop - static_cast<u32>(code.size()) - 1u));
AppendEnd(&code);
test.bda_mappings = {{GuestBase, 0}};
test.fault_free_address = GuestBase + 0x100000u;
test.compute_info.threads_num[0] = 1;
test.compute_info.threads_num[1] = test.compute_info.threads_num[2] = 1;
test.compute_info.wave_size = wave;
test.has_compute_info = true;
test.required_spirv = {"PhysicalStorageBuffer", "OpLoopMerge", formatted ? "OpSwitch" : "Volatile"};
if (!formatted) test.forbidden_spirv = {"convert_indirect_buffer_format",
"indirect_buffer_format_geometry"};
test.opcodes = {O::S_MOV_B32, O::V_MOV_B32, O::V_LSHLREV_B32,
O::BUFFER_LOAD_DWORD, O::V_READFIRSTLANE_B32, O::S_LSHL_B32,
O::S_BUFFER_LOAD_DWORDX8, load,
O::BUFFER_STORE_DWORD, O::S_ADD_U32, O::S_CMP_LT_U32,
O::S_CBRANCH_SCC1, O::S_ENDPGM};
return test;
}
TestCase BufferLoadFormatXGpuSelectedDescriptors() {
return BufferLoadGpuSelectedDescriptors(32u, ShaderOpcode::BUFFER_LOAD_FORMAT_X);
}
TestCase BufferLoadFormatXGpuSelectedDescriptors64() {
return BufferLoadGpuSelectedDescriptors(64u, ShaderOpcode::BUFFER_LOAD_FORMAT_X);
}
TestCase BufferLoadUshortGpuSelectedDescriptors() {
return BufferLoadGpuSelectedDescriptors(32u, ShaderOpcode::BUFFER_LOAD_USHORT);
}
TestCase BufferLoadUshortGpuSelectedDescriptors64() {
return BufferLoadGpuSelectedDescriptors(64u, ShaderOpcode::BUFFER_LOAD_USHORT);
}
TestCase BufferLoadUbyteGpuSelectedDescriptors() {
return BufferLoadGpuSelectedDescriptors(32u, ShaderOpcode::BUFFER_LOAD_UBYTE);
}
TestCase BufferLoadUbyteGpuSelectedDescriptors64() {
return BufferLoadGpuSelectedDescriptors(64u, ShaderOpcode::BUFFER_LOAD_UBYTE);
}
void CheckSharedIndirectBufferLoads(VulkanHarness &vulkan) {
using namespace ShaderRecompiler::IR;
for (const u32 wave : {32u, 64u}) {
for (const auto &test : {BufferLoadsGpuSelectedDescriptors(1u, wave),
BufferLoadsGpuSelectedDescriptors(3u, wave),
BufferLoadsGpuSelectedDescriptors(4u, wave),
BufferLoadGpuSelectedDescriptors(wave, ShaderOpcode::BUFFER_LOAD_UBYTE),
BufferLoadGpuSelectedDescriptors(wave, ShaderOpcode::BUFFER_LOAD_USHORT)}) {
auto compiled = CompileCase(test, vulkan.SubgroupSize());
struct Profile { u32 components = 0, uses = 0, definitions = 0, calls = 0, reads = 0; };
std::map<std::string, Profile> profiles;
for (const auto *block : compiled.program.blocks)
for (const auto &inst : *block) {
const auto components = BufferComponentCount(inst.GetOpcode());
if (components == 0u || inst.GetOpcode() == ValueOpcode::ReadConstBuffer) continue;
const auto &memory = compiled.program.memory_info[inst.Flags<MemoryFlags>().index];
if (memory.kind != ResourceKind::IndirectBuffer || memory.formatted) continue;
const auto name = "load_indirect_buffer_" + std::to_string(memory.data_bits) + "x" +
std::to_string(components) + (memory.coherent ? "_coherent" : "");
auto &profile = profiles[name];
profile.components = components;
++profile.uses;
}
std::map<u32, std::string> functions;
u32 function = 0, bda = 0, inline_reads = 0;
for (size_t offset = 5; offset < compiled.spirv.size();) {
const auto words = std::span(compiled.spirv).subspan(offset, compiled.spirv[offset] >> 16u);
const auto opcode = static_cast<spv::Op>(words[0] & 0xffffu);
if (opcode == spv::OpName) {
const std::string_view name(reinterpret_cast<const char *>(words.data() + 2));
if (name.starts_with("load_indirect_buffer_")) functions.emplace(words[1], name);
if (name == "get_bda_pointer") bda = words[1];
}
if (opcode == spv::OpFunction) {
function = words[2];
if (functions.contains(function)) ++profiles[functions.at(function)].definitions;
}
if (opcode == spv::OpFunctionCall) {
if (functions.contains(words[3])) ++profiles[functions.at(words[3])].calls;
if (words[3] == bda) {
if (functions.contains(function)) ++profiles[functions.at(function)].reads;
else ++inline_reads;
}
}
offset += words.size();
}
Require(test.name, "raw load function scope", functions.size() == profiles.size() && inline_reads == 0u,
"a raw load profile was unused, missing or still emitted inline");
for (const auto &[name, profile] : profiles)
Require(test.name, name.c_str(), profile.uses != 0u && profile.definitions == 1u &&
profile.calls == profile.uses * (wave == 64u && vulkan.SubgroupSize() == 32u ? 2u : 1u) &&
profile.reads == profile.components,
"raw load helpers duplicated definitions, calls or component reads");
auto buffer = vulkan.CreateStorageBuffer(test.name, test.initial, test.initial.size(), true);
vulkan.Dispatch(test, compiled, buffer);
CompareWords(test, "shared raw loads, lane addressing, EXEC and bounds", test.expected,
vulkan.ReadBuffer(test.name, buffer, test.expected.size()));
vulkan.DestroyBuffer(&buffer);
}
}
std::printf("[compute] %-32s ok\n", "SharedIndirectBufferLoads");
}
TestCase BufferStoreDwordx4DropsOnlyOutOfBoundsTail() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovLiteral(&code, 0, 0xaaaaaaaau);
AppendVMovLiteral(&code, 1, 0xbbbbbbbbu);
AppendVMovLiteral(&code, 2, 0xccccccccu);
AppendVMovLiteral(&code, 3, 0xddddddddu);
AppendVMovU32(&code, 20, 8);
code.push_back(EncodeMubuf0(0x1eu));
code.push_back(EncodeMubuf1(0, 0, 20));
AppendEnd(&code);
TestCase test;
test.name = "BufferStoreDwordx4DropsOnlyOutOfBoundsTail";
test.code = std::move(code);
test.initial = {0x11111111u, 0x22222222u, 0x33333333u, 0x44444444u};
test.expected = {0x11111111u, 0x22222222u, 0xaaaaaaaau, 0xbbbbbbbbu};
test.storage_buffer_range_bytes = 16;
test.opcodes = {O::V_MOV_B32, O::BUFFER_STORE_DWORDX4, O::S_ENDPGM};
return test;
}
TestCase BufferLoadFormatXyzwRejectsPartialRecord() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 20, 8);
AppendBufferLoadOpcode(&code, 0x03, 0, 20);
for (u32 i = 0; i < 4; i++) {
AppendStoreVgpr(&code, i, i);
}
AppendEnd(&code);
TestCase test;
test.name = "BufferLoadFormatXyzwRejectsPartialRecord";
test.code = std::move(code);
test.initial = {0x3f800000u, 0x40000000u, 0x40400000u, 0x40800000u};
test.expected = {0u, 0u, 0u, 0u};
test.storage_buffer_range_bytes = 16;
test.user_data = MakeStructuredStorageBufferData(
0, 4, false, BufferFormat(Prospero::BufferFormat::k32_32_32_32Float));
test.has_user_data = true;
test.opcodes = {O::V_MOV_B32, O::BUFFER_LOAD_FORMAT_XYZW,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
return test;
}
TestCase BufferStoreFormatXyzwDropsPartialRecord() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovLiteral(&code, 0, 0xaaaaaaaau);
AppendVMovLiteral(&code, 1, 0xbbbbbbbbu);
AppendVMovLiteral(&code, 2, 0xccccccccu);
AppendVMovLiteral(&code, 3, 0xddddddddu);
AppendVMovU32(&code, 20, 8);
code.push_back(EncodeMubuf0(0x07u));
code.push_back(EncodeMubuf1(0, 0, 20));
AppendEnd(&code);
TestCase test;
test.name = "BufferStoreFormatXyzwDropsPartialRecord";
test.code = std::move(code);
test.initial = {0x11111111u, 0x22222222u, 0x33333333u, 0x44444444u};
test.expected = test.initial;
test.storage_buffer_range_bytes = 16;
test.user_data = MakeStructuredStorageBufferData(
0, 4, false, BufferFormat(Prospero::BufferFormat::k32_32_32_32Float));
test.has_user_data = true;
test.opcodes = {O::V_MOV_B32, O::BUFFER_STORE_FORMAT_XYZW, O::S_ENDPGM};
return test;
}
TestCase BufferLoadFormatXRejectsPartialRecord() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 20, 12);
AppendBufferLoadOpcode(&code, 0x00, 0, 20);
// Bounds precede alignment: rounding offset 1 down must not make its full format fit.
AppendVMovU32(&code, 20, 1);
AppendBufferLoadOpcode(&code, 0x00, 1, 20);
AppendStoreVgpr(&code, 0, 0);
AppendStoreVgpr(&code, 1, 1);
AppendEnd(&code);
TestCase test;
test.name = "BufferLoadFormatXRejectsPartialRecord";
test.code = std::move(code);
test.initial = {0x11111111u, 0x22222222u, 0x33333333u, 0x44444444u};
test.expected = {0u, 0u, 0x33333333u, 0x44444444u};
test.storage_buffer_range_bytes = 16;
test.user_data = MakeStructuredStorageBufferData(
0, 16, false, BufferFormat(Prospero::BufferFormat::k32_32_32_32Float));
test.has_user_data = true;
test.opcodes = {O::V_MOV_B32, O::BUFFER_LOAD_FORMAT_X, O::BUFFER_STORE_DWORD,
O::S_ENDPGM};
return test;
}
TestCase BufferStoreFormatXRejectsPartialRecord() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovLiteral(&code, 0, 0xaaaaaaaau);
AppendVMovU32(&code, 20, 12);
code.push_back(EncodeMubuf0(0x04u));
code.push_back(EncodeMubuf1(0, 0, 20));
AppendEnd(&code);
TestCase test;
test.name = "BufferStoreFormatXRejectsPartialRecord";
test.code = std::move(code);
test.initial = {0x11111111u, 0x22222222u, 0x33333333u, 0x44444444u};
test.expected = test.initial;
test.storage_buffer_range_bytes = 16;
test.user_data = MakeStructuredStorageBufferData(
0, 4, false, BufferFormat(Prospero::BufferFormat::k32_32_32_32Float));
test.has_user_data = true;
test.opcodes = {O::V_MOV_B32, O::BUFFER_STORE_FORMAT_X, O::S_ENDPGM};
return test;
}
TestCase BufferLoadFormatXyRejectsPartialRecord() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 20, 8);
AppendBufferLoadOpcode(&code, 0x01, 0, 20);
AppendStoreVgpr(&code, 0, 0);
AppendStoreVgpr(&code, 1, 1);
AppendEnd(&code);
TestCase test;
test.name = "BufferLoadFormatXyRejectsPartialRecord";
test.code = std::move(code);
test.initial = {0x11111111u, 0x22222222u, 0x33333333u, 0x44444444u};
test.expected = {0u, 0u, 0x33333333u, 0x44444444u};
test.storage_buffer_range_bytes = 16;
test.user_data = MakeStructuredStorageBufferData(
0, 16, false, BufferFormat(Prospero::BufferFormat::k32_32_32_32Float));
test.has_user_data = true;
test.opcodes = {O::V_MOV_B32, O::BUFFER_LOAD_FORMAT_XY, O::BUFFER_STORE_DWORD,
O::S_ENDPGM};
return test;
}
TestCase BufferStoreFormatXyRejectsPartialRecord() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovLiteral(&code, 0, 0xaaaaaaaau);
AppendVMovLiteral(&code, 1, 0xbbbbbbbbu);
AppendVMovU32(&code, 20, 8);
code.push_back(EncodeMubuf0(0x05u));
code.push_back(EncodeMubuf1(0, 0, 20));
AppendEnd(&code);
TestCase test;
test.name = "BufferStoreFormatXyRejectsPartialRecord";
test.code = std::move(code);
test.initial = {0x11111111u, 0x22222222u, 0x33333333u, 0x44444444u};
test.expected = test.initial;
test.storage_buffer_range_bytes = 16;
test.user_data = MakeStructuredStorageBufferData(
0, 4, false, BufferFormat(Prospero::BufferFormat::k32_32_32_32Float));
test.has_user_data = true;
test.opcodes = {O::V_MOV_B32, O::BUFFER_STORE_FORMAT_XY, O::S_ENDPGM};
return test;
}
TestCase BufferStoreVariants() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 20, 1);
AppendVMovU32(&code, 0, 0xaa);
AppendBufferStoreOpcode(&code, 0x18, 0, 20);
AppendVMovU32(&code, 20, 2);
AppendVMovLiteral(&code, 1, 0x0000bbccu);
AppendBufferStoreOpcode(&code, 0x1a, 1, 20);
AppendVMovU32(&code, 20, 4);
AppendVMovLiteral(&code, 2, 0x11111111u);
AppendVMovLiteral(&code, 3, 0x22222222u);
AppendBufferStoreOpcode(&code, 0x1d, 2, 20);
AppendVMovU32(&code, 20, 12);
AppendVMovLiteral(&code, 4, 0x33333333u);
AppendVMovLiteral(&code, 5, 0x44444444u);
AppendVMovLiteral(&code, 6, 0x55555555u);
AppendBufferStoreOpcode(&code, 0x1f, 4, 20);
AppendVMovU32(&code, 20, 24);
AppendVMovLiteral(&code, 7, 0x66666666u);
AppendVMovLiteral(&code, 8, 0x77777777u);
AppendVMovLiteral(&code, 9, 0x88888888u);
AppendVMovLiteral(&code, 10, 0x99999999u);
AppendBufferStoreOpcode(&code, 0x1e, 7, 20);
AppendEnd(&code);
return {"BufferStoreVariants",
code,
std::vector<u32>(10, 0),
{0xbbccaa00u, 0x11111111u, 0x22222222u, 0x33333333u, 0x44444444u,
0x55555555u, 0x66666666u, 0x77777777u, 0x88888888u, 0x99999999u},
{O::V_MOV_B32, O::BUFFER_STORE_BYTE, O::BUFFER_STORE_SHORT,
O::BUFFER_STORE_DWORDX2, O::BUFFER_STORE_DWORDX3,
O::BUFFER_STORE_DWORDX4, O::S_ENDPGM}};
}
TestCase BufferStoreD16Hi(bool halfword, bool masked, u32 wave_size) {
using O = ShaderOpcode;
TestCase test;
if (halfword) {
test.name = wave_size == 32u
? (masked ? "BufferStoreShortD16HiMasked32" : "BufferStoreShortD16HiAdjacentLanes32")
: (masked ? "BufferStoreShortD16HiMasked64" : "BufferStoreShortD16HiAdjacentLanes64");
} else {
test.name = masked ? "BufferStoreByteD16HiMasked" : "BufferStoreByteD16HiAdjacentLanes";
}
const u32 element_bytes = halfword ? 2u : 1u;
AppendVMovLiteral(&test.code, 4, 0xa5c39b72u);
test.code.push_back(EncodeVop2(0x25, 25, InlineU32(4), 0));
if (masked) {
AppendSMovLiteral(&test.code, 126, 0x55555555u);
AppendSMovLiteral(&test.code, 127, wave_size == 64 ? 0xaaaaaaaau : 0u);
}
// Captured b21e373811f28e31 pc 0x1b8c: e0646000 80020419.
// Short variant uses 3e8244db54b38b0c pc 0x211c with the same test registers.
test.code.push_back(EncodeMubuf0(halfword ? 0x1bu : 0x19u, 0, true, false, true));
test.code.push_back(EncodeMubuf1(4, 2, 25));
AppendEnd(&test.code);
test.initial = std::vector<u32>((wave_size * element_bytes + 16u) / 4u, 0x11223344u);
test.expected = test.initial;
for (u32 lane = 0; lane < wave_size - 4u; lane++) {
const u32 mask = !masked ? 0xffffffffu : lane < 32 ? 0x55555555u : 0xaaaaaaaau;
if (((mask >> (lane % 32)) & 1u) == 0) continue;
const u32 byte = (lane + 4u) * element_bytes, shift = (byte % 4u) * 8u;
const u32 bits = halfword ? 0xffffu : 0xffu, value = halfword ? 0xa5c3u : 0xc3u;
test.expected[byte / 4u] = (test.expected[byte / 4u] & ~(bits << shift)) | (value << shift);
}
const auto descriptor = MakeStructuredStorageBufferData(element_bytes, wave_size);
std::copy_n(descriptor.begin(), 4, test.user_data.begin() + 8);
test.has_user_data = true;
test.storage_buffer_range_bytes = wave_size * element_bytes;
test.compute_info.threads_num[0] = wave_size;
test.compute_info.threads_num[1] = test.compute_info.threads_num[2] = 1;
test.compute_info.thread_ids_num = 1;
test.compute_info.wave_size = static_cast<uint8_t>(wave_size);
test.has_compute_info = true;
test.opcodes = {O::V_MOV_B32, O::V_ADD_NC_U32,
halfword ? O::BUFFER_STORE_SHORT_D16_HI : O::BUFFER_STORE_BYTE_D16_HI, O::S_ENDPGM};
test.required_spirv = {"Volatile"};
test.forbidden_spirv = {"OpAtomicLoad", "OpAtomicCompareExchange"};
return test;
}
TestCase BufferFormatVariants() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 20, 0);
AppendBufferLoadOpcode(&code, 0x00, 0, 20);
AppendBufferLoadOpcode(&code, 0x01, 1, 20);
AppendBufferLoadOpcode(&code, 0x02, 3, 20);
AppendBufferLoadOpcode(&code, 0x03, 6, 20);
for (u32 i = 0; i < 10u; i++) {
AppendStoreVgpr(&code, i, i);
}
AppendEnd(&code);
TestCase load;
load.name = "BufferFormatLoadVariants";
load.code = code;
load.initial = {0x01020304u, 0x11121314u, 0x21222324u, 0x31323334u};
load.expected = {0x01020304u, 0x01020304u, 0x11121314u, 0x01020304u,
0x11121314u, 0x21222324u, 0x01020304u, 0x11121314u,
0x21222324u, 0x31323334u};
load.opcodes = {O::BUFFER_LOAD_FORMAT_X,
O::BUFFER_LOAD_FORMAT_XY,
O::BUFFER_LOAD_FORMAT_XYZ,
O::BUFFER_LOAD_FORMAT_XYZW,
O::V_MOV_B32,
O::BUFFER_STORE_DWORD,
O::S_ENDPGM};
return load;
}
TestCase BufferLoadFormatXyzwSnapshotsOverlappingAddress() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 21, 0);
AppendVMovU32(&code, 22, 0);
code.push_back(EncodeMubuf0(0x03u, 0, true, true));
code.push_back(EncodeMubuf1(21, 0, 21));
for (u32 i = 0; i < 4; i++) {
AppendStoreVgpr(&code, 21 + i, 4 + i);
}
AppendEnd(&code);
TestCase test;
test.name = "BufferLoadFormatXyzwSnapshotsOverlappingAddress";
test.code = std::move(code);
test.initial = {0x3f800000u, 0x40000000u, 0x40400000u, 0x40800000u,
0, 0, 0, 0};
test.expected = {0x3f800000u, 0x40000000u, 0x40400000u, 0x40800000u,
0x3f800000u, 0x40000000u, 0x40400000u, 0x40800000u};
test.opcodes = {O::V_MOV_B32, O::BUFFER_LOAD_FORMAT_XYZW,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.user_data = MakeStructuredStorageBufferData(
16, 2, false, BufferFormat(Prospero::BufferFormat::k32_32_32_32Float));
test.has_user_data = true;
return test;
}
TestCase BufferLoadFormatXyzwResource32_32FloatAppliesXy01() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 20, 0);
AppendBufferLoadOpcode(&code, 0x03, 0, 20);
for (u32 i = 0; i < 4; i++) {
AppendStoreVgpr(&code, i, i);
}
AppendEnd(&code);
TestCase test;
test.name = "BufferLoadFormatXyzwResource32_32FloatAppliesXy01";
test.code = std::move(code);
test.initial = {0x3f800000u, 0x40000000u, 0, 0};
test.expected = {0x3f800000u, 0x40000000u, 0, 0x3f800000u};
test.opcodes = {O::V_MOV_B32, O::BUFFER_LOAD_FORMAT_XYZW,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.user_data = MakeStructuredStorageBufferData(
16, 1, false, BufferFormat(Prospero::BufferFormat::k32_32Float));
test.user_data[3] = (test.user_data[3] & ~0xfffu) | DstSel(4, 5, 0, 1);
test.has_user_data = true;
return test;
}
TestCase BufferLoadFormatXyzwResource323232FloatRepeatsSource() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 20, 0);
AppendBufferLoadOpcode(&code, 0x03, 0, 20);
for (u32 i = 0; i < 4; ++i) AppendStoreVgpr(&code, i, i + 4);
AppendEnd(&code);
TestCase test;
test.name = "BufferLoadFormatXyzwResource323232FloatRepeatsSource";
test.code = std::move(code);
test.initial = {0x3e800000u, 0x3f000000u, 0x3f400000u, 0xdeadbeefu, 0, 0, 0, 0};
test.expected = {0x3e800000u, 0x3f000000u, 0x3f400000u, 0xdeadbeefu,
0x3e800000u, 0x3f000000u, 0x3f400000u, 0x3e800000u};
test.opcodes = {O::V_MOV_B32, O::BUFFER_LOAD_FORMAT_XYZW,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.user_data = MakeStructuredStorageBufferData(
0, 32, false, BufferFormat(Prospero::BufferFormat::k32_32_32Float));
test.has_user_data = true;
return test;
}
TestCase BufferLoadFormatXyzwResource8_8_8_8UintAppliesWzy1() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 20, 0);
AppendBufferLoadOpcode(&code, 0x03, 0, 20);
for (u32 i = 0; i < 4; i++) {
AppendStoreVgpr(&code, i, i);
}
AppendEnd(&code);
TestCase test;
test.name = "BufferLoadFormatXyzwResource8_8_8_8UintAppliesWzy1";
test.code = std::move(code);
test.initial = {0x11223344u, 0x55667788u, 0x99aabbccu, 0xddeeff00u};
test.expected = {0x11u, 0x22u, 0x33u, 1u};
test.opcodes = {O::V_MOV_B32, O::BUFFER_LOAD_FORMAT_XYZW,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.user_data = MakeStructuredStorageBufferData(
16, 1, false, BufferFormat(Prospero::BufferFormat::k8_8_8_8UInt));
test.user_data[3] = (test.user_data[3] & ~0xfffu) | DstSel(7, 6, 5, 1);
test.has_user_data = true;
return test;
}
TestCase BufferLoadFormatXyzwInactiveExecPreservesOverlappingAddress() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovLiteral(&code, 21, 0x11111111u);
AppendVMovLiteral(&code, 22, 0x22222222u);
AppendVMovLiteral(&code, 23, 0x33333333u);
AppendVMovLiteral(&code, 24, 0x44444444u);
code.push_back(EncodeSop1(0x04, 126, InlineU32(0)));
code.push_back(EncodeMubuf0(0x03u, 0, true, true));
code.push_back(EncodeMubuf1(21, 0, 21));
code.push_back(EncodeSMovB32(126, InlineU32(1)));
code.push_back(EncodeSMovB32(127, InlineU32(0)));
for (u32 i = 0; i < 4; i++) {
AppendStoreVgpr(&code, 21 + i, 4 + i);
}
AppendEnd(&code);
TestCase test;
test.name = "BufferLoadFormatXyzwInactiveExecPreservesOverlappingAddress";
test.code = std::move(code);
test.initial = {0xaaaaaaaa, 0xbbbbbbbb, 0xcccccccc, 0xdddddddd, 0, 0, 0, 0};
test.expected = {0xaaaaaaaau, 0xbbbbbbbbu, 0xccccccccu, 0xddddddddu,
0x11111111u, 0x22222222u, 0x33333333u, 0x44444444u};
test.opcodes = {
O::V_MOV_B32, O::S_MOV_B64, O::BUFFER_LOAD_FORMAT_XYZW,
O::S_MOV_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.user_data = MakeStructuredStorageBufferData(
16, 2, false, BufferFormat(Prospero::BufferFormat::k32_32_32_32Float));
test.has_user_data = true;
return test;
}
TestCase BufferFormatStoreVariants() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 20, 0);
AppendVMovLiteral(&code, 0, 0x01020304u);
AppendBufferStoreOpcode(&code, 0x04, 0, 20);
AppendVMovU32(&code, 20, 4);
AppendVMovLiteral(&code, 1, 0x11121314u);
AppendVMovLiteral(&code, 2, 0x21222324u);
AppendBufferStoreOpcode(&code, 0x05, 1, 20);
AppendVMovU32(&code, 20, 12);
AppendVMovLiteral(&code, 3, 0x31323334u);
AppendVMovLiteral(&code, 4, 0x41424344u);
AppendVMovLiteral(&code, 5, 0x51525354u);
AppendBufferStoreOpcode(&code, 0x06, 3, 20);
AppendVMovU32(&code, 20, 24);
AppendVMovLiteral(&code, 6, 0x61626364u);
AppendVMovLiteral(&code, 7, 0x71727374u);
AppendVMovLiteral(&code, 8, 0x81828384u);
AppendVMovLiteral(&code, 9, 0x91929394u);
AppendBufferStoreOpcode(&code, 0x07, 6, 20);
AppendEnd(&code);
return {"BufferFormatStoreVariants",
code,
std::vector<u32>(10, 0),
{0x01020304u, 0x11121314u, 0x21222324u, 0x31323334u, 0x41424344u,
0x51525354u, 0x61626364u, 0x71727374u, 0x81828384u, 0x91929394u},
{O::V_MOV_B32, O::BUFFER_STORE_FORMAT_X, O::BUFFER_STORE_FORMAT_XY,
O::BUFFER_STORE_FORMAT_XYZ, O::BUFFER_STORE_FORMAT_XYZW,
O::S_ENDPGM}};
}
TestCase BufferLoadFormatXResource8UintZeroExtendsByte() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 20, 0);
code.push_back(EncodeMubuf0(0x00u));
code.push_back(EncodeMubuf1(0, 0, 20));
AppendStoreVgpr(&code, 0, 1);
AppendEnd(&code);
TestCase test;
test.name = "BufferLoadFormatXResource8UintZeroExtendsByte";
test.code = code;
test.initial = {0x11223344u, 0};
test.expected = {0x11223344u, 0x00000044u};
test.opcodes = {O::V_MOV_B32, O::BUFFER_LOAD_FORMAT_X, O::BUFFER_STORE_DWORD,
O::S_ENDPGM};
test.user_data = MakeStructuredStorageBufferData(0, 8, false, 5);
test.has_user_data = true;
return test;
}
TestCase BufferStoreFormatXResource16UintWritesHalfword() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 20, 0);
AppendVMovLiteral(&code, 0, 0x0000aaaau);
code.push_back(EncodeMubuf0(0x04u));
code.push_back(EncodeMubuf1(0, 0, 20));
AppendEnd(&code);
TestCase test;
test.name = "BufferStoreFormatXResource16UintWritesHalfword";
test.code = code;
test.initial = {0x11223344u};
test.expected = {0x1122aaaau};
test.opcodes = {O::V_MOV_B32, O::BUFFER_STORE_FORMAT_X, O::S_ENDPGM};
test.user_data = MakeStructuredStorageBufferData(0, 4, false, 11);
test.has_user_data = true;
return test;
}
TestCase BufferStoreFormatXResource16UintPreservesAdjacentLanes() {
using O = ShaderOpcode;
std::vector<u32> code;
code.push_back(EncodeVop2(0x25, 1, InlineU32(1), 0));
code.push_back(EncodeMubuf0(0x04u, 0, true, false));
code.push_back(EncodeMubuf1(1, 0, 0));
AppendEnd(&code);
TestCase test;
test.name = "BufferStoreFormatXResource16UintPreservesAdjacentLanes";
test.code = code;
test.initial = std::vector<u32>(32, 0);
for (u32 index = 0; index < 32; index++) {
test.expected.push_back(((index * 2 + 2) << 16) | (index * 2 + 1));
}
test.opcodes = {O::V_ADD_NC_U32, O::BUFFER_STORE_FORMAT_X, O::S_ENDPGM};
test.compute_info.threads_num[0] = 64;
test.compute_info.threads_num[1] = 1;
test.compute_info.threads_num[2] = 1;
test.compute_info.thread_ids_num = 1;
test.compute_info.wave_size = 64;
test.has_compute_info = true;
test.user_data = MakeStructuredStorageBufferData(2, 64, false, 11);
test.has_user_data = true;
test.forbidden_spirv = {"OpAtomicLoad", "OpAtomicCompareExchange"};
return test;
}
TestCase BufferStoreFormatXyzwPackedUnormSkinningVectors() {
using O = ShaderOpcode;
// The captured skinning dispatch stores a compressed quaternion with W = 1.
constexpr std::array<float, 12> values = {
0.25f, 0.5f, 0.75f, 1.0f,
-1.0f, 0.0f, 2.0f, 1.0f,
0.5f, 0.5f, 0.5f, 0.5f};
std::vector<u32> code;
for (u32 record = 0; record < 3; record++) {
for (u32 component = 0; component < 4; component++) {
AppendVMovLiteral(&code, component,
std::bit_cast<u32>(values[record * 4 + component]));
}
AppendVMovU32(&code, 20, record);
code.push_back(EncodeMubuf0(0x07u, 0, true, false));
code.push_back(EncodeMubuf1(0, 0, 20));
}
AppendEnd(&code);
TestCase test;
test.name = "BufferStoreFormatXyzwPackedUnormSkinningVectors";
test.code = std::move(code);
test.initial = std::vector<u32>(4, 0xdeadbeefu);
test.expected = {0xeff80100u, 0xfff00000u, 0xa0080200u, 0xdeadbeefu};
test.user_data = MakeStructuredStorageBufferData(
4, 3, false, BufferFormat(Prospero::BufferFormat::k10_10_10_2UNorm));
test.has_user_data = true;
test.opcodes = {O::V_MOV_B32, O::BUFFER_STORE_FORMAT_XYZW, O::S_ENDPGM};
return test;
}
TestCase BufferStoreFormatXZeroFillsPackedRecord() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovLiteral(&code, 0, std::bit_cast<u32>(0.5f));
AppendVMovU32(&code, 20, 4);
code.push_back(EncodeMubuf0(0x04u));
code.push_back(EncodeMubuf1(0, 0, 20));
AppendEnd(&code);
TestCase test;
test.name = "BufferStoreFormatXZeroFillsPackedRecord";
test.code = std::move(code);
test.initial = {0x11111111u, 0xffffffffu, 0x33333333u};
test.expected = {0x11111111u, 0x00000200u, 0x33333333u};
test.user_data = MakeStructuredStorageBufferData(
0, 12, false, BufferFormat(Prospero::BufferFormat::k10_10_10_2UNorm));
test.has_user_data = true;
test.opcodes = {O::V_MOV_B32, O::BUFFER_STORE_FORMAT_X, O::S_ENDPGM};
return test;
}
TestCase BufferStoreFormatXyzwFloat16ConvertsComponents() {
using O = ShaderOpcode;
// This finite F32 has low bits 0x7e63, which form an F16 NaN if truncated.
constexpr std::array<float, 8> values = {
std::bit_cast<float>(0x3f807e63u), -2.0f, 0.5f, 0.25f,
-4.0f, 3.0f, 0.0f, 2.0f};
std::vector<u32> code;
for (u32 record = 0; record < 2; record++) {
for (u32 component = 0; component < 4; component++) {
AppendVMovLiteral(&code, component,
std::bit_cast<u32>(values[record * 4 + component]));
}
AppendVMovU32(&code, 20, record);
code.push_back(EncodeMubuf0(0x07u, 0, true, false));
code.push_back(EncodeMubuf1(0, 0, 20));
}
AppendEnd(&code);
TestCase test;
test.name = "BufferStoreFormatXyzwFloat16ConvertsComponents";
test.code = std::move(code);
test.initial = std::vector<u32>(4, 0xdeadbeefu);
test.expected = {0xc0003c04u, 0x34003800u, 0x4200c400u, 0x40000000u};
test.user_data = MakeStructuredStorageBufferData(
8, 2, false, BufferFormat(Prospero::BufferFormat::k16_16_16_16Float));
test.has_user_data = true;
test.opcodes = {O::V_MOV_B32, O::BUFFER_STORE_FORMAT_XYZW, O::S_ENDPGM};
test.required_spirv = {"PackHalf2x16"};
return test;
}
TestCase BufferStoreFormatXyzwSnorm16CapturedSkinningVectors() {
using O = ShaderOpcode;
// Normal and tangent from the character skinning dispatch (eadfd178c07feebd).
constexpr std::array<u32, 8> values = {
0x3d98f7c5u, 0xbedae712u, 0xbf66a19eu, 0x3f800000u,
0x3f7e6baeu, 0xbd339a7du, 0x3dd0a2e2u, 0xbf800000u};
std::vector<u32> code;
for (u32 record = 0; record < 2; record++) {
for (u32 component = 0; component < 4; component++) {
AppendVMovLiteral(&code, component, values[record * 4 + component]);
}
AppendVMovU32(&code, 20, record);
code.push_back(EncodeMubuf0(0x07u, 0, true, false));
code.push_back(EncodeMubuf1(0, 0, 20));
}
AppendEnd(&code);
TestCase test;
test.name = "BufferStoreFormatXyzwSnorm16CapturedSkinningVectors";
test.code = std::move(code);
test.initial = std::vector<u32>(4, 0xdeadbeefu);
test.expected = {0xc947098fu, 0x7fff8cb0u, 0xfa637f35u, 0x80010d0au};
test.user_data = MakeStructuredStorageBufferData(
8, 2, false, BufferFormat(Prospero::BufferFormat::k16_16_16_16SNorm));
test.has_user_data = true;
test.opcodes = {O::V_MOV_B32, O::BUFFER_STORE_FORMAT_XYZW, O::S_ENDPGM};
return test;
}
TestCase BufferStoreFormatXSnorm16ClampsRoundsAndPreservesHalfwords() {
using O = ShaderOpcode;
constexpr std::array<float, 9> values = {
-2.0f, -1.0f, -0.75f, -0.25f, 0.0f, 0.25f, 0.75f, 1.0f, 2.0f};
std::vector<u32> code;
for (u32 i = 0; i < values.size(); i++) {
AppendVMovLiteral(&code, 0, std::bit_cast<u32>(values[i]));
AppendVMovU32(&code, 20, i * 2 + 2);
code.push_back(EncodeMubuf0(0x04u));
code.push_back(EncodeMubuf1(0, 0, 20));
}
AppendEnd(&code);
TestCase test;
test.name = "BufferStoreFormatXSnorm16ClampsRoundsAndPreservesHalfwords";
test.code = std::move(code);
test.initial = std::vector<u32>(6, 0xdeadbeefu);
test.expected = {0x8001beefu, 0xa0018001u, 0x0000e000u,
0x5fff2000u, 0x7fff7fffu, 0xdeadbeefu};
test.user_data = MakeStructuredStorageBufferData(
0, 24, false, BufferFormat(Prospero::BufferFormat::k16SNorm));
test.has_user_data = true;
test.opcodes = {O::V_MOV_B32, O::BUFFER_STORE_FORMAT_X, O::S_ENDPGM};
return test;
}
std::vector<TestCase> BufferLoadFormatD16XCases() {
using F = Prospero::BufferFormat;
using O = ShaderOpcode;
struct LoadCase {
const char *name;
F format;
u32 input;
u32 expected;
u32 selector = 4;
u32 index = 0;
bool active = true;
bool overlap = false;
u32 stride = 4;
};
constexpr LoadCase inputs[] = {
{"BufferLoadFormatD16XFloat16", F::k16Float, 0x7e63c000u, 0xabcdc000u},
{"BufferLoadFormatD16XFloat32", F::k32Float, 0x3f000000u, 0xabcd3800u},
{"BufferLoadFormatD16XUint32", F::k32UInt, 0x1234fffeu, 0xabcdffffu},
{"BufferLoadFormatD16XUintAtMax", F::k32UInt, 0x0000ffffu, 0xabcdffffu},
{"BufferLoadFormatD16XUintMax32", F::k32UInt, 0xffffffffu, 0xabcdffffu},
{"BufferLoadFormatD16XSint32", F::k32SInt, 0xffff8001u, 0xabcd8001u},
{"BufferLoadFormatD16XSintAboveMax", F::k32SInt, 0x00008000u, 0xabcd7fffu},
{"BufferLoadFormatD16XSintBelowMin", F::k32SInt, 0xffff7fffu, 0xabcd8000u},
{"BufferLoadFormatD16XSintAtMax", F::k32SInt, 0x00007fffu, 0xabcd7fffu},
{"BufferLoadFormatD16XSintAtMin", F::k32SInt, 0xffff8000u, 0xabcd8000u},
{"BufferLoadFormatD16XSintMax32", F::k32SInt, 0x7fffffffu, 0xabcd7fffu},
{"BufferLoadFormatD16XSintMin32", F::k32SInt, 0x80000000u, 0xabcd8000u},
{"BufferLoadFormatD16XUnorm8", F::k8UNorm, 0xffu, 0xabcd3c00u},
{"BufferLoadFormatD16XSnorm16", F::k16SNorm, 0x8001u, 0xabcdbc00u},
{"BufferLoadFormatD16XFloatOne", F::k32Float, 0u, 0xabcd3c00u, 1},
{"BufferLoadFormatD16XOobFloatOne", F::k32Float, 0u, 0xabcd3c00u, 1, 4},
{"BufferLoadFormatD16XOobUintOne", F::k32UInt, 0u, 0xabcd0001u, 1, 4},
{"BufferLoadFormatD16XOobZero", F::k32Float, 0u, 0xabcd0000u, 4, 4},
{"BufferLoadFormatD16XZeroStrideFloatOne", F::k32Float, 0u, 0xabcd3c00u, 1, 0, true, false, 0},
{"BufferLoadFormatD16XZeroStrideUintOne", F::k32UInt, 0u, 0xabcd0001u, 1, 0, true, false, 0},
{"BufferLoadFormatD16XInactive", F::k32Float, 0u, 0xabcd1234u, 4, 0, false},
{"BufferLoadFormatD16XOverlappingAddress", F::k16UInt, 7u, 7u, 4, 0, true, true},
};
std::vector<TestCase> cases;
for (const auto &input : inputs) {
TestCase test;
test.name = input.name;
AppendVMovLiteral(&test.code, 0, input.overlap ? input.index : 0xabcd1234u);
AppendVMovU32(&test.code, 1, input.index);
if (!input.active) {
test.code.push_back(EncodeSop1(0x04, 126, InlineU32(0)));
}
test.code.push_back(EncodeMubuf0(0x80u, 0, true, false));
test.code.push_back(EncodeMubuf1(0, 0, input.overlap ? 0 : 1));
if (!input.active) {
test.code.push_back(EncodeSMovB32(126, InlineU32(1)));
test.code.push_back(EncodeSMovB32(127, InlineU32(0)));
}
AppendStoreVgpr(&test.code, 0, 2);
AppendEnd(&test.code);
test.initial = {input.input, 0xdeadbeefu, 0u, 0xcafef00du};
test.expected = {input.input, 0xdeadbeefu, input.expected, 0xcafef00du};
test.user_data = MakeStructuredStorageBufferData(input.stride, 4, false, BufferFormat(input.format));
test.user_data[3] = (test.user_data[3] & ~0x30000007u) | input.selector;
test.has_user_data = true;
test.opcodes = {O::V_MOV_B32, O::BUFFER_LOAD_FORMAT_D16_X,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
cases.push_back(std::move(test));
}
return cases;
}
std::vector<TestCase> BufferLoadFormatD16XyzwCases() {
using F = Prospero::BufferFormat;
using O = ShaderOpcode;
struct LoadCase {
const char *name;
F format;
std::array<u32, 4> input;
std::array<u32, 2> expected;
u32 swizzle = DstSel(4, 5, 6, 7);
u32 stride = 16;
u32 index = 0;
bool active = true;
bool overlap = false;
};
constexpr LoadCase inputs[] = {
{"BufferLoadFormatD16XyzwFloat32", F::k32_32_32_32Float,
{0x3f800000u, 0x3f000000u, 0u, 0xc0000000u}, {0x38003c00u, 0xc0000000u}},
{"BufferLoadFormatD16XyzwFloat16", F::k16_16_16_16Float,
{0x38003c00u, 0xc0000000u}, {0x38003c00u, 0xc0000000u}},
{"BufferLoadFormatD16XyzwUintSaturates", F::k32_32_32_32UInt,
{0u, 0x10000u, 0xffffu, 7u}, {0xffff0000u, 0x0007ffffu}},
{"BufferLoadFormatD16XyzwSintSaturates", F::k32_32_32_32SInt,
{0xffff7fffu, 7u, 0x8000u, 0xffff8000u}, {0x00078000u, 0x80007fffu}},
{"BufferLoadFormatD16XyzwUnorm", F::k8_8_8_8UNorm,
{0xffff0000u}, {0u, 0x3c003c00u}},
{"BufferLoadFormatD16XyzwSwizzle", F::k32_32_32_32Float,
{0u, 0x3f800000u, 0xc0000000u, 0x3f000000u},
{0xc0003800u, 0x3c003c00u}, DstSel(7, 6, 5, 1)},
{"BufferLoadFormatD16XyzwPartialRecord", F::k32_32_32_32Float,
{0x3f800000u, 0x3f800000u, 0x3f800000u, 0x3f800000u},
{0u, 0u}, DstSel(4, 5, 6, 7), 12},
{"BufferLoadFormatD16XyzwOobOne", F::k32_32_32_32Float,
{}, {0x3c00u, 0x3c00u}, DstSel(1, 4, 1, 4), 16, 1},
{"BufferLoadFormatD16XyzwZeroStrideFloatOne", F::k32_32_32_32Float,
{}, {0x3c003c00u, 0x3c000000u}, DstSel(1, 1, 4, 1), 0},
{"BufferLoadFormatD16XyzwZeroStrideUintOne", F::k32_32_32_32UInt,
{}, {0x10000u, 1u}, DstSel(4, 1, 1, 4), 0},
{"BufferLoadFormatD16XyzwInactive", F::k32_32_32_32Float,
{}, {0xabcd1234u, 0xbeef5678u}, DstSel(4, 5, 6, 7), 16, 0, false},
{"BufferLoadFormatD16XyzwOverlappingAddress", F::k32_32_32_32Float,
{0x3f800000u, 0x3f000000u, 0u, 0xc0000000u},
{0x38003c00u, 0xc0000000u}, DstSel(4, 5, 6, 7), 16, 0, true, true},
};
std::vector<TestCase> cases;
for (const auto &input : inputs) {
TestCase test;
test.name = input.name;
AppendVMovLiteral(&test.code, 16, input.overlap ? input.index : 0xabcd1234u);
AppendVMovLiteral(&test.code, 17, 0xbeef5678u);
AppendVMovLiteral(&test.code, 18, 0xcafef00du);
AppendVMovU32(&test.code, 12, input.index);
if (!input.active) test.code.push_back(EncodeSop1(0x04, 126, InlineU32(0)));
// Captured b21e373811f28e31 pc 0x1914: e20c2000 8001100c.
test.code.push_back(EncodeMubuf0(0x83u, 0, true, false));
test.code.push_back(EncodeMubuf1(16, 1, input.overlap ? 16 : 12));
if (!input.active) {
test.code.push_back(EncodeSMovB32(126, InlineU32(1)));
test.code.push_back(EncodeSMovB32(127, InlineU32(0)));
}
for (u32 word = 0; word < 3; word++) AppendStoreVgpr(&test.code, 16 + word, 4 + word);
AppendEnd(&test.code);
test.initial.assign(input.input.begin(), input.input.end());
test.initial.resize(8, 0xdeadbeefu);
test.expected = test.initial;
test.expected[4] = input.expected[0];
test.expected[5] = input.expected[1];
test.expected[6] = 0xcafef00du;
const auto descriptor = MakeStructuredStorageBufferData(
input.stride, 1, false, BufferFormat(input.format));
std::copy_n(descriptor.begin(), 4, test.user_data.begin() + 4);
test.user_data[7] = (test.user_data[7] & ~0x30000fffu) | input.swizzle;
test.has_user_data = true;
if (input.stride != 0 && input.active) {
test.storage_buffer_ranges = {{0, input.stride}, {0, 32}};
}
test.opcodes = {O::V_MOV_B32, O::BUFFER_LOAD_FORMAT_D16_XYZW,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
cases.push_back(std::move(test));
}
return cases;
}
std::vector<TestCase> BufferStoreFormatD16Cases() {
using F = Prospero::BufferFormat;
using O = ShaderOpcode;
struct StoreCase {
const char *name;
F format;
std::array<u32, 2> input;
std::array<u32, 4> expected;
u32 words = 1;
u32 opcode = 0x84u;
};
constexpr StoreCase inputs[] = {
{"BufferStoreFormatD16XFloat16", F::k16Float, {0x7e63c000u}, {0xffffc000u}},
{"BufferStoreFormatD16XFloat32", F::k32Float, {0x7e633800u}, {0x3f000000u}},
{"BufferStoreFormatD16XUint32", F::k32UInt, {0x7e63fffeu}, {0x0000fffeu}},
{"BufferStoreFormatD16XSint32", F::k32SInt, {0x7e63fffeu}, {0xfffffffeu}},
{"BufferStoreFormatD16XUnorm8", F::k8UNorm, {0x7e633800u}, {0xffffff80u}},
{"BufferStoreFormatD16XSnorm16", F::k16SNorm, {0x7e63b800u}, {0xffffc000u}},
{"BufferStoreFormatD16XZeroFill", F::k16_16Float, {0x7e633800u}, {0x00003800u}},
{"BufferStoreFormatD16HiXUint16", F::k16UInt, {0xfffe1234u}, {0xfffeu}, 1, 0x27u},
{"BufferStoreFormatD16HiXFloat16", F::k16Float, {0xc0003c00u}, {0xc000u}, 1, 0x27u},
{"BufferStoreFormatD16HiXFloat32", F::k32Float, {0x3800c000u}, {0x3f000000u}, 1, 0x27u},
{"BufferStoreFormatD16HiXSint32", F::k32SInt, {0xfffe1234u}, {0xfffffffeu}, 1, 0x27u},
{"BufferStoreFormatD16HiXUnorm8", F::k8UNorm, {0x38003c00u}, {0x80u}, 1, 0x27u},
{"BufferStoreFormatD16HiXZeroFill", F::k16_16Float, {0x38003c00u}, {0x00003800u}, 1, 0x27u},
{"BufferStoreFormatD16XyzwFloat16", F::k16_16_16_16Float,
{0xc0003c00u, 0x34003800u}, {0xc0003c00u, 0x34003800u}, 2, 0x87u},
{"BufferStoreFormatD16XyzwFloat32", F::k32_32_32_32Float,
{0xc0003c00u, 0x34003800u}, {0x3f800000u, 0xc0000000u, 0x3f000000u, 0x3e800000u}, 4, 0x87u},
{"BufferStoreFormatD16XyzwUint32", F::k32_32_32_32UInt,
{0xffff0001u, 0x80007fffu}, {1u, 0xffffu, 0x7fffu, 0x8000u}, 4, 0x87u},
{"BufferStoreFormatD16XyzwSint32", F::k32_32_32_32SInt,
{0x8000ffffu, 0x7fff0001u}, {0xffffffffu, 0xffff8000u, 1u, 0x7fffu}, 4, 0x87u},
{"BufferStoreFormatD16XyzwSint8", F::k8_8_8_8SInt,
{0xff80ff7fu, 0x0080007fu}, {0x7f7f8080u}, 1, 0x87u},
{"BufferStoreFormatD16XyzwUnorm8", F::k8_8_8_8UNorm,
{0x3800c000u, 0x40003c00u}, {0xffff8000u}, 1, 0x87u},
};
std::vector<TestCase> cases;
for (const auto &input : inputs)
for (const u32 wave : {32u, 64u})
for (u32 mask = 0; mask < (input.opcode == 0x27u ? 2u : 1u); ++mask) {
const bool wide = input.opcode == 0x87u;
const bool high = input.opcode == 0x27u;
const bool multi_lane = wide || high;
const bool masked = wide || mask != 0;
if (!multi_lane && wave == 64u) continue;
const u32 payload_bytes = Prospero::NumBytesPerElement(input.format);
const u32 stride = high ? payload_bytes : input.words * 4u;
TestCase test;
test.name = input.name;
AppendVMovLiteral(&test.code, 66, input.input[0]);
if (wide) AppendVMovLiteral(&test.code, 67, input.input[1]);
if (multi_lane) {
test.code.push_back(EncodeVop2(0x25, 65, InlineU32(1), 0));
} else AppendVMovU32(&test.code, 65, 1);
if (masked) {
AppendSMovLiteral(&test.code, 126, 0x55555555u);
AppendSMovLiteral(&test.code, 127, wave == 64u ? 0x55555555u : 0u);
}
const auto store = [&] {
test.code.push_back(EncodeMubuf0(input.opcode, 0, true, false, multi_lane));
test.code.push_back(EncodeMubuf1(66, 0, 65));
};
store();
if (multi_lane) {
AppendVMovU32(&test.code, 65, wave + 1u);
store(); // Active lanes must not write beyond the descriptor range.
AppendVMovU32(&test.code, 65, 0u);
test.code.push_back(EncodeSop1(0x04, 126, InlineU32(0)));
store(); // EXEC-zero must preserve the leading record.
}
AppendEnd(&test.code);
if (multi_lane) {
test.initial.assign(((wave + 2u) * stride + 3u) / 4u, 0xdeadbeefu);
test.expected = test.initial;
auto expected_bytes = std::as_writable_bytes(std::span(test.expected));
const auto payload = std::as_bytes(std::span(input.expected));
for (u32 lane = 0; lane + 2u < wave; lane += masked ? 2u : 1u)
std::copy_n(payload.begin(), payload_bytes,
expected_bytes.begin() + (lane + 1u) * stride);
test.storage_buffer_range_bytes = wave * stride - 1u;
test.has_compute_info = true;
test.compute_info.wave_size = wave;
test.compute_info.threads_num[0] = wave;
test.compute_info.threads_num[1] = test.compute_info.threads_num[2] = 1u;
test.compute_info.thread_ids_num = 1u;
test.required_spirv = {"Volatile"};
} else {
test.initial = {0xdeadbeefu, 0xffffffffu, 0xcafef00du};
test.expected = {0xdeadbeefu, input.expected[0], 0xcafef00du};
}
test.user_data = MakeStructuredStorageBufferData(stride, multi_lane ? wave : 3u,
false, BufferFormat(input.format));
test.has_user_data = true;
test.opcodes = {O::V_MOV_B32, high ? O::BUFFER_STORE_FORMAT_D16_HI_X :
wide ? O::BUFFER_STORE_FORMAT_D16_XYZW :
O::BUFFER_STORE_FORMAT_D16_X, O::S_ENDPGM};
if (high) test.decoded_counts = {{"BUFFER_STORE_FORMAT_D16_HI_X", 3u}};
cases.push_back(std::move(test));
}
return cases;
}
std::vector<TestCase> FormattedStoreConversionCases() {
using F = Prospero::BufferFormat;
using O = ShaderOpcode;
struct ConversionCase {
const char *name[2];
F format;
u32 components;
std::array<float, 4> values;
u32 expected;
};
constexpr float nan = std::bit_cast<float>(0x7fc12345u);
constexpr float inf = std::bit_cast<float>(0x7f800000u);
constexpr ConversionCase inputs[] = {
{{"BufferStoreSnorm8Range", "TBufferStoreSnorm8Range"},
F::k8_8_8_8SNorm, 4, {-2.0f, 0.25f, 0.75f, 2.0f}, 0x7f5f2081u},
{{"BufferStoreSnorm8Ties", "TBufferStoreSnorm8Ties"},
F::k8_8SNorm, 2, {0.5f, -0.5f}, 0xffffc040u},
{{"BufferStoreSnormPackedAlpha", "TBufferStoreSnormPackedAlpha"},
F::k10_10_10_2SNorm, 4, {-1.0f, -0.5f, 0.5f, 1.0f}, 0xd00c0201u},
{{"BufferStoreSnormPackedFirstChannel", "TBufferStoreSnormPackedFirstChannel"},
F::k2_10_10_10SNorm, 4, {1.0f, -1.0f, 0.5f, -0.5f}, 0xc0100807u},
{{"BufferStoreSnormPackedZeroFillX", "TBufferStoreSnormPackedZeroFillX"},
F::k10_10_10_2SNorm, 1, {-1.0f}, 0x00000201u},
{{"BufferStoreSnormPackedZeroFillXy", "TBufferStoreSnormPackedZeroFillXy"},
F::k10_10_10_2SNorm, 2, {-1.0f, 1.0f}, 0x0007fe01u},
{{"BufferStoreSnormPackedZeroFillXyz", "TBufferStoreSnormPackedZeroFillXyz"},
F::k10_10_10_2SNorm, 3, {-1.0f, 1.0f, -1.0f}, 0x2017fe01u},
{{"BufferStoreSnorm8ZeroFillX", "TBufferStoreSnorm8ZeroFillX"},
F::k8_8_8_8SNorm, 1, {-1.0f}, 0x00000081u},
{{"BufferStoreSnorm16ZeroFillX", "TBufferStoreSnorm16ZeroFillX"},
F::k16_16SNorm, 1, {-1.0f}, 0x00008001u},
{{"BufferStoreUscaledZeroFillXy", "TBufferStoreUscaledZeroFillXy"},
F::k8_8_8_8UScaled, 2, {42.0f, 99.0f}, 0x0000632au},
{{"BufferStoreSscaledZeroFillXyz", "TBufferStoreSscaledZeroFillXyz"},
F::k8_8_8_8SScaled, 3, {-42.0f, 127.0f, -128.0f}, 0x00807fd6u},
{{"BufferStoreUscaled8Fraction", "TBufferStoreUscaled8Fraction"},
F::k8UScaled, 1, {42.875f}, 0xffffff2au},
{{"BufferStoreSscaled8Fraction", "TBufferStoreSscaled8Fraction"},
F::k8SScaled, 1, {-42.875f}, 0xffffffd6u},
{{"BufferStoreUscaled8Range", "TBufferStoreUscaled8Range"},
F::k8_8_8_8UScaled, 4, {-1.0f, 255.875f, 256.0f, 0.0f}, 0x00ffff00u},
{{"BufferStoreSscaled8Range", "TBufferStoreSscaled8Range"},
F::k8_8_8_8SScaled, 4, {-129.0f, -128.75f, 127.875f, 128.0f}, 0x7f7f8080u},
{{"BufferStoreUscaled16Range", "TBufferStoreUscaled16Range"},
F::k16_16UScaled, 2, {65536.0f, 42.875f}, 0x002affffu},
{{"BufferStoreSscaled16Range", "TBufferStoreSscaled16Range"},
F::k16_16SScaled, 2, {-32769.0f, 32768.0f}, 0x7fff8000u},
{{"BufferStoreUscaledPacked", "TBufferStoreUscaledPacked"},
F::k10_10_10_2UScaled, 4, {1024.0f, 3.875f, -1.0f, 4.0f}, 0xc0000fffu},
{{"BufferStoreSscaledPackedAlpha", "TBufferStoreSscaledPackedAlpha"},
F::k10_10_10_2SScaled, 4, {-512.75f, 511.875f, -3.875f, 3.875f}, 0xffd7fe00u},
{{"BufferStoreSscaledPackedFirstChannel", "TBufferStoreSscaledPackedFirstChannel"},
F::k2_10_10_10SScaled, 4, {3.0f, -512.0f, 511.0f, -1.0f}, 0xffdff803u},
{{"BufferStoreSnorm8SpecialValues", "TBufferStoreSnorm8SpecialValues"},
F::k8_8_8_8SNorm, 4, {nan, inf, -inf, -0.0f}, 0x00817f00u},
{{"BufferStoreUscaled8SpecialValues", "TBufferStoreUscaled8SpecialValues"},
F::k8_8_8_8UScaled, 4, {nan, inf, -inf, -0.0f}, 0x0000ff00u},
{{"BufferStoreSscaled8SpecialValues", "TBufferStoreSscaled8SpecialValues"},
F::k8_8_8_8SScaled, 4, {nan, inf, -inf, -0.0f}, 0x00807f00u},
};
constexpr O store_opcodes[2][4] = {
{O::BUFFER_STORE_FORMAT_X, O::BUFFER_STORE_FORMAT_XY,
O::BUFFER_STORE_FORMAT_XYZ, O::BUFFER_STORE_FORMAT_XYZW},
{O::TBUFFER_STORE_FORMAT_X, O::TBUFFER_STORE_FORMAT_XY,
O::TBUFFER_STORE_FORMAT_XYZ, O::TBUFFER_STORE_FORMAT_XYZW},
};
std::vector<TestCase> cases;
for (const auto &input : inputs) {
for (const bool typed : {false, true}) {
TestCase test;
test.name = input.name[typed];
for (u32 component = 0; component < input.components; ++component) {
AppendVMovLiteral(&test.code, component,
std::bit_cast<u32>(input.values[component]));
}
AppendVMovU32(&test.code, 20, 4);
const u32 opcode = 3 + input.components;
if (typed) {
const u32 format = BufferFormat(input.format);
test.code.push_back(EncodeMtbuf0(opcode, format & 0xfu, format >> 4u));
test.code.push_back(EncodeMtbuf1(opcode, 0, 0, 20));
} else {
test.code.push_back(EncodeMubuf0(opcode));
test.code.push_back(EncodeMubuf1(0, 0, 20));
}
AppendEnd(&test.code);
test.initial = {0xdeadbeefu, 0xffffffffu, 0xcafef00du};
test.expected = {0xdeadbeefu, input.expected, 0xcafef00du};
test.user_data = MakeStructuredStorageBufferData(
0, 12, false, BufferFormat(input.format));
test.has_user_data = true;
test.opcodes = {O::V_MOV_B32, store_opcodes[typed][input.components - 1],
O::S_ENDPGM};
cases.push_back(std::move(test));
}
}
return cases;
}
TestCase BufferLoadFormatXyResource88UintExtractsBytes() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 20, 0);
code.push_back(EncodeMubuf0(0x01u));
code.push_back(EncodeMubuf1(0, 0, 20));
AppendStoreVgpr(&code, 0, 2);
AppendStoreVgpr(&code, 1, 3);
AppendEnd(&code);
TestCase test;
test.name = "BufferLoadFormatXyResource8_8UintExtractsBytes";
test.code = code;
test.initial = {0x0000807fu, 0x55667788u, 0, 0};
test.expected = {0x0000807fu, 0x55667788u, 0x0000007fu, 0x00000080u};
test.opcodes = {O::V_MOV_B32, O::BUFFER_LOAD_FORMAT_XY, O::BUFFER_STORE_DWORD,
O::S_ENDPGM};
test.user_data = MakeStructuredStorageBufferData(0, 8, false, 18);
test.has_user_data = true;
return test;
}
TestCase BufferLoadFormatXyResource8888UnormConvertsFirstTwoComponents() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 20, 0);
code.push_back(EncodeMubuf0(0x01u));
code.push_back(EncodeMubuf1(0, 0, 20));
AppendStoreVgpr(&code, 0, 0);
AppendStoreVgpr(&code, 1, 1);
AppendEnd(&code);
TestCase test;
test.name =
"BufferLoadFormatXyResource8_8_8_8UnormConvertsFirstTwoComponents";
test.code = code;
test.initial = {0x44332211u, 0xdeadbeefu};
test.expected = {0x3d888889u, 0x3e088889u};
test.opcodes = {O::V_MOV_B32, O::BUFFER_LOAD_FORMAT_XY, O::BUFFER_STORE_DWORD,
O::S_ENDPGM};
test.user_data = MakeStructuredStorageBufferData(0, 8, false, 56);
test.has_user_data = true;
return test;
}
TestCase BufferStoreFormatXyResource88UintWritesBytes() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 20, 0);
AppendVMovLiteral(&code, 0, 0x000000aau);
AppendVMovLiteral(&code, 1, 0x000000bbu);
code.push_back(EncodeMubuf0(0x05u));
code.push_back(EncodeMubuf1(0, 0, 20));
AppendEnd(&code);
TestCase test;
test.name = "BufferStoreFormatXyResource8_8UintWritesBytes";
test.code = code;
test.initial = {0x11223344u, 0x55667788u};
test.expected = {0x1122bbaau, 0x55667788u};
test.opcodes = {O::V_MOV_B32, O::BUFFER_STORE_FORMAT_XY, O::S_ENDPGM};
test.user_data = MakeStructuredStorageBufferData(0, 8, false, 18);
test.has_user_data = true;
return test;
}
TestCase BufferStoreFormatXyResource32UintWritesOneDword() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 20, 0);
AppendVMovLiteral(&code, 0, 0xabcdef01u);
AppendVMovLiteral(&code, 1, 0x12345678u);
code.push_back(EncodeMubuf0(0x05u));
code.push_back(EncodeMubuf1(0, 0, 20));
AppendEnd(&code);
TestCase test;
test.name = "BufferStoreFormatXyResource32UintWritesOneDword";
test.code = code;
test.initial = {0x11111111u, 0x22222222u};
test.expected = {0xabcdef01u, 0x22222222u};
test.opcodes = {O::V_MOV_B32, O::BUFFER_STORE_FORMAT_XY, O::S_ENDPGM};
test.user_data = MakeStructuredStorageBufferData(0, 8, false, 20);
test.has_user_data = true;
return test;
}
TestCase BufferStoreFormatXyzResource3232UintWritesTwoDwords() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 20, 0);
AppendVMovLiteral(&code, 0, 0xabcdef01u);
AppendVMovLiteral(&code, 1, 0x12345678u);
AppendVMovLiteral(&code, 2, 0x0badc0deu);
code.push_back(EncodeMubuf0(0x06u));
code.push_back(EncodeMubuf1(0, 0, 20));
AppendEnd(&code);
TestCase test;
test.name = "BufferStoreFormatXyzResource32_32UintWritesTwoDwords";
test.code = code;
test.initial = {0x11111111u, 0x22222222u, 0x33333333u};
test.expected = {0xabcdef01u, 0x12345678u, 0x33333333u};
test.opcodes = {O::V_MOV_B32, O::BUFFER_STORE_FORMAT_XYZ, O::S_ENDPGM};
test.user_data = MakeStructuredStorageBufferData(0, 8, false, 62);
test.has_user_data = true;
return test;
}
TestCase BufferStoreFormatXyzwResource323232UintWritesThreeDwords() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 20, 0);
AppendVMovLiteral(&code, 0, 0xabcdef01u);
AppendVMovLiteral(&code, 1, 0x12345678u);
AppendVMovLiteral(&code, 2, 0x0badc0deu);
AppendVMovLiteral(&code, 3, 0xfeedfaceu);
code.push_back(EncodeMubuf0(0x07u));
code.push_back(EncodeMubuf1(0, 0, 20));
AppendEnd(&code);
TestCase test;
test.name = "BufferStoreFormatXyzwResource32_32_32UintWritesThreeDwords";
test.code = code;
test.initial = {0x11111111u, 0x22222222u, 0x33333333u, 0x44444444u};
test.expected = {0xabcdef01u, 0x12345678u, 0x0badc0deu, 0x44444444u};
test.opcodes = {O::V_MOV_B32, O::BUFFER_STORE_FORMAT_XYZW, O::S_ENDPGM};
test.user_data = MakeStructuredStorageBufferData(0, 8, false, 72);
test.has_user_data = true;
return test;
}
TestCase BufferStoreFormatXyzResource8UintWritesOneByte() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 20, 0);
AppendVMovLiteral(&code, 0, 0x000000aau);
AppendVMovLiteral(&code, 1, 0x000000bbu);
AppendVMovLiteral(&code, 2, 0x000000ccu);
code.push_back(EncodeMubuf0(0x06u));
code.push_back(EncodeMubuf1(0, 0, 20));
AppendEnd(&code);
TestCase test;
test.name = "BufferStoreFormatXyzResource8UintWritesOneByte";
test.code = code;
test.initial = {0x11223344u, 0x55667788u, 0x99aabbccu};
test.expected = {0x112233aau, 0x55667788u, 0x99aabbccu};
test.opcodes = {O::V_MOV_B32, O::BUFFER_STORE_FORMAT_XYZ, O::S_ENDPGM};
test.user_data = MakeStructuredStorageBufferData(0, 8, false, 5);
test.has_user_data = true;
return test;
}
TestCase BufferStoreFormatXAddTidUsesLaneIndex() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovLiteral(&code, 0, 0x12345678u);
code.push_back(EncodeMubuf0(0x04u, 0, false, false));
code.push_back(EncodeMubuf1(0, 0, 20));
AppendEnd(&code);
TestCase test;
test.name = "BufferStoreFormatXAddTidUsesLaneIndex";
test.code = code;
test.initial = std::vector<u32>(64, 0);
test.expected = std::vector<u32>(64, 0);
std::fill_n(test.expected.begin(), 32, 0x12345678u);
test.opcodes = {O::V_MOV_B32, O::BUFFER_STORE_FORMAT_X, O::S_ENDPGM};
test.compute_info.threads_num[0] = 64;
test.compute_info.threads_num[1] = 1;
test.compute_info.threads_num[2] = 1;
test.compute_info.thread_ids_num = 0;
test.compute_info.wave_size = 32;
test.has_compute_info = true;
test.user_data = MakeStructuredStorageBufferData(4, 64, true);
test.has_user_data = true;
test.required_spirv = {"BuiltIn SubgroupLocalInvocationId"};
test.forbidden_spirv = {"BuiltIn LocalInvocationId",
"OpGroupNonUniformBallot"};
return test;
}
TestCase BufferStoreFormatXDropsOutOfRangeRecord() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 20, 1);
AppendVMovLiteral(&code, 0, 0xabcdef01u);
code.push_back(EncodeMubuf0(0x04u, 0, true, false));
code.push_back(EncodeMubuf1(0, 0, 20));
AppendEnd(&code);
TestCase test;
test.name = "BufferStoreFormatXDropsOutOfRangeRecord";
test.code = code;
test.initial = {0x11111111u, 0x22222222u};
test.expected = {0x11111111u, 0x22222222u};
test.storage_buffer_range_bytes = 4;
test.opcodes = {O::V_MOV_B32, O::BUFFER_STORE_FORMAT_X, O::S_ENDPGM};
test.user_data = MakeStructuredStorageBufferData(4, 1, false, 20);
test.has_user_data = true;
return test;
}
TestCase TBufferCapturedZeroStrideOob(bool raw_bounds = false, bool scalar = false) {
using O = ShaderOpcode;
TestCase test;
test.name = scalar ? "TBufferZeroStrideOobPreservesScalarRead"
: raw_bounds ? "TBufferZeroStrideRawBoundsStillLoads"
: "TBufferCapturedZeroStrideOob";
test.initial = {0x11223344u, 0x55667788u, 0x99aabbccu, 0xddeeff00u,
0x12345678u, 0xdeadbeefu, 0xdeadbeefu};
test.expected = test.initial;
for (u32 i = 0; i < 4; i++) {
test.expected[1 + i] = raw_bounds ? test.initial[i] : 0u;
}
test.expected[5] = test.initial[4];
if (scalar) test.expected[6] = test.initial[0];
test.user_data = MakeStructuredStorageBufferData(4, test.initial.size());
std::copy_n(test.user_data.begin(), 4, test.user_data.begin() + 4);
test.user_data[4] = 0x2000u;
// Captured HITMAN descriptor in s[32:35]: mode 0, zero stride, arbitrary base.
test.user_data[32] = 0x4a398620u;
test.user_data[33] = 0;
test.user_data[34] = 0x2b5u;
test.user_data[35] = 0x0004d000u | (raw_bounds ? 3u << 28u : 0u);
test.has_user_data = true;
test.storage_buffer_offsets = {0, 0, 16};
test.expected_buffer_resources = raw_bounds || scalar
? std::vector<u32>{0, 1, 2}
: std::vector<u32>{0, 2};
auto &code = test.code;
AppendVMovLiteral(&code, 0, test.initial[0]);
AppendStoreVgpr(&code, 0, 0); // Keep resource zero before the removed middle slot.
AppendVMovU32(&code, 30, 0);
code.push_back(0xea6b2000u);
code.push_back(0x8008191eu); // Exact pc 0x47c: typed float4, s[32:35], v30.
AppendVMovU32(&code, 20, 0);
code.push_back(EncodeMubuf0(0x0cu));
code.push_back(EncodeMubuf1(33, 1, 20));
if (scalar) {
code.push_back(EncodeSmem0(0x08u, 40, 16));
code.push_back(EncodeSmem1(0));
code.push_back(EncodeVop1(0x01u, 32, 40));
}
for (u32 i = 0; i < 4; i++) AppendStoreVgpr(&code, 25 + i, 1 + i);
AppendStoreVgpr(&code, 33, 5);
if (scalar) AppendStoreVgpr(&code, 32, 6);
AppendEnd(&code);
test.opcodes = {O::V_MOV_B32, O::TBUFFER_LOAD_FORMAT_XYZW,
O::BUFFER_LOAD_DWORD, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
if (scalar) test.opcodes.push_back(O::S_BUFFER_LOAD_DWORD);
test.ir_counts = {{"LoadBufferU32x4", raw_bounds ? 1u : 0u}};
return test;
}
TestCase BufferZeroStrideOobFormatsAndWidths() {
using O = ShaderOpcode;
TestCase test;
test.name = "BufferZeroStrideOobFormatsAndWidths";
test.initial = std::vector<u32>(20, 0xdeadbeefu);
test.expected = std::vector<u32>(20, 0);
test.expected[10] = 0x3f800000u;
test.expected[12] = 0x3f800000u;
test.expected[14] = 1;
test.expected[16] = 1;
test.expected[18] = 0xabcdef01u;
test.expected[19] = 0xdeadbeefu;
test.user_data = MakeStructuredStorageBufferData(4, 20);
for (u32 base : {4u, 8u}) {
test.user_data[base] = 0x4a398620u;
test.user_data[base + 2] = 0x2b5u;
const u32 format = base == 4 ? 77u : 75u; // Four 32-bit float or unsigned integer components.
test.user_data[base + 3] = (format << 12u) | DstSel(1, 4, 1, 0);
}
test.has_user_data = true;
test.expected_buffer_resources = std::vector<u32>{0};
auto &code = test.code;
AppendVMovU32(&code, 0, 0);
AppendStoreVgpr(&code, 0, 0);
AppendVMovU32(&code, 20, 0);
u32 destination = 0;
for (const auto [opcode, width] :
{std::pair{0x08u, 1u}, {0x0au, 1u}, {0x0cu, 1u}, {0x0du, 2u}, {0x0fu, 3u}}) {
code.push_back(EncodeMubuf0(opcode));
code.push_back(EncodeMubuf1(destination, 1, 20));
destination += width;
}
AppendVMovU32(&code, 8, 0);
AppendVMovU32(&code, 9, 0);
for (u32 base : {1u, 2u}) {
code.push_back(EncodeMubuf0(0x03u));
code.push_back(EncodeMubuf1(base == 1 ? 10 : 14, base, 20));
}
AppendVMovLiteral(&code, 18, 0xabcdef01u);
code.push_back(EncodeSop1(0x04u, 60, 126u)); // Preserve the partial wave's active lanes.
code.push_back(EncodeSop1(0x04u, 126, InlineU32(0)));
code.push_back(EncodeMubuf0(0x00u));
code.push_back(EncodeMubuf1(18, 1, 20));
code.push_back(EncodeSop1(0x04u, 126, 60u));
for (u32 i = 0; i < 19; i++) AppendStoreVgpr(&code, i, i);
AppendEnd(&code);
test.opcodes = {O::V_MOV_B32, O::S_MOV_B64, O::BUFFER_LOAD_UBYTE,
O::BUFFER_LOAD_USHORT, O::BUFFER_LOAD_DWORD,
O::BUFFER_LOAD_DWORDX2, O::BUFFER_LOAD_DWORDX3,
O::BUFFER_LOAD_FORMAT_X, O::BUFFER_LOAD_FORMAT_XYZW,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.ir_counts = {{"LoadBuffer", 0}};
return test;
}
TestCase TBufferLoadVariants() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 20, 0);
AppendTBufferLoadOpcode(&code, 0x00, 0, 20);
AppendTBufferLoadOpcode(&code, 0x01, 1, 20);
AppendTBufferLoadOpcode(&code, 0x02, 3, 20);
AppendTBufferLoadOpcode(&code, 0x03, 6, 20);
for (u32 i = 0; i < 10u; i++) {
AppendStoreVgpr(&code, i, i);
}
AppendEnd(&code);
return {"TBufferLoadVariants",
code,
{0x01020304u, 0x11121314u, 0x21222324u, 0x31323334u},
{0x01020304u, 0x01020304u, 0x11121314u, 0x01020304u, 0x11121314u,
0x21222324u, 0x01020304u, 0x11121314u, 0x21222324u, 0x31323334u},
{O::TBUFFER_LOAD_FORMAT_X, O::TBUFFER_LOAD_FORMAT_XY,
O::TBUFFER_LOAD_FORMAT_XYZ, O::TBUFFER_LOAD_FORMAT_XYZW,
O::V_MOV_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}
TestCase TBufferLoadFormatXyzwSnapshotsOverlappingAddress() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 21, 0);
AppendVMovU32(&code, 22, 0);
constexpr auto format =
BufferFormat(Prospero::BufferFormat::k32_32_32_32Float);
code.push_back(
EncodeMtbuf0(0x03u, format & 0xfu, (format >> 4u) & 0x7u, 0, true, true));
code.push_back(EncodeMtbuf1(0x03u, 21, 0, 21));
for (u32 i = 0; i < 4; i++) {
AppendStoreVgpr(&code, 21 + i, 4 + i);
}
AppendEnd(&code);
TestCase test;
test.name = "TBufferLoadFormatXyzwSnapshotsOverlappingAddress";
test.code = std::move(code);
test.initial = {0x3f800000u, 0x40000000u, 0x40400000u, 0x40800000u,
0, 0, 0, 0};
test.expected = {0x3f800000u, 0x40000000u, 0x40400000u, 0x40800000u,
0x3f800000u, 0x40000000u, 0x40400000u, 0x40800000u};
test.opcodes = {O::V_MOV_B32, O::TBUFFER_LOAD_FORMAT_XYZW,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.user_data = MakeStructuredStorageBufferData(16, 2);
test.has_user_data = true;
return test;
}
TestCase TBufferLoadFormatXyzwPackedSnapshotsOverlappingAddress() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 21, 0);
AppendVMovU32(&code, 22, 0);
constexpr auto format = BufferFormat(Prospero::BufferFormat::k8_8_8_8UInt);
code.push_back(
EncodeMtbuf0(0x03u, format & 0xfu, (format >> 4u) & 0x7u, 0, true, true));
code.push_back(EncodeMtbuf1(0x03u, 21, 0, 21));
for (u32 i = 0; i < 4; i++) {
AppendStoreVgpr(&code, 21 + i, 4 + i);
}
AppendEnd(&code);
TestCase test;
test.name = "TBufferLoadFormatXyzwPackedSnapshotsOverlappingAddress";
test.code = std::move(code);
test.initial = {0x44332211u, 0, 0, 0, 0, 0, 0, 0};
test.expected = {0x44332211u, 0, 0, 0, 0x11u, 0x22u, 0x33u, 0x44u};
test.opcodes = {O::V_MOV_B32, O::TBUFFER_LOAD_FORMAT_XYZW,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.user_data = MakeStructuredStorageBufferData(4, 8);
test.has_user_data = true;
return test;
}
TestCase TBufferStoreFormatX8UintWritesOneByte() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 20, 0);
AppendVMovU32(&code, 0, 0xaa);
code.push_back(EncodeMtbuf0(0x04, 5, 0));
code.push_back(EncodeMtbuf1(0x04, 0, 0, 20));
AppendEnd(&code);
return {"TBufferStoreFormatX8UintWritesOneByte",
code,
{0x11223344u},
{0x112233aau},
{O::V_MOV_B32, O::TBUFFER_STORE_FORMAT_X, O::S_ENDPGM}};
}
TestCase TBufferLoadFormatX8UintZeroExtendsByte() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 20, 0);
code.push_back(EncodeMtbuf0(0x00, 5, 0));
code.push_back(EncodeMtbuf1(0x00, 0, 0, 20));
AppendStoreVgpr(&code, 0, 0);
AppendEnd(&code);
return {"TBufferLoadFormatX8UintZeroExtendsByte",
code,
{0x11223344u, 0},
{0x00000044u, 0},
{O::V_MOV_B32, O::TBUFFER_LOAD_FORMAT_X, O::BUFFER_STORE_DWORD,
O::S_ENDPGM}};
}
TestCase TBufferLoadFormatX8888UintExtractsFirstByte() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 20, 0);
AppendTBufferLoadFormatOpcode(&code, 0x00, 0, 20,
Prospero::BufferFormat::k8_8_8_8UInt);
AppendStoreVgpr(&code, 0, 0);
AppendEnd(&code);
return {"TBufferLoadFormatX8_8_8_8UintExtractsFirstByte",
code,
{0x44332211u, 0},
{0x00000011u, 0},
{O::V_MOV_B32, O::TBUFFER_LOAD_FORMAT_X, O::BUFFER_STORE_DWORD,
O::S_ENDPGM}};
}
TestCase TBufferLoadFormatXIdxenUsesDescriptorStride() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 20, 2);
code.push_back(EncodeMtbuf0(0x00, 5, 0, 0, true, false));
code.push_back(EncodeMtbuf1(0x00, 0, 0, 20));
AppendStoreVgpr(&code, 0, 0);
AppendEnd(&code);
TestCase test;
test.name = "TBufferLoadFormatXIdxenUsesDescriptorStride";
test.code = code;
test.initial = {0x01020304u, 0, 0, 0, 0, 0, 0, 0, 0x0000007eu};
test.expected = {0x0000007eu, 0, 0, 0, 0, 0, 0, 0, 0x0000007eu};
test.opcodes = {O::V_MOV_B32, O::TBUFFER_LOAD_FORMAT_X, O::BUFFER_STORE_DWORD,
O::S_ENDPGM};
test.user_data = MakeStructuredStorageBufferData(16, 3);
test.has_user_data = true;
return test;
}
TestCase TBufferLoadFormatX16FloatConvertsToFloat() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 20, 0);
AppendTBufferLoadFormatOpcode(&code, 0x00, 0, 20,
Prospero::BufferFormat::k16Float);
AppendStoreVgpr(&code, 0, 0);
AppendEnd(&code);
return {"TBufferLoadFormatX16FloatConvertsToFloat",
code,
{0x00003c00u, 0},
{0x3f800000u, 0},
{O::V_MOV_B32, O::TBUFFER_LOAD_FORMAT_X, O::BUFFER_STORE_DWORD,
O::S_ENDPGM}};
}
TestCase TBufferLoadFormatXSintSignExtendsSubDword() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 20, 0);
code.push_back(EncodeMtbuf0(0x00, 6, 0));
code.push_back(EncodeMtbuf1(0x00, 0, 0, 20));
AppendVMovU32(&code, 20, 4);
code.push_back(EncodeMtbuf0(0x00, 12, 0));
code.push_back(EncodeMtbuf1(0x00, 1, 0, 20));
AppendStoreVgpr(&code, 0, 0);
AppendStoreVgpr(&code, 1, 1);
AppendEnd(&code);
return {"TBufferLoadFormatXSintSignExtendsSubDword",
code,
{0x00000080u, 0x00008001u},
{0xffffff80u, 0xffff8001u},
{O::V_MOV_B32, O::TBUFFER_LOAD_FORMAT_X, O::BUFFER_STORE_DWORD,
O::S_ENDPGM}};
}
TestCase TBufferStoreFormatXSintWritesSubDword() {
using O = ShaderOpcode;
TestCase test;
test.name = "TBufferStoreFormatXSintWritesSubDword";
constexpr std::array<u32, 6> inputs{0xffffff80u, 0xffffff7fu, 128u,
0xffff8001u, 0xffff7fffu, 32768u};
constexpr std::array<u32, 6> encoded{0x80u, 0x80u, 0x7fu, 0x8001u, 0x8000u, 0x7fffu};
test.initial.assign(inputs.size(), 0x11223344u);
test.expected = test.initial;
for (u32 index = 0; index < inputs.size(); ++index) {
AppendVMovU32(&test.code, 20, index * 4u);
AppendVMovLiteral(&test.code, 0, inputs[index]);
test.code.push_back(EncodeMtbuf0(0x04, index < 3u ? 6u : 12u, 0));
test.code.push_back(EncodeMtbuf1(0x04, 0, 0, 20));
test.expected[index] = (test.expected[index] & (index < 3u ? 0xffffff00u : 0xffff0000u)) | encoded[index];
}
AppendEnd(&test.code);
test.opcodes = {O::V_MOV_B32, O::TBUFFER_STORE_FORMAT_X, O::S_ENDPGM};
return test;
}
TestCase TBufferLoadFormatXy88IntegerComponents() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 20, 0);
code.push_back(EncodeMtbuf0(0x01, 2, 1));
code.push_back(EncodeMtbuf1(0x01, 0, 0, 20));
AppendVMovU32(&code, 20, 4);
code.push_back(EncodeMtbuf0(0x01, 3, 1));
code.push_back(EncodeMtbuf1(0x01, 2, 0, 20));
for (u32 i = 0; i < 4; i++) {
AppendStoreVgpr(&code, i, i);
}
AppendEnd(&code);
return {"TBufferLoadFormatXy8_8IntegerComponents",
code,
{0x0000807fu, 0x00007f80u, 0, 0},
{0x0000007fu, 0x00000080u, 0xffffff80u, 0x0000007fu},
{O::V_MOV_B32, O::TBUFFER_LOAD_FORMAT_XY, O::BUFFER_STORE_DWORD,
O::S_ENDPGM}};
}
TestCase TBufferStoreFormatXy88IntegerComponents() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 20, 0);
AppendVMovU32(&code, 0, 0xaa);
AppendVMovU32(&code, 1, 0xbb);
code.push_back(EncodeMtbuf0(0x05, 2, 1));
code.push_back(EncodeMtbuf1(0x05, 0, 0, 20));
AppendVMovU32(&code, 20, 4);
AppendVMovLiteral(&code, 2, 0xffffff80u);
AppendVMovU32(&code, 3, 0x7f);
code.push_back(EncodeMtbuf0(0x05, 3, 1));
code.push_back(EncodeMtbuf1(0x05, 2, 0, 20));
AppendEnd(&code);
return {"TBufferStoreFormatXy8_8IntegerComponents",
code,
{0x11223344u, 0x55667788u},
{0x1122bbaau, 0x55667f80u},
{O::V_MOV_B32, O::TBUFFER_STORE_FORMAT_XY, O::S_ENDPGM}};
}
TestCase TBufferLoadFormatXy1616IntegerComponents() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 20, 0);
code.push_back(EncodeMtbuf0(0x01, 11, 1));
code.push_back(EncodeMtbuf1(0x01, 0, 0, 20));
AppendVMovU32(&code, 20, 4);
code.push_back(EncodeMtbuf0(0x01, 12, 1));
code.push_back(EncodeMtbuf1(0x01, 2, 0, 20));
for (u32 i = 0; i < 4; i++) {
AppendStoreVgpr(&code, i, i);
}
AppendEnd(&code);
return {"TBufferLoadFormatXy16_16IntegerComponents",
code,
{0x80017fffu, 0x7fff8000u, 0, 0},
{0x00007fffu, 0x00008001u, 0xffff8000u, 0x00007fffu},
{O::V_MOV_B32, O::TBUFFER_LOAD_FORMAT_XY, O::BUFFER_STORE_DWORD,
O::S_ENDPGM}};
}
TestCase TBufferStoreFormatXy1616IntegerComponents() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 20, 0);
AppendVMovU32(&code, 0, 0xaaaa);
AppendVMovU32(&code, 1, 0xbbbb);
code.push_back(EncodeMtbuf0(0x05, 11, 1));
code.push_back(EncodeMtbuf1(0x05, 0, 0, 20));
AppendVMovU32(&code, 20, 4);
AppendVMovLiteral(&code, 2, 0xffff8000u);
AppendVMovU32(&code, 3, 0x7fff);
code.push_back(EncodeMtbuf0(0x05, 12, 1));
code.push_back(EncodeMtbuf1(0x05, 2, 0, 20));
AppendEnd(&code);
return {"TBufferStoreFormatXy16_16IntegerComponents",
code,
{0x11223344u, 0x55667788u},
{0xbbbbaaaau, 0x7fff8000u},
{O::V_MOV_B32, O::TBUFFER_STORE_FORMAT_XY, O::S_ENDPGM}};
}
TestCase TBufferLoadFormatXyz16161616UintLoadsHalfwords() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 20, 0);
AppendTBufferLoadFormatOpcode(&code, 0x02, 0, 20,
Prospero::BufferFormat::k16_16_16_16UInt);
for (u32 i = 0; i < 3; i++) {
AppendStoreVgpr(&code, i, i);
}
AppendEnd(&code);
return {"TBufferLoadFormatXyz16_16_16_16UintLoadsHalfwords",
code,
{0x22221111u, 0x44443333u, 0},
{0x00001111u, 0x00002222u, 0x00003333u},
{O::V_MOV_B32, O::TBUFFER_LOAD_FORMAT_XYZ, O::BUFFER_STORE_DWORD,
O::S_ENDPGM}};
}
TestCase TBufferLoadFormatXy88UnormConvertsToFloat() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 20, 0);
code.push_back(EncodeMtbuf0(0x01, 14, 0));
code.push_back(EncodeMtbuf1(0x01, 0, 0, 20));
AppendStoreVgpr(&code, 0, 0);
AppendStoreVgpr(&code, 1, 1);
AppendEnd(&code);
return {"TBufferLoadFormatXy8_8UnormConvertsToFloat",
code,
{0x0000ff80u, 0},
{0x3f008081u, 0x3f800000u},
{O::V_MOV_B32, O::TBUFFER_LOAD_FORMAT_XY, O::BUFFER_STORE_DWORD,
O::S_ENDPGM}};
}
TestCase TBufferLoadFormatXy88SnormConvertsToFloat() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 20, 0);
AppendTBufferLoadFormatOpcode(&code, 0x01, 0, 20,
Prospero::BufferFormat::k8_8SNorm);
AppendStoreVgpr(&code, 0, 0);
AppendStoreVgpr(&code, 1, 1);
AppendEnd(&code);
return {"TBufferLoadFormatXy8_8SnormConvertsToFloat",
code,
{0x00007f80u, 0},
{0xbf800000u, 0x3f800000u},
{O::V_MOV_B32, O::TBUFFER_LOAD_FORMAT_XY, O::BUFFER_STORE_DWORD,
O::S_ENDPGM}};
}
TestCase TBufferLoadFormatXy1616UnormConvertsToFloat() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 20, 0);
code.push_back(EncodeMtbuf0(0x01, 7, 1));
code.push_back(EncodeMtbuf1(0x01, 0, 0, 20));
AppendStoreVgpr(&code, 0, 0);
AppendStoreVgpr(&code, 1, 1);
AppendEnd(&code);
return {"TBufferLoadFormatXy16_16UnormConvertsToFloat",
code,
{0xffff8000u, 0},
{0x3f000080u, 0x3f800000u},
{O::V_MOV_B32, O::TBUFFER_LOAD_FORMAT_XY, O::BUFFER_STORE_DWORD,
O::S_ENDPGM}};
}
TestCase TBufferLoadFormatXy8888UnormConvertsFirstTwoComponents() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 20, 0);
code.push_back(EncodeMtbuf0(0x01, 8, 3));
code.push_back(EncodeMtbuf1(0x01, 0, 0, 20));
AppendStoreVgpr(&code, 0, 0);
AppendStoreVgpr(&code, 1, 1);
AppendEnd(&code);
return {"TBufferLoadFormatXy8_8_8_8UnormConvertsFirstTwoComponents",
code,
{0x44332211u, 0xdeadbeefu},
{0x3d888889u, 0x3e088889u},
{O::V_MOV_B32, O::TBUFFER_LOAD_FORMAT_XY, O::BUFFER_STORE_DWORD,
O::S_ENDPGM}};
}
TestCase TBufferLoadFormatXyzw8888UintExtractsBytes() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 20, 0);
AppendTBufferLoadFormatOpcode(&code, 0x03, 0, 20,
Prospero::BufferFormat::k8_8_8_8UInt);
for (u32 i = 0; i < 4; i++) {
AppendStoreVgpr(&code, i, i);
}
AppendEnd(&code);
return {"TBufferLoadFormatXyzw8_8_8_8UintExtractsBytes",
code,
{0x44332211u, 0xaaaaaaaau, 0xbbbbbbbbu, 0xccccccccu},
{0x00000011u, 0x00000022u, 0x00000033u, 0x00000044u},
{O::V_MOV_B32, O::TBUFFER_LOAD_FORMAT_XYZW, O::BUFFER_STORE_DWORD,
O::S_ENDPGM}};
}
TestCase TBufferLoadFormatXyzw1010102SnormConvertsToFloat() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 20, 0);
AppendTBufferLoadFormatOpcode(&code, 0x03, 0, 20,
Prospero::BufferFormat::k10_10_10_2SNorm);
for (u32 i = 0; i < 4; i++) {
AppendStoreVgpr(&code, i, i);
}
AppendEnd(&code);
return {"TBufferLoadFormatXyzw10_10_10_2SnormConvertsToFloat",
code,
{0x800801ffu, 0, 0, 0},
{0x3f800000u, 0xbf800000u, 0, 0xbf800000u},
{O::V_MOV_B32, O::TBUFFER_LOAD_FORMAT_XYZW, O::BUFFER_STORE_DWORD,
O::S_ENDPGM}};
}
TestCase TBufferLoadFormatXyz111110FloatUnpacks() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 20, 0);
AppendTBufferLoadFormatOpcode(&code, 0x02, 0, 20,
Prospero::BufferFormat::k11_11_10Float);
for (u32 i = 0; i < 3; i++) {
AppendStoreVgpr(&code, i, i);
}
AppendEnd(&code);
return {"TBufferLoadFormatXyz11_11_10FloatUnpacks",
code,
{0x781e03c0u, 0, 0},
{0x3f800000u, 0x3f800000u, 0x3f800000u},
{O::V_MOV_B32, O::TBUFFER_LOAD_FORMAT_XYZ, O::BUFFER_STORE_DWORD,
O::S_ENDPGM}};
}
TestCase TBufferLoadFormatXyzw3232FloatZerosMissingComponents() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 20, 0);
code.push_back(EncodeMtbuf0(0x03, 0, 4));
code.push_back(EncodeMtbuf1(0x03, 0, 0, 20));
for (u32 i = 0; i < 4; i++) {
AppendStoreVgpr(&code, i, i);
}
AppendEnd(&code);
return {"TBufferLoadFormatXyzw32_32FloatZerosMissingComponents",
code,
{0x11111111u, 0x22222222u, 0x33333333u, 0x44444444u},
{0x11111111u, 0x22222222u, 0, 0},
{O::V_MOV_B32, O::TBUFFER_LOAD_FORMAT_XYZW, O::BUFFER_STORE_DWORD,
O::S_ENDPGM}};
}
TestCase TBufferStoreFormatXyzw3232FloatWritesOnlyPresentComponents() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 20, 0);
AppendVMovLiteral(&code, 0, 0x3f800000u);
AppendVMovLiteral(&code, 1, 0x40000000u);
AppendVMovLiteral(&code, 2, 0x40400000u);
AppendVMovLiteral(&code, 3, 0x40800000u);
code.push_back(EncodeMtbuf0(0x07, 0, 4));
code.push_back(EncodeMtbuf1(0x07, 0, 0, 20));
AppendEnd(&code);
return {"TBufferStoreFormatXyzw32_32FloatWritesOnlyPresentComponents",
code,
{0xaaaaaaaau, 0xbbbbbbbbu, 0xccccccccu, 0xddddddddu},
{0x3f800000u, 0x40000000u, 0xccccccccu, 0xddddddddu},
{O::V_MOV_B32, O::TBUFFER_STORE_FORMAT_XYZW, O::S_ENDPGM}};
}
TestCase TBufferStoreVariants() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 20, 0);
AppendVMovLiteral(&code, 0, 0x01020304u);
AppendTBufferStoreOpcode(&code, 0x04, 0, 20);
AppendVMovU32(&code, 20, 4);
AppendVMovLiteral(&code, 1, 0x11121314u);
AppendVMovLiteral(&code, 2, 0x21222324u);
AppendTBufferStoreOpcode(&code, 0x05, 1, 20);
AppendVMovU32(&code, 20, 12);
AppendVMovLiteral(&code, 3, 0x31323334u);
AppendVMovLiteral(&code, 4, 0x41424344u);
AppendVMovLiteral(&code, 5, 0x51525354u);
AppendTBufferStoreOpcode(&code, 0x06, 3, 20);
AppendVMovU32(&code, 20, 24);
AppendVMovLiteral(&code, 6, 0x61626364u);
AppendVMovLiteral(&code, 7, 0x71727374u);
AppendVMovLiteral(&code, 8, 0x81828384u);
AppendVMovLiteral(&code, 9, 0x91929394u);
AppendTBufferStoreOpcode(&code, 0x07, 6, 20);
AppendEnd(&code);
return {"TBufferStoreVariants",
code,
std::vector<u32>(10, 0),
{0x01020304u, 0x11121314u, 0x21222324u, 0x31323334u, 0x41424344u,
0x51525354u, 0x61626364u, 0x71727374u, 0x81828384u, 0x91929394u},
{O::V_MOV_B32, O::TBUFFER_STORE_FORMAT_X, O::TBUFFER_STORE_FORMAT_XY,
O::TBUFFER_STORE_FORMAT_XYZ, O::TBUFFER_STORE_FORMAT_XYZW,
O::S_ENDPGM}};
}
TestCase FlatLoadDlcCaptured(u32 wave_size) {
using O = ShaderOpcode;
TestCase test;
test.name = wave_size == 64 ? "FlatLoadDlcCapturedWave64" : "FlatLoadDlcCapturedWave32";
auto &code = test.code;
code.push_back(EncodeSop1(0x04, 126, InlineU32(1)));
AppendVMovU32(&code, 4, 0);
AppendVMovU32(&code, 5, 31);
// PPSA04288 CS 05b58beb842bacbf:0x638, FLAT_LOAD_DWORDX2 v[0:1], v[4:5] DLC.
code.insert(code.end(), {0xdc341000u, 0x007d0004u});
AppendStoreVgpr(&code, 0, 0);
AppendStoreVgpr(&code, 1, 1);
// The descriptor store aliases the physical address read by the next load.
AppendStoreVgpr(&code, 1, 8);
code.push_back(0xbbfd0000u); // S_WAITCNT_VSCNT null, 0.
code.insert(code.end(), {0xdc341000u, 0x007d0004u});
AppendStoreVgpr(&code, 0, 2);
AppendStoreVgpr(&code, 1, 3);
AppendEnd(&code);
test.initial = {0, 0, 0, 0, 0, 0, 0, 0, 0x11223344u, 0x55667788u};
test.expected = {0x11223344u, 0x55667788u, 0x55667788u, 0x55667788u,
0, 0, 0, 0, 0x55667788u, 0x55667788u};
test.bda_mappings = {{0x1f00000000ull, 32}};
test.opcodes = {O::S_MOV_B64, O::V_MOV_B32, O::FLAT_LOAD_DWORDX2,
O::BUFFER_STORE_DWORD, O::S_WAITCNT_VSCNT, O::S_ENDPGM};
test.required_spirv = {"OpDecorate %buffers Coherent", " 0 Coherent", "Volatile|Aligned 4"};
test.decoded_counts = {{"dlc=1", 2}};
test.compute_info.threads_num[0] = wave_size;
test.compute_info.threads_num[1] = 1;
test.compute_info.threads_num[2] = 1;
test.compute_info.wave_size = wave_size;
test.has_compute_info = true;
return test;
}
TestCase FlatStoreSlcCaptured() {
TestCase test;
test.name = "FlatStoreSlcCaptured";
auto& code = test.code;
code.push_back(EncodeSop1(0x0b, 26, InlineU32(1))); // s26 = 0x80000000.
AppendVMovLiteral(&code, 12, 0x000c0008u); // Packed halfwords 8 and 12 select byte offset 8.
code.insert(code.end(), {0x260c18f9u, 0x0504060cu, // Unsigned minimum of the two halfwords.
0xd5590007u, 0x0431fe1au, 0x70000000u});
AppendVMovLiteral(&code, 3, 0x12345678u);
// Exercise an SLC store through v[6:7].
code.insert(code.end(), {0xdc720000u, 0x007d0306u});
AppendVMovLiteral(&code, 3, 0xdeadbeefu);
code.push_back(EncodeSop1(0x04, 126, InlineU32(0)));
code.insert(code.end(), {0xdc720000u, 0x007d0306u});
code.push_back(EncodeSop1(0x04, 126, 193)); // Restore EXEC before readback.
code.insert(code.end(), {0xd5590007u, 0x0431fe1au, 0x70000000u,
EncodeFlat0(0x0c, 0), EncodeFlat1(4, 0x7d, 0, 6)});
AppendStoreVgprAtLaneDwordOffset(&code, 4, 0, 0);
AppendEnd(&code);
test.initial.assign(65, 0xa5a5a5a5u);
test.expected.assign(64, 0x12345678u);
test.expected.push_back(0xa5a5a5a5u);
test.opcodes = {ShaderOpcode::S_BREV_B32, ShaderOpcode::V_MIN_U32,
ShaderOpcode::V_MED3_U32, ShaderOpcode::V_MOV_B32, ShaderOpcode::FLAT_STORE_DWORD,
ShaderOpcode::S_MOV_B64, ShaderOpcode::FLAT_LOAD_DWORD,
ShaderOpcode::BUFFER_STORE_DWORD, ShaderOpcode::S_ENDPGM};
test.compute_info.scratch_size_dwords = 4;
test.compute_info.threads_num[0] = 64;
test.compute_info.threads_num[1] = test.compute_info.threads_num[2] = 1;
test.compute_info.thread_ids_num = 1;
test.has_compute_info = true;
test.decoded_counts = {{"slc=1", 2}};
test.forbidden_spirv = {"get_bda_pointer"};
return test;
}
TestCase FlatLoadVariants() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 20, 0);
AppendVMovU32(&code, 21, 0);
code.push_back(EncodeFlat0(0x08, 0, 0));
code.push_back(EncodeFlat1(0, 0x7d, 0, 20));
code.push_back(EncodeFlat0(0x09, 0, 2));
code.push_back(EncodeFlat1(1, 0x7d, 0, 20));
code.push_back(EncodeFlat0(0x0a, 0, 0));
code.push_back(EncodeFlat1(2, 0x7d, 0, 20));
code.push_back(EncodeFlat0(0x0b, 0, 2));
code.push_back(EncodeFlat1(3, 0x7d, 0, 20));
code.push_back(EncodeFlat0(0x0c, 0, 8));
code.push_back(EncodeFlat1(13, 0x7d, 0, 20));
code.push_back(EncodeFlat0(0x0d, 0, 8));
code.push_back(EncodeFlat1(4, 0x7d, 0, 20));
code.push_back(EncodeFlat0(0x0f, 0, 8));
code.push_back(EncodeFlat1(6, 0x7d, 0, 20));
code.push_back(EncodeFlat0(0x0e, 0, 8));
code.push_back(EncodeFlat1(9, 0x7d, 0, 20));
for (u32 i = 0; i < 14u; i++) {
AppendStoreVgpr(&code, i, i);
}
AppendEnd(&code);
const std::vector<u32> guest_memory{0x80ff7f01u, 0x00008001u, 0x11223344u,
0x55667788u, 0x99aabbccu, 0xddeeff00u};
constexpr size_t byte_offset = 8;
std::vector<u32> host_memory(
(byte_offset + guest_memory.size() * sizeof(u32) + sizeof(u32) - 1u) /
sizeof(u32));
std::memcpy(reinterpret_cast<uint8_t *>(host_memory.data()) + byte_offset,
guest_memory.data(), guest_memory.size() * sizeof(u32));
TestCase test{"FlatLoadVariants",
code,
std::move(host_memory),
{0x01u, 0xffffffffu, 0x7f01u, 0xffff80ffu, 0x11223344u,
0x55667788u, 0x11223344u, 0x55667788u, 0x99aabbccu,
0x11223344u, 0x55667788u, 0x99aabbccu, 0xddeeff00u,
0x11223344u},
{O::V_MOV_B32, O::FLAT_LOAD_UBYTE, O::FLAT_LOAD_SBYTE,
O::FLAT_LOAD_USHORT, O::FLAT_LOAD_SSHORT, O::FLAT_LOAD_DWORD,
O::FLAT_LOAD_DWORDX2, O::FLAT_LOAD_DWORDX3,
O::FLAT_LOAD_DWORDX4, O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
test.bda_mappings = {{0, static_cast<u32>(byte_offset)}};
return test;
}
TestCase FlatSubdwordLoadsApplyByteOffset() {
using O = ShaderOpcode;
constexpr size_t byte_offset = 3;
std::vector<u32> code;
AppendVMovU32(&code, 20, static_cast<u32>(byte_offset));
AppendVMovU32(&code, 21, 0);
code.push_back(EncodeFlat0(0x08, 0, 0));
code.push_back(EncodeFlat1(0, 0x7d, 0, 20));
code.push_back(EncodeFlat0(0x0a, 0, 1));
code.push_back(EncodeFlat1(1, 0x7d, 0, 20));
AppendStoreVgpr(&code, 0, 0);
AppendStoreVgpr(&code, 1, 1);
AppendEnd(&code);
constexpr u32 guest_word = 0x80ff7f01u;
std::vector<u32> host_memory(
(byte_offset + sizeof(guest_word) + sizeof(u32) - 1u) / sizeof(u32));
std::memcpy(reinterpret_cast<uint8_t *>(host_memory.data()) + byte_offset,
&guest_word, sizeof(guest_word));
TestCase test;
test.name = "FlatSubdwordLoadsApplyByteOffset";
test.code = std::move(code);
test.initial = std::move(host_memory);
test.expected = {0x01u, 0x0000ff7fu};
test.bda_mappings = {{0, 0}};
test.opcodes = {O::V_MOV_B32, O::FLAT_LOAD_UBYTE, O::FLAT_LOAD_USHORT,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
return test;
}
TestCase GlobalLoadShortD16Captured(u32 wave_size) {
using O = ShaderOpcode;
constexpr uint64_t GuestBase = 0x0000000110000000ull;
constexpr u32 Marker = 0xa5a50000u;
constexpr uint64_t ScalarBase = GuestBase - Marker - 0x70u;
constexpr uint64_t ExecMask = 0xaaaaaaaa55555555ull;
constexpr u32 Values[] = {0u, 0x7fffu, 0x8000u, 0xffffu};
TestCase test;
test.name = wave_size == 32u ? "GlobalLoadShortD16CapturedWave32"
: "GlobalLoadShortD16CapturedWave64";
test.initial.resize(2u * wave_size + wave_size / 2u);
test.expected.resize(2u * wave_size);
for (u32 lane = 0; lane < wave_size; ++lane) {
const u32 value = Values[lane % std::size(Values)];
test.initial[2u * wave_size + lane / 2u] |= value << (16u * (lane % 2u));
test.expected[lane] = Marker | value;
test.expected[wave_size + lane] = Marker |
((ExecMask & (uint64_t{1} << lane)) != 0u ? value : lane * 2u);
}
auto &code = test.code;
AppendSMovLiteral(&code, 2, static_cast<u32>(ScalarBase));
AppendSMovLiteral(&code, 3, static_cast<u32>(ScalarBase >> 32u));
code.push_back(EncodeVop2(0x1a, 6, InlineU32(1), 0));
AppendVMovLiteral(&code, 5, Marker);
code.push_back(EncodeVop2(0x25, 6, Vgpr(5), 6));
for (bool masked : {false, true}) {
code.push_back(EncodeVop1(0x01, 5, Vgpr(6)));
if (masked) {
code.push_back(EncodeSop1(0x04, 8, 126));
AppendSMovLiteral(&code, 126, static_cast<u32>(ExecMask));
AppendSMovLiteral(&code, 127, static_cast<u32>(ExecMask >> 32u));
}
// Captured GLOBAL_LOAD_SHORT_D16: v5 supplies the byte offset and destination.
code.insert(code.end(), {0xdc908070u, 0x05020005u});
if (masked) code.push_back(EncodeSop1(0x04, 126, 8));
AppendStoreVgprAtLaneDwordOffset(&code, 5, 0, masked ? wave_size : 0u);
}
AppendEnd(&code);
test.bda_mappings = {{GuestBase, 8u * wave_size}};
test.compute_info.threads_num[0] = wave_size;
test.compute_info.threads_num[1] = 1;
test.compute_info.threads_num[2] = 1;
test.compute_info.thread_ids_num = 1;
test.compute_info.wave_size = wave_size;
test.has_compute_info = true;
test.opcodes = {O::S_MOV_B32, O::S_MOV_B64, O::V_MOV_B32, O::V_LSHLREV_B32,
O::V_ADD_NC_U32, O::FLAT_LOAD_SHORT_D16,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.decoded_counts = {{"FLAT_LOAD_SHORT_D16", 2}};
return test;
}
TestCase FlatLoadShortD16AddressSegments() {
using O = ShaderOpcode;
constexpr uint64_t GuestBase = 0x0000000110000000ull;
std::vector<u32> code;
AppendVMovLiteral(&code, 20, static_cast<u32>(GuestBase) + 2u);
AppendVMovLiteral(&code, 21, static_cast<u32>(GuestBase >> 32u));
for (u32 segment : {0u, 2u}) {
AppendVMovLiteral(&code, 3, 0x12345678u);
code.push_back(EncodeFlat0(0x24, segment, segment == 0u ? 0u : 0xffeu));
code.push_back(EncodeFlat1(3, 0x7d, 0, 20));
AppendStoreVgpr(&code, 3, segment == 0u ? 0u : 1u);
}
AppendVMovU32(&code, 20, 0u);
AppendVMovLiteral(&code, 3, 0xbeef5678u);
code.push_back(EncodeFlat0(0x1c, 1, 0));
code.push_back(EncodeFlat1(0, 0x7d, 3, 20));
AppendVMovLiteral(&code, 3, 0x98761234u);
code.push_back(EncodeFlat0(0x24, 1, 2));
code.push_back(EncodeFlat1(3, 0x7d, 0, 20));
AppendStoreVgpr(&code, 3, 2);
AppendEnd(&code);
TestCase test{"FlatLoadShortD16AddressSegments", code,
{0, 0, 0, 0, 0x80017fffu},
{0x12348001u, 0x12347fffu, 0x9876beefu},
{O::V_MOV_B32, O::FLAT_LOAD_SHORT_D16, O::FLAT_STORE_DWORD,
O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
test.bda_mappings = {{GuestBase, 16}};
test.compute_info.scratch_size_dwords = 1;
return test;
}
TestCase BranchVccnzUsesCarryProducedWaveMask() {
using O = ShaderOpcode;
std::vector<u32> code = {
EncodeVop1(0x01, 1, 0), EncodeVop2(0x1a, 1, InlineU32(2), 1),
EncodeVop2(0x25, 1, Vgpr(0), 1), EncodeVop2(0x1a, 3, InlineU32(2), 1),
EncodeVopc(0xc0, Vgpr(0), 0),
};
AppendVMovLiteral(&code, 2, 0xffffffffu);
code.push_back(EncodeVop2(0x28, 5, Vgpr(2), 1));
code.push_back(EncodeSopp(0x07, 2));
code.push_back(EncodeVop1(0x01, 2, InlineU32(11)));
code.push_back(EncodeSopp(0x02, 1));
AppendVMovU32(&code, 2, 42);
AppendBufferStoreDword(&code, 2, 3);
AppendEnd(&code);
TestCase test;
test.name = "BranchVccnzUsesCarryProducedWaveMask";
test.code = code;
test.expected = std::vector<u32>(8, 42);
test.opcodes = {O::V_MOV_B32, O::V_LSHLREV_B32, O::V_ADD_NC_U32,
O::V_CMP_F_U32, O::V_ADDC_U32, O::S_CBRANCH_VCCNZ,
O::S_BRANCH, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.compute_info.threads_num[0] = 4;
test.compute_info.threads_num[1] = 1;
test.compute_info.threads_num[2] = 1;
test.compute_info.group_id[0] = true;
test.compute_info.thread_ids_num = 1;
test.compute_info.workgroup_register = 0;
test.has_compute_info = true;
test.dispatch_x = 2;
return test;
}
TestCase ScalarLoadAlignsComponentsAndMasksAddress() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendSMovLiteral(&code, 0, 1u);
AppendSMovLiteral(&code, 2, 3u);
AppendSMovLiteral(&code, 3, 0xffff0000u);
code.push_back(EncodeSmem0(0x00, 1, 1));
code.push_back(EncodeSmem1(3, 0));
AppendStoreSgpr(&code, 1, 0);
AppendEnd(&code);
return {"ScalarLoadAlignsComponentsAndMasksAddress",
code,
{0x11111111u, 0x22222222u},
{0x11111111u, 0x22222222u},
{O::S_MOV_B32, O::S_LOAD_DWORD, O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}
TestCase ScalarLoadAlignsDynamicBase() {
using O = ShaderOpcode;
constexpr uint64_t GuestBase = 0x10000;
std::vector<u32> code;
AppendSMovLiteral(&code, 8, GuestBase);
AppendSMovLiteral(&code, 9, 0);
code.push_back(EncodeSop2(0x00, 8, 8, 4)); // group ID supplies each base low-bit pattern
AppendSMovLiteral(&code, 10, 7);
code.push_back(EncodeSmem0(0x01, 12, 4));
code.push_back(EncodeSmem1(7, 10)); // all three address components align independently
code.push_back(EncodeVop1(0x01, 0, 12));
code.push_back(EncodeVop1(0x01, 1, 13));
code.push_back(EncodeVop1(0x01, 2, 4));
code.push_back(EncodeVop2(0x1a, 3, InlineU32(3), 2));
AppendBufferStoreDword(&code, 0, 3);
code.push_back(EncodeVop2(0x25, 3, InlineU32(4), 3));
AppendBufferStoreDword(&code, 1, 3);
AppendEnd(&code);
TestCase test;
test.name = "ScalarLoadAlignsDynamicBase";
test.code = std::move(code);
test.initial.resize(20);
test.initial[18] = 0x12345678;
test.initial[19] = 0x9abcdef0;
test.expected = test.initial;
for (u32 group = 0; group < 4; ++group) {
test.expected[group * 2] = test.initial[18];
test.expected[group * 2 + 1] = test.initial[19];
}
test.bda_mappings = {{GuestBase, 64}};
test.required_spirv = {"get_bda_pointer"};
test.forbidden_spirv = {"flattened_srt"};
test.opcodes = {O::S_MOV_B32, O::S_ADD_U32, O::S_LOAD_DWORDX2, O::V_MOV_B32,
O::V_LSHLREV_B32, O::V_ADD_NC_U32, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.compute_info.wave_size = 32;
test.compute_info.threads_num[0] = 1;
test.compute_info.threads_num[1] = 1;
test.compute_info.threads_num[2] = 1;
test.compute_info.group_id[0] = true;
test.compute_info.workgroup_register = 4;
test.has_compute_info = true;
test.dispatch_x = 4;
return test;
}
TestCase ScalarBufferFromLoopReadlane(u32 wave_size) {
using O = ShaderOpcode;
constexpr u32 GuestBase = 0x10000;
constexpr u32 DescriptorCount = 5;
constexpr u32 PayloadBase = GuestBase + DescriptorCount * 32;
std::vector<u32> code;
code.push_back(EncodeVop2(0x1a, 1, InlineU32(2), 0));
AppendBufferLoadDword(&code, 2, 1);
AppendSMovLiteral(&code, 8, GuestBase + 3);
AppendSMovLiteral(&code, 9, 0);
AppendSMovLiteral(&code, 28, 0);
const auto loop = static_cast<u32>(code.size());
code.push_back(EncodeSop1(0x04, 126, InlineU32(0))); // SMEM ignores EXEC.
code.push_back(EncodeSmem0(0x03, 16, 4)); // descriptor and runtime soffset
code.push_back(EncodeSmem1(3, 125)); // null keeps this a flattening candidate.
code.push_back(EncodeSmem0(0x0a, 32, 8));
code.push_back(EncodeSmem1(7, 20));
code.push_back(EncodeSop1(0x04, 126, InlineU32(1)));
code.push_back(EncodeVop1(0x01, 3, 28));
code.push_back(EncodeVop2(0x1a, 31, InlineU32(4), 3));
for (u32 component = 0; component < 4; ++component) {
code.push_back(EncodeVop1(0x01, 4, 32 + component));
AppendBufferStoreDword(&code, 4, 31);
code.push_back(EncodeVop2(0x25, 31, InlineU32(4), 31));
}
code.push_back(EncodeSop1(0x04, 126, 193));
code.push_back(EncodeSop2(0x00, 29, 28, InlineU32(wave_size == 64 ? 32 : 0)));
AppendVop3(&code, 0x360, 8, Vgpr(2), 29);
code.push_back(EncodeSop2(0x00, 28, 28, InlineU32(1)));
code.push_back(EncodeSopc(0x0a, 28, InlineU32(DescriptorCount)));
code.push_back(EncodeSopp(0x05, loop - static_cast<u32>(code.size()) - 1u));
AppendEnd(&code);
TestCase test;
test.name = wave_size == 32 ? "ScalarBufferFromLoopReadlane32"
: "ScalarBufferFromLoopReadlane64";
test.code = std::move(code);
test.initial.resize(128 + DescriptorCount * 8 + 8);
constexpr u32 records[] = {20, 5, 3, 0x40000005u, 20};
for (u32 i = 0; i < DescriptorCount; ++i) {
test.initial[(wave_size == 64 ? 32 : 0) + i] =
GuestBase + ((i + 1) % DescriptorCount) * 32 + 3;
const auto row = 128 + i * 8;
test.initial[row] = PayloadBase + 3;
test.initial[row + 1] = (i == 1 || i == 3 ? 4u << 16u : 0u) |
(i == 1 ? 1u << 31u : 0u);
test.initial[row + 2] = records[i];
// Scalar addressing ignores FORMAT, swizzle, ADD_TID, INDEX_STRIDE and OOB_SELECT.
test.initial[row + 3] = i == 1 ? (3u << 21u) | (1u << 23u) | (3u << 28u) : 0u;
test.initial[row + 4] = i == 4 ? 0xffffffffu : 11u;
}
for (u32 i = 0; i < 8; ++i)
test.initial[128 + DescriptorCount * 8 + i] = 0x11111111u * (i + 1u);
test.expected = {0x44444444u, 0x55555555u, 0, 0,
0x44444444u, 0x55555555u, 0, 0,
0, 0, 0, 0,
0x44444444u, 0x55555555u, 0x66666666u, 0x77777777u,
0, 0, 0, 0};
test.bda_mappings = {{GuestBase, 512}};
test.required_spirv = {"get_bda_pointer", "OpLoopMerge", "OpGroupNonUniformShuffle"};
test.forbidden_spirv = {"flattened_srt"};
test.opcodes = {O::V_LSHLREV_B32, O::BUFFER_LOAD_DWORD, O::S_MOV_B32,
O::S_MOV_B64, O::S_LOAD_DWORDX8, O::S_BUFFER_LOAD_DWORDX4,
O::V_MOV_B32, O::BUFFER_STORE_DWORD, O::V_ADD_NC_U32,
O::S_ADD_U32, O::V_READLANE_B32, O::S_CMP_LT_U32,
O::S_CBRANCH_SCC1, O::S_ENDPGM};
test.compute_info.wave_size = wave_size;
test.compute_info.threads_num[0] = wave_size;
test.compute_info.threads_num[1] = test.compute_info.threads_num[2] = 1;
test.compute_info.thread_ids_num = 1;
test.has_compute_info = true;
return test;
}
TestCase FlatVirtualAddressRebasesGuestAllocation() {
using O = ShaderOpcode;
constexpr uint64_t GuestBase = 0x0000000110000000ull;
constexpr uint64_t ExtendedBase = Libs::LibKernel::Memory::kExtendedMemoryBase + GuestBase;
static_assert(BufferCache::PageIndex(ExtendedBase) ==
((Libs::Graphics::LOWER_ADDRESS_SIZE + GuestBase) >> BufferCache::CACHING_PAGEBITS));
static_assert(BufferCache::GuestAddress(BufferCache::PageIndex(ExtendedBase) <<
BufferCache::CACHING_PAGEBITS) == ExtendedBase);
std::vector<u32> code;
AppendVMovLiteral(&code, 20, static_cast<u32>(GuestBase + 4u));
AppendVMovLiteral(&code, 21, static_cast<u32>(GuestBase >> 32u));
code.push_back(EncodeFlat0(0x0c, 0, 0));
code.push_back(EncodeFlat1(0, 0x7d, 0, 20));
code.push_back(EncodeFlat0(0x0c, 2, 0));
code.push_back(EncodeFlat1(1, 0x7d, 0, 20));
AppendVMovLiteral(&code, 21, static_cast<u32>((GuestBase >> 32u) + 1u));
code.push_back(EncodeFlat0(0x0c, 0, 0));
code.push_back(EncodeFlat1(2, 0x7d, 0, 20));
AppendVMovLiteral(&code, 20, static_cast<u32>(ExtendedBase + 4u));
AppendVMovLiteral(&code, 21, static_cast<u32>(ExtendedBase >> 32u));
code.push_back(EncodeFlat0(0x0c, 0, 0));
code.push_back(EncodeFlat1(3, 0x7d, 0, 20));
AppendStoreVgpr(&code, 0, 0);
AppendStoreVgpr(&code, 1, 1);
AppendStoreVgpr(&code, 2, 2);
AppendStoreVgpr(&code, 3, 3);
AppendEnd(&code);
TestCase test;
test.name = "FlatVirtualAddressRebasesGuestAllocation";
test.code = std::move(code);
test.initial = {0xfeedfaceu, 0xcafebabeu, 0, 0x12345678u};
test.expected = {0x12345678u, 0x12345678u, 0, 0xcafebabeu};
test.bda_mappings = {{GuestBase, 8}, {ExtendedBase, 0}};
test.opcodes = {O::V_MOV_B32, O::FLAT_LOAD_DWORD, O::BUFFER_STORE_DWORD,
O::S_ENDPGM};
return test;
}
TestCase GlobalSignedImmediateRebasesBeforeSaddr() {
using O = ShaderOpcode;
constexpr uint64_t GuestBase = 0x0000000110000000ull;
std::vector<u32> code;
AppendSMovLiteral(&code, 0, static_cast<u32>(GuestBase));
AppendSMovLiteral(&code, 1, static_cast<u32>(GuestBase >> 32u));
AppendVMovU32(&code, 20, 8);
code.push_back(EncodeFlat0(0x0c, 2, 0xffcu));
code.push_back(EncodeFlat1(0, 0, 0, 20));
AppendStoreVgpr(&code, 0, 0);
AppendEnd(&code);
TestCase test;
test.name = "GlobalSignedImmediateRebasesBeforeSaddr";
test.code = std::move(code);
test.initial = {0xfeedfaceu, 0xcafebabeu, 0x11111111u, 0x12345678u};
test.expected = {0x12345678u};
test.bda_mappings = {{GuestBase, 8}};
test.opcodes = {O::S_MOV_B32, O::V_MOV_B32, O::FLAT_LOAD_DWORD,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
return test;
}
TestCase FlatSegmentIgnoresSaddrAndMasksOffsetMsb() {
using O = ShaderOpcode;
std::vector<u32> code;
code.push_back(EncodeSMovB32(0, InlineU32(4)));
AppendVMovU32(&code, 20, 0);
AppendVMovU32(&code, 21, 0);
code.push_back(EncodeFlat0(0x0c, 0, 4));
code.push_back(EncodeFlat1(0, 0, 0, 20));
AppendStoreVgpr(&code, 0, 0);
AppendVMovU32(&code, 20, 0);
code.push_back(EncodeFlat0(0x0c, 0, 0x804));
code.push_back(EncodeFlat1(1, 0x7d, 0, 20));
AppendStoreVgpr(&code, 1, 1);
AppendEnd(&code);
std::vector<u32> initial(514, 0);
initial[1] = 0x11111111u;
initial[2] = 0x22222222u;
initial[513] = 0x33333333u;
TestCase test;
test.name = "FlatSegmentIgnoresSaddrAndMasksOffsetMsb";
test.code = code;
test.initial = initial;
test.expected = {0x11111111u, 0x11111111u};
test.opcodes = {O::S_MOV_B32, O::V_MOV_B32, O::FLAT_LOAD_DWORD,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.bda_mappings = {{0, 0}};
return test;
}
TestCase ScratchIsPrivatePerInvocation() {
using O = ShaderOpcode;
std::vector<u32> code;
code.push_back(EncodeSMovB32(20, InlineU32(0)));
AppendVMovU32(&code, 21, 0);
code.push_back(EncodeFlat0(0x1c, 1, 4));
code.push_back(EncodeFlat1(0, 20, 0, 21));
code.push_back(EncodeFlat0(0x0c, 1, 4));
code.push_back(EncodeFlat1(5, 20, 0, 21));
AppendStoreVgprAtLaneDwordOffset(&code, 5, 0, 0);
AppendEnd(&code);
TestCase test;
test.name = "ScratchIsPrivatePerInvocation";
test.code = std::move(code);
for (u32 lane = 0; lane < 64; lane++) {
test.expected.push_back(lane);
}
test.opcodes = {O::S_MOV_B32, O::V_MOV_B32,
O::FLAT_STORE_DWORD, O::FLAT_LOAD_DWORD,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.compute_info.threads_num[0] = 64;
test.compute_info.threads_num[1] = 1;
test.compute_info.threads_num[2] = 1;
test.compute_info.thread_ids_num = 1;
test.compute_info.workgroup_register = 0;
test.compute_info.scratch_size_dwords = 2;
test.has_compute_info = true;
return test;
}
#if KYTY_PLATFORM != KYTY_PLATFORM_WINDOWS
TestCase FlatStoreVariants() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 21, 0);
AppendVMovU32(&code, 20, 1);
AppendVMovU32(&code, 0, 0xaa);
code.push_back(EncodeFlat0(0x18, 0, 0));
code.push_back(EncodeFlat1(0, 0x7d, 0, 20));
AppendVMovU32(&code, 20, 2);
AppendVMovLiteral(&code, 1, 0x0000bbccu);
code.push_back(EncodeFlat0(0x1a, 0, 0));
code.push_back(EncodeFlat1(0, 0x7d, 1, 20));
AppendVMovU32(&code, 20, 4);
AppendVMovLiteral(&code, 2, 0x11111111u);
AppendVMovLiteral(&code, 3, 0x22222222u);
code.push_back(EncodeFlat0(0x1d, 0, 0));
code.push_back(EncodeFlat1(0, 0x7d, 2, 20));
AppendVMovU32(&code, 20, 12);
AppendVMovLiteral(&code, 4, 0x33333333u);
AppendVMovLiteral(&code, 5, 0x44444444u);
AppendVMovLiteral(&code, 6, 0x55555555u);
code.push_back(EncodeFlat0(0x1f, 0, 0));
code.push_back(EncodeFlat1(0, 0x7d, 4, 20));
AppendVMovU32(&code, 20, 24);
AppendVMovLiteral(&code, 7, 0x66666666u);
AppendVMovLiteral(&code, 8, 0x77777777u);
AppendVMovLiteral(&code, 9, 0x88888888u);
AppendVMovLiteral(&code, 10, 0x99999999u);
code.push_back(EncodeFlat0(0x1e, 0, 0));
code.push_back(EncodeFlat1(0, 0x7d, 7, 20));
AppendVMovU32(&code, 20, 0);
code.push_back(EncodeFlat0(0x1c, 0, 40));
code.push_back(EncodeFlat1(0, 0x7d, 2, 20));
AppendEnd(&code);
TestCase test{"FlatStoreVariants",
code,
std::vector<u32>(12, 0),
{0xbbccaa00u, 0x11111111u, 0x22222222u, 0x33333333u,
0x44444444u, 0x55555555u, 0x66666666u, 0x77777777u,
0x88888888u, 0x99999999u, 0x11111111u, 0},
{O::V_MOV_B32, O::FLAT_STORE_BYTE, O::FLAT_STORE_SHORT,
O::FLAT_STORE_DWORD, O::FLAT_STORE_DWORDX2,
O::FLAT_STORE_DWORDX3, O::FLAT_STORE_DWORDX4, O::S_ENDPGM}};
return test;
}
#endif
TestCase DsReadWriteVariants() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 1, 0);
AppendVMovLiteral(&code, 2, 0x11223344u);
code.push_back(EncodeDs0(0x0d, 0));
code.push_back(EncodeDs1(0, 2, 1));
code.push_back(EncodeDs0(0x36, 0));
code.push_back(EncodeDs1(3, 0, 1));
AppendVMovU32(&code, 4, 0xaa);
code.push_back(EncodeDs0(0x1e, 4));
code.push_back(EncodeDs1(0, 4, 1));
code.push_back(EncodeDs0(0x3a, 4));
code.push_back(EncodeDs1(5, 0, 1));
code.push_back(EncodeDs0(0x39, 4));
code.push_back(EncodeDs1(6, 0, 1));
AppendVMovLiteral(&code, 7, 0x000080ffu);
code.push_back(EncodeDs0(0x1f, 8));
code.push_back(EncodeDs1(0, 7, 1));
code.push_back(EncodeDs0(0x3c, 8));
code.push_back(EncodeDs1(8, 0, 1));
code.push_back(EncodeDs0(0x3b, 8));
code.push_back(EncodeDs1(9, 0, 1));
AppendVMovLiteral(&code, 10, 0x10101010u);
AppendVMovLiteral(&code, 11, 0x11111111u);
code.push_back(EncodeDs0(0x4d, 12));
code.push_back(EncodeDs1(0, 10, 1));
code.push_back(EncodeDs0(0x76, 12));
code.push_back(EncodeDs1(14, 0, 1));
AppendVMovLiteral(&code, 16, 0x20202020u);
AppendVMovLiteral(&code, 17, 0x21212121u);
AppendVMovLiteral(&code, 18, 0x22222222u);
code.push_back(EncodeDs0(0xde, 20));
code.push_back(EncodeDs1(0, 16, 1));
code.push_back(EncodeDs0(0xfe, 20));
code.push_back(EncodeDs1(19, 0, 1));
AppendVMovLiteral(&code, 22, 0x30303030u);
AppendVMovLiteral(&code, 23, 0x31313131u);
AppendVMovLiteral(&code, 24, 0x32323232u);
AppendVMovLiteral(&code, 25, 0x33333333u);
code.push_back(EncodeDs0(0xdf, 32));
code.push_back(EncodeDs1(0, 22, 1));
code.push_back(EncodeDs0(0xff, 32));
code.push_back(EncodeDs1(26, 0, 1));
const u32 results[] = {3, 5, 6, 8, 9, 14, 15, 19, 20, 21, 26, 27, 28, 29};
for (u32 i = 0; i < static_cast<u32>(std::size(results)); i++) {
AppendStoreVgpr(&code, results[i], i);
}
AppendEnd(&code);
return {"DsReadWriteVariants",
code,
std::vector<u32>(14, 0),
{0x11223344u, 0xaau, 0xffffffaau, 0x80ffu, 0xffff80ffu, 0x10101010u,
0x11111111u, 0x20202020u, 0x21212121u, 0x22222222u, 0x30303030u,
0x31313131u, 0x32323232u, 0x33333333u},
{O::V_MOV_B32, O::DS_WRITE_B32, O::DS_READ_B32, O::DS_WRITE_B8,
O::DS_READ_U8, O::DS_READ_I8, O::DS_WRITE_B16, O::DS_READ_U16,
O::DS_READ_I16, O::DS_WRITE_B64, O::DS_READ_B64, O::DS_WRITE_B96,
O::DS_READ_B96, O::DS_WRITE_B128, O::DS_READ_B128,
O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}
TestCase DsReadWriteFullRegionAddress(u32 wave_size, bool gds) {
using O = ShaderOpcode;
TestCase test;
test.name = gds ? (wave_size == 32u ? "DsGdsFullAddress32" : "DsGdsFullAddress64")
: (wave_size == 32u ? "DsLdsFullAddress32" : "DsLdsFullAddress64");
auto &code = test.code;
code.push_back(EncodeVop2(0x1a, 1, InlineU32(2), 0));
code.push_back(EncodeVop2(0x25, 2, InlineU32(1), 0));
code.insert(code.end(), {EncodeDs0(0x0d, 4, gds), EncodeDs1(0, 2, 1)});
AppendVMovLiteral(&code, 3, 0x10000u);
code.push_back(EncodeVop2(0x25, 3, Vgpr(1), 3));
AppendVMovLiteral(&code, 4, 0xdeadbeefu);
// A high region address must not alias the valid word in the same lane.
code.insert(code.end(), {EncodeDs0(0x0d, 4, gds), EncodeDs1(0, 4, 3),
EncodeDs0(0x36, 4, gds), EncodeDs1(5, 0, 1),
EncodeDs0(0x36, 4, gds), EncodeDs1(6, 0, 3)});
AppendStoreVgprAtLaneDwordOffset(&code, 5, 0, 0);
AppendStoreVgprAtLaneDwordOffset(&code, 6, 0, wave_size);
AppendEnd(&code);
test.initial.assign(wave_size * 2u, 0xdeadbeefu);
test.expected.resize(wave_size * 2u);
std::iota(test.expected.begin(), test.expected.begin() + wave_size, 1u);
test.compute_info.threads_num[0] = wave_size;
test.compute_info.threads_num[1] = test.compute_info.threads_num[2] = 1u;
test.compute_info.thread_ids_num = 1;
test.compute_info.wave_size = wave_size;
test.compute_info.lds_size_dwords = wave_size + 1u;
test.has_compute_info = true;
if (gds) {
test.gds_initial.resize(wave_size + 1u);
test.expected_gds.resize(wave_size + 1u);
std::iota(test.expected_gds.begin(), test.expected_gds.end(), 0u);
}
test.opcodes = {O::V_LSHLREV_B32, O::V_ADD_NC_U32, O::V_MOV_B32, O::DS_WRITE_B32,
O::DS_READ_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.forbidden_spirv = {"%uint_65535"};
return test;
}
TestCase DsReadU16D16CapturedPreservesHighHalf() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 1, 0);
AppendVMovLiteral(&code, 2, 0x89abcdefu);
code.push_back(EncodeDs0(0x0d));
code.push_back(EncodeDs1(0, 2, 1)); // ds_write_b32 v2, v1
AppendVMovLiteral(&code, 3, 0x12345678u);
code.push_back(EncodeDs0(0xa6));
code.push_back(EncodeDs1(3, 0, 1)); // ds_read_u16_d16 v3, v1
AppendVMovU32(&code, 7, 0);
code.push_back(0xda980000u);
code.push_back(0x07000007u); // captured ds_read_u16_d16 v7, v7
AppendStoreVgpr(&code, 3, 0);
AppendStoreVgpr(&code, 7, 1);
AppendEnd(&code);
TestCase test{"DsReadU16D16CapturedPreservesHighHalf",
code,
std::vector<u32>(2, 0),
{0x1234cdefu, 0x0000cdefu},
{O::V_MOV_B32, O::DS_WRITE_B32, O::DS_READ_U16_D16,
O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
test.decoded_counts = {{"DS_READ_U16_D16", 2}};
return test;
}
TestCase DsWriteB16D16HiCapturedUsesHighHalf() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 1, 0);
AppendVMovLiteral(&code, 3, 0xdeadbeefu);
code.push_back(EncodeDs0(0x0d));
code.push_back(EncodeDs1(0, 3, 1)); // ds_write_b32 v1, v3
AppendVMovU32(&code, 20, 2);
AppendVMovLiteral(&code, 2, 0x1234abcdu);
AppendVMovLiteral(&code, 0, 0x87654321u);
code.push_back(0xda840000u);
code.push_back(0x00000214u); // captured ds_write_b16_d16_hi v20, v2
code.push_back(EncodeDs0(0x36));
code.push_back(EncodeDs1(4, 0, 1)); // ds_read_b32 v4, v1
AppendStoreVgpr(&code, 4, 0);
AppendStoreVgpr(&code, 0, 1);
AppendEnd(&code);
TestCase test{"DsWriteB16D16HiCapturedUsesHighHalf",
code,
std::vector<u32>(2, 0),
{0x1234beefu, 0x87654321u},
{O::V_MOV_B32, O::DS_WRITE_B32, O::DS_WRITE_B16_D16_HI,
O::DS_READ_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
test.decoded_counts = {{"DS_WRITE_B16_D16_HI", 1}};
return test;
}
TestCase DsWriteB8D16HiWritesByteTwo() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 1, 0);
AppendVMovLiteral(&code, 3, 0xdeadbeefu);
code.push_back(EncodeDs0(0x0d));
code.push_back(EncodeDs1(0, 3, 1)); // ds_write_b32 v1, v3
// Every byte of the source differs, so selecting any byte but 23:16 fails.
AppendVMovU32(&code, 20, 1);
AppendVMovLiteral(&code, 2, 0x11a22233u);
AppendVMovLiteral(&code, 0, 0x87654321u);
code.push_back(EncodeDs0(0xa0));
code.push_back(EncodeDs1(0, 2, 20)); // ds_write_b8_d16_hi v20, v2
code.push_back(EncodeDs0(0x36));
code.push_back(EncodeDs1(4, 0, 1)); // ds_read_b32 v4, v1
AppendStoreVgpr(&code, 4, 0);
AppendStoreVgpr(&code, 0, 1);
AppendEnd(&code);
TestCase test{"DsWriteB8D16HiWritesByteTwo",
code,
std::vector<u32>(2, 0),
{0xdeada2efu, 0x87654321u},
{O::V_MOV_B32, O::DS_WRITE_B32, O::DS_WRITE_B8_D16_HI,
O::DS_READ_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
test.decoded_counts = {{"DS_WRITE_B8_D16_HI", 1}};
return test;
}
TestCase DsReadU16D16HiCapturedPreservesLowHalf() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 1, 0);
AppendVMovLiteral(&code, 2, 0x89abcdefu);
code.push_back(EncodeDs0(0x0d));
code.push_back(EncodeDs1(0, 2, 1)); // ds_write_b32 v2, v1
AppendVMovLiteral(&code, 5, 0x12345678u);
AppendVMovU32(&code, 6, 0);
code.push_back(0xda9c0000u);
code.push_back(0x05000006u); // captured ds_read_u16_d16_hi v5, v6
AppendStoreVgpr(&code, 5, 0);
AppendEnd(&code);
TestCase test{"DsReadU16D16HiCapturedPreservesLowHalf",
code,
std::vector<u32>(1, 0),
{0xcdef5678u},
{O::V_MOV_B32, O::DS_WRITE_B32, O::DS_READ_U16_D16_HI,
O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
test.decoded_counts = {{"DS_READ_U16_D16_HI", 1}};
return test;
}
TestCase DsReadWrite2Variants() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 1, 0);
AppendVMovLiteral(&code, 2, 0x11111111u);
AppendVMovLiteral(&code, 3, 0x22222222u);
code.push_back(EncodeDs0(0x0e, (3u << 8u) | 1u));
code.push_back(EncodeDs1Ex(0, 3, 2, 1));
code.push_back(EncodeDs0(0x37, (3u << 8u) | 1u));
code.push_back(EncodeDs1Ex(4, 0, 0, 1));
AppendVMovLiteral(&code, 6, 0x33333333u);
AppendVMovLiteral(&code, 7, 0x44444444u);
AppendVMovLiteral(&code, 8, 0x55555555u);
AppendVMovLiteral(&code, 9, 0x66666666u);
code.push_back(EncodeDs0(0x4d, 32));
code.push_back(EncodeDs1(0, 6, 1));
code.push_back(EncodeDs0(0x4d, 48));
code.push_back(EncodeDs1(0, 8, 1));
code.push_back(EncodeDs0(0x77, (6u << 8u) | 4u));
code.push_back(EncodeDs1Ex(10, 0, 0, 1));
AppendVMovLiteral(&code, 14, 0x77777777u);
AppendVMovLiteral(&code, 15, 0x88888888u);
code.push_back(EncodeDs0(0x0f, (2u << 8u) | 1u));
code.push_back(EncodeDs1Ex(0, 15, 14, 1));
code.push_back(EncodeDs0(0x38, (2u << 8u) | 1u));
code.push_back(EncodeDs1Ex(16, 0, 0, 1));
AppendVMovLiteral(&code, 18, 0x99999999u);
AppendVMovLiteral(&code, 19, 0xaaaaaaaau);
AppendVMovLiteral(&code, 20, 0xbbbbbbbbu);
AppendVMovLiteral(&code, 21, 0xccccccccu);
code.push_back(EncodeDs0(0x4e, (10u << 8u) | 8u));
code.push_back(EncodeDs1Ex(0, 20, 18, 1));
code.push_back(EncodeDs0(0x77, (10u << 8u) | 8u));
code.push_back(EncodeDs1Ex(22, 0, 0, 1));
AppendVMovLiteral(&code, 26, 0xddddddddu);
AppendVMovLiteral(&code, 27, 0xeeeeeeeeu);
AppendVMovLiteral(&code, 28, 0xf0f0f0f0u);
AppendVMovLiteral(&code, 29, 0x12345678u);
code.push_back(EncodeDs0(0x4f, (2u << 8u) | 1u));
code.push_back(EncodeDs1Ex(0, 28, 26, 1));
code.push_back(EncodeDs0(0x78, (2u << 8u) | 1u));
code.push_back(EncodeDs1Ex(34, 0, 0, 1));
const u32 results[] = {4, 5, 10, 11, 12, 13, 22, 23,
24, 25, 16, 17, 34, 35, 36, 37};
for (u32 i = 0; i < static_cast<u32>(std::size(results)); i++) {
AppendStoreVgpr(&code, results[i], i);
}
AppendEnd(&code);
return {"DsReadWrite2Variants",
code,
std::vector<u32>(16, 0),
{0x11111111u, 0x22222222u, 0x33333333u, 0x44444444u, 0x55555555u,
0x66666666u, 0x99999999u, 0xaaaaaaaau, 0xbbbbbbbbu, 0xccccccccu,
0x77777777u, 0x88888888u, 0xddddddddu, 0xeeeeeeeeu, 0xf0f0f0f0u,
0x12345678u},
{O::V_MOV_B32, O::DS_WRITE2_B32, O::DS_READ2_B32, O::DS_WRITE_B64,
O::DS_READ2_B64, O::DS_WRITE2_B64, O::DS_WRITE2ST64_B32,
O::DS_READ2ST64_B32, O::DS_WRITE2ST64_B64, O::DS_READ2ST64_B64,
O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}
TestCase DsWideReadSnapshotsOverlappingAddress() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 1, 0);
AppendVMovLiteral(&code, 6, 0x11111111u);
AppendVMovLiteral(&code, 7, 0x22222222u);
code.push_back(EncodeDs0(0x4d, 0));
code.push_back(EncodeDs1(0, 6, 1));
AppendVMovLiteral(&code, 8, 0x33333333u);
AppendVMovLiteral(&code, 9, 0x44444444u);
code.push_back(EncodeDs0(0x4d, 16));
code.push_back(EncodeDs1(0, 8, 1));
code.push_back(EncodeDs0(0x77, 2u << 8u));
code.push_back(EncodeDs1Ex(1, 0, 0, 1));
for (u32 i = 0; i < 4; i++) {
AppendStoreVgpr(&code, 1u + i, i);
}
AppendEnd(&code);
return {"DsWideReadSnapshotsOverlappingAddress",
code,
std::vector<u32>(4, 0),
{0x11111111u, 0x22222222u, 0x33333333u, 0x44444444u},
{O::V_MOV_B32, O::DS_WRITE_B64, O::DS_READ2_B64,
O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}
TestCase DsReadWrite2EqualOffsetsUseData0() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 1, 0);
AppendVMovLiteral(&code, 2, 0x11111111u);
AppendVMovLiteral(&code, 3, 0x22222222u);
code.push_back(EncodeDs0(0x0e, (1u << 8u) | 1u));
code.push_back(EncodeDs1Ex(0, 3, 2, 1));
AppendVMovLiteral(&code, 4, 0x33333333u);
AppendVMovLiteral(&code, 5, 0x44444444u);
AppendVMovLiteral(&code, 6, 0x55555555u);
AppendVMovLiteral(&code, 7, 0x66666666u);
code.push_back(EncodeDs0(0x4e, (2u << 8u) | 2u));
code.push_back(EncodeDs1Ex(0, 6, 4, 1));
code.push_back(EncodeDs0(0x37, (1u << 8u) | 1u));
code.push_back(EncodeDs1Ex(8, 0, 0, 1));
code.push_back(EncodeDs0(0x77, (2u << 8u) | 2u));
code.push_back(EncodeDs1Ex(10, 0, 0, 1));
for (u32 i = 0; i < 6; i++) {
AppendStoreVgpr(&code, 8u + i, i);
}
AppendEnd(&code);
return {"DsReadWrite2EqualOffsetsUseData0",
code,
std::vector<u32>(6, 0),
{0x11111111u, 0x11111111u, 0x33333333u, 0x44444444u, 0x33333333u,
0x44444444u},
{O::V_MOV_B32, O::DS_WRITE2_B32, O::DS_WRITE2_B64, O::DS_READ2_B32,
O::DS_READ2_B64, O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}
TestCase DsWideLdsPartialBounds() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 1, 0);
AppendVMovLiteral(&code, 2, 0x11111111u);
AppendVMovLiteral(&code, 3, 0x22222222u);
AppendVMovLiteral(&code, 4, 0x33333333u);
AppendVMovLiteral(&code, 5, 0x44444444u);
code.push_back(EncodeDs0(0xdf));
code.push_back(EncodeDs1(0, 2, 1));
code.push_back(EncodeDs0(0xff));
code.push_back(EncodeDs1(6, 0, 1));
for (u32 i = 0; i < 4; i++) {
AppendStoreVgpr(&code, 6u + i, i);
}
AppendEnd(&code);
TestCase test{"DsWideLdsPartialBounds",
code,
std::vector<u32>(4, 0),
{0x11111111u, 0x22222222u, 0x33333333u, 0},
{O::V_MOV_B32, O::DS_WRITE_B128, O::DS_READ_B128,
O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
test.compute_info.lds_size_dwords = 3;
return test;
}
TestCase DsWideGdsPartialBounds() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 1, 0);
AppendVMovLiteral(&code, 2, 0x11111111u);
AppendVMovLiteral(&code, 3, 0x22222222u);
AppendVMovLiteral(&code, 4, 0x33333333u);
AppendVMovLiteral(&code, 5, 0x44444444u);
code.push_back(EncodeDs0(0xdf, 0, true));
code.push_back(EncodeDs1(0, 2, 1));
code.push_back(EncodeDs0(0xff, 0, true));
code.push_back(EncodeDs1(6, 0, 1));
for (u32 i = 0; i < 4; i++) {
AppendStoreVgpr(&code, 6u + i, i);
}
AppendEnd(&code);
TestCase test{"DsWideGdsPartialBounds",
code,
std::vector<u32>(4, 0),
{0x11111111u, 0x22222222u, 0x33333333u, 0},
{O::V_MOV_B32, O::DS_WRITE_B128, O::DS_READ_B128,
O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
test.gds_initial = {0, 0, 0};
test.expected_gds = {0x11111111u, 0x22222222u, 0x33333333u};
return test;
}
TestCase DsAtomicNoReturnVariants() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 1, 0);
const u32 initial[] = {10, 10, 2, 1, 0, 3, 9, 0xfffffff0u, 0xfffffff0u,
10, 10, 0xf0f0u, 0xf000u, 0xf00fu};
const u32 values[] = {5, 3, 2, 5, 2, 5, 5, 5, 5,
5, 20, 0x0ff0u, 0x0f00u, 0x00ffu};
const u32 ops[] = {0x00, 0x01, 0x03, 0x03, 0x04, 0x04, 0x04,
0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b};
for (u32 i = 0; i < static_cast<u32>(std::size(values)); i++) {
AppendVMovLiteral(&code, 2, initial[i]);
code.push_back(EncodeDs0(0x0d, i * 4u));
code.push_back(EncodeDs1(0, 2, 1));
AppendVMovLiteral(&code, 3, values[i]);
code.push_back(EncodeDs0(ops[i], i * 4u));
code.push_back(EncodeDs1(0, 3, 1));
code.push_back(EncodeDs0(0x36, i * 4u));
code.push_back(EncodeDs1(10u + i, 0, 1));
}
for (u32 i = 0; i < static_cast<u32>(std::size(values)); i++) {
AppendStoreVgpr(&code, 10u + i, i);
}
AppendEnd(&code);
return {"DsAtomicNoReturnVariants",
code,
std::vector<u32>(14, 0),
{15, 7, 0, 2, 2, 2, 5, 0xfffffff0u, 5, 5, 20, 0x00f0u, 0xff00u,
0xf0f0u},
{O::V_MOV_B32, O::DS_WRITE_B32, O::DS_ADD_U32, O::DS_SUB_U32,
O::DS_INC_U32, O::DS_DEC_U32, O::DS_MIN_I32, O::DS_MAX_I32,
O::DS_MIN_U32, O::DS_MAX_U32, O::DS_AND_B32, O::DS_OR_B32,
O::DS_XOR_B32, O::DS_READ_B32, O::BUFFER_STORE_DWORD,
O::S_ENDPGM}};
}
TestCase DsMaskedOrBounds(bool gds) {
using O = ShaderOpcode;
TestCase test;
test.name = gds ? "DsMaskedOrGdsBounds" : "DsMaskedOrLdsBounds";
auto &code = test.code;
constexpr u32 initial[] = {0xf0f00ff0u, 0x12345678u, 0x87654321u};
constexpr u32 masks[] = {0x00ff00ffu, 0u, 0xffffffffu};
constexpr u32 values[] = {0x33000300u, 0x80000001u, 0xabcdef01u};
AppendVMovU32(&code, 0, 0x1234);
for (u32 i = 0; i < std::size(initial); ++i) {
AppendVMovU32(&code, 2, i * 4u);
AppendVMovLiteral(&code, 8, initial[i]);
code.push_back(EncodeDs0(0x0d, 0x144, gds));
code.push_back(EncodeDs1(0, 8, 2));
// Captured mask v8 and data v10 are distinct; LDS wraps and aligns the address.
AppendVMovU32(&code, 2, (gds ? 0u : 0x10000u) + i * 4u + 3u);
AppendVMovLiteral(&code, 8, masks[i]);
AppendVMovLiteral(&code, 10, values[i]);
code.insert(code.end(), {0xd8300144u | (gds ? 1u << 17u : 0u), 0x000a0802u});
}
AppendVMovLiteral(&code, 8, 0xffffffffu);
AppendVMovU32(&code, 10, 0);
AppendVMovU32(&code, 2, 12); // First DWORD beyond the allocation.
code.insert(code.end(), {EncodeDs0(0x0c, 0x144, gds), 0x000a0802u});
AppendVMovU32(&code, 2, 0);
code.push_back(EncodeSMovB32(126, InlineU32(0)));
code.insert(code.end(), {EncodeDs0(0x0c, 0x144, gds), 0x000a0802u});
code.push_back(EncodeSMovB32(126, InlineU32(1)));
for (u32 i = 0; i < std::size(initial); ++i) {
code.push_back(EncodeDs0(0x36, 0x144u + i * 4u, gds));
code.push_back(EncodeDs1(11, 0, 2));
AppendStoreVgpr(&code, 11, i);
test.expected.push_back((initial[i] & ~masks[i]) | values[i]);
}
AppendStoreVgpr(&code, 0, 3); // The encoded VDst must stay untouched.
test.expected.push_back(0x1234);
AppendEnd(&code);
test.compute_info.lds_size_dwords = 0x144 / 4 + 3;
if (gds) {
test.gds_initial.resize(0x144 / 4 + 3);
test.expected_gds = test.gds_initial;
std::copy_n(test.expected.begin(), 3, test.expected_gds.begin() + 0x144 / 4);
}
test.opcodes = {O::V_MOV_B32, O::S_MOV_B32, O::DS_WRITE_B32, O::DS_MSKOR_B32,
O::DS_READ_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.required_spirv = {"OpAtomicCompareExchange"};
return test;
}
TestCase DsMaskedOrContention(u32 wave_size) {
using O = ShaderOpcode;
TestCase test;
test.name = wave_size == 32 ? "DsMaskedOrContentionWave32" : "DsMaskedOrContentionWave64";
auto &code = test.code;
AppendVMovU32(&code, 2, 0);
AppendVMovLiteral(&code, 8, 0xffffffffu);
code.push_back(EncodeSop1(0x04, 10, 126));
code.push_back(EncodeVopc(0xc2, InlineU32(0), 0));
code.push_back(EncodeSop1(0x04, 126, 106));
code.push_back(EncodeDs0(0x0d));
code.push_back(EncodeDs1(0, 8, 2));
code.push_back(EncodeSop1(0x04, 126, 10));
code.push_back(EncodeSopp(0x0a, 0));
AppendVMovU32(&code, 8, 1);
code.push_back(EncodeVop2(0x1a, 8, Vgpr(0), 8));
AppendVMovU32(&code, 9, 0);
AppendSMovLiteral(&code, 126, 0x55555555u);
AppendSMovLiteral(&code, 127, 0xaaaaaaaau);
code.push_back(EncodeDs0(0x0c));
code.push_back(EncodeDs1Ex(0, 9, 8, 2));
code.push_back(EncodeSop1(0x04, 126, 10));
code.push_back(EncodeSopp(0x0a, 0));
code.push_back(EncodeDs0(0x36));
code.push_back(EncodeDs1(11, 0, 2));
AppendStoreVgprAtLaneDwordOffset(&code, 11, 0, 0);
AppendEnd(&code);
test.expected.assign(128, wave_size == 32 ? 0xaaaaaaaau : 0u);
test.compute_info.threads_num[0] = 128;
test.compute_info.threads_num[1] = test.compute_info.threads_num[2] = 1;
test.compute_info.thread_ids_num = 1;
test.compute_info.wave_size = wave_size;
test.compute_info.lds_size_dwords = 1;
test.has_compute_info = true;
test.opcodes = {O::V_MOV_B32, O::S_MOV_B32, O::S_MOV_B64, O::V_CMP_EQ_U32,
O::V_LSHLREV_B32, O::S_BARRIER, O::DS_WRITE_B32, O::DS_MSKOR_B32,
O::DS_READ_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.required_spirv = {"OpAtomicCompareExchange"};
return test;
}
TestCase DsAtomicReturnVariants() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 1, 0);
const u32 initial[] = {10, 10, 0xfffffff0u, 0xfffffff0u, 10,
10, 0xf0f0u, 0xf000u, 0xf00fu, 10};
const u32 values[] = {5, 3, 5, 5, 5, 20, 0x0ff0u, 0x0f00u, 0x00ffu, 99};
const u32 ops[] = {0x20, 0x21, 0x25, 0x26, 0x27,
0x28, 0x29, 0x2a, 0x2b, 0x2d};
for (u32 i = 0; i < static_cast<u32>(std::size(values)); i++) {
AppendVMovLiteral(&code, 2, initial[i]);
code.push_back(EncodeDs0(0x0d, i * 4u));
code.push_back(EncodeDs1(0, 2, 1));
AppendVMovLiteral(&code, 3, values[i]);
code.push_back(EncodeDs0(ops[i], i * 4u));
code.push_back(EncodeDs1(10u + i, 3, 1));
code.push_back(EncodeDs0(0x36, i * 4u));
code.push_back(EncodeDs1(20u + i, 0, 1));
}
for (u32 i = 0; i < static_cast<u32>(std::size(values)); i++) {
AppendStoreVgpr(&code, 10u + i, i);
}
for (u32 i = 0; i < static_cast<u32>(std::size(values)); i++) {
AppendStoreVgpr(&code, 20u + i, i + 10u);
}
AppendEnd(&code);
return {
"DsAtomicReturnVariants",
code,
std::vector<u32>(20, 0),
{10, 10, 0xfffffff0u, 0xfffffff0u, 10, 10, 0xf0f0u, 0xf000u, 0xf00fu, 10,
15, 7, 0xfffffff0u, 5, 5, 20, 0x00f0u, 0xff00u, 0xf0f0u, 99},
{O::V_MOV_B32, O::DS_WRITE_B32, O::DS_ADD_RTN_U32, O::DS_SUB_RTN_U32,
O::DS_MIN_RTN_I32, O::DS_MAX_RTN_I32, O::DS_MIN_RTN_U32,
O::DS_MAX_RTN_U32, O::DS_AND_RTN_B32, O::DS_OR_RTN_B32,
O::DS_XOR_RTN_B32, O::DS_WRXCHG_RTN_B32, O::DS_READ_B32,
O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}
TestCase DsAtomic64Bounds(bool add) {
using O = ShaderOpcode;
TestCase test;
test.name = add ? "DsAddU64Bounds" : "DsOrB64Bounds";
const u32 atomic_word = add ? 0xd9000000u : 0xd9280000u;
auto &code = test.code;
AppendVMovU32(&code, 0, 0x1234);
AppendVMovU32(&code, 7, 0);
const std::array<u32, 5> initial = add
? std::array<u32, 5>{0xffffffffu, 1, 0xffffffffu, 0xfffffffeu, 0x13579bdf}
: std::array<u32, 5>{0x100, 0x200, 0x400, 0x800, 0x13579bdf};
for (u32 i = 0; i < std::size(initial); ++i) {
AppendVMovLiteral(&code, 3, initial[i]);
code.push_back(EncodeDs0(0x0d, i * 4u));
code.push_back(EncodeDs1(0, 3, 7));
}
AppendVMovU32(&code, 63, 1);
AppendVMovU32(&code, 64, 2);
AppendVMovU32(&code, 7, 3); // The 64-bit address ignores the low three bits.
code.insert(code.end(), {atomic_word, 0x00003f07u});
AppendVMovLiteral(&code, 7, 0x10007);
code.push_back(EncodeDs0(add ? 0x40 : 0x4a, 2)); // Offset before wrap/alignment -> 8.
code.push_back(EncodeDs1(0, 63, 7));
AppendVMovLiteral(&code, 63, 0xffffffffu);
AppendVMovLiteral(&code, 64, 0xffffffffu);
AppendVMovU32(&code, 7, 16); // Only one DWORD remains: ignore the entire atomic.
code.insert(code.end(), {atomic_word, 0x00003f07u});
AppendVMovU32(&code, 7, 0);
code.push_back(EncodeSMovB32(126, InlineU32(0)));
code.insert(code.end(), {atomic_word, 0x00003f07u});
code.push_back(EncodeSMovB32(126, InlineU32(1)));
for (u32 i = 0; i < std::size(initial); ++i) {
code.push_back(EncodeDs0(0x36, i * 4u));
code.push_back(EncodeDs1(8, 0, 7));
AppendStoreVgpr(&code, 8, i);
}
AppendStoreVgpr(&code, 0, 5); // No-return DS must not overwrite the encoded VDst.
AppendEnd(&code);
test.expected = add ? std::vector<u32>{0, 4, 0, 1, 0x13579bdf, 0x1234}
: std::vector<u32>{0x101, 0x202, 0x401, 0x802, 0x13579bdf, 0x1234};
test.compute_info.lds_size_dwords = 5;
test.opcodes = {O::V_MOV_B32, O::S_MOV_B32, O::DS_WRITE_B32,
add ? O::DS_ADD_U64 : O::DS_OR_B64,
O::DS_READ_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.required_spirv = {add ? "OpAtomicIAdd %ulong" : "OpAtomicOr %ulong",
"WorkgroupMemoryExplicitLayoutKHR", "Aliased"};
return test;
}
TestCase DsAtomicReturn64() {
using O = ShaderOpcode;
TestCase test;
test.name = "DsAtomicReturn64";
auto &code = test.code;
const auto pair = [&](u32 reg, uint64_t value) {
AppendVMovLiteral(&code, reg, static_cast<u32>(value));
AppendVMovLiteral(&code, reg + 1u, static_cast<u32>(value >> 32u));
};
const auto result = [&](u32 reg, uint64_t value) {
AppendStoreVgpr(&code, reg, static_cast<u32>(test.expected.size()));
test.expected.push_back(static_cast<u32>(value));
AppendStoreVgpr(&code, reg + 1u, static_cast<u32>(test.expected.size()));
test.expected.push_back(static_cast<u32>(value >> 32u));
};
AppendVMovU32(&code, 7, 0);
AppendVMovLiteral(&code, 6, 0x13579bdfu);
code.push_back(EncodeDs0(0x0d, 24));
code.push_back(EncodeDs1(0, 6, 7));
struct Case { uint64_t before, value; u32 address, offset; };
constexpr std::array cases{
Case{0x7fffffffffffffffull, 0x8000000000000000ull, 0, 8},
Case{0x8000000000000000ull, 0x7fffffffffffffffull, 0, 8},
Case{0x8000000000000001ull, 0x8000000000000002ull, 0x10007, 2},
Case{0xffffffffffffffffull, 0, 0, 8}};
for (const auto &item : cases) {
pair(2, item.before);
code.push_back(EncodeDs0(0x4d, 8));
code.push_back(EncodeDs1(0, 2, 7));
pair(4, item.value);
AppendVMovLiteral(&code, 0, item.address);
// CS23e96bf5029f62fd pc 0xec8: d9a00008 00000400. VDst aliases ADDR.
code.push_back(EncodeDs0(0x68, item.offset));
code.push_back(0x00000400u);
result(0, item.before);
code.push_back(EncodeDs0(0x76, 8));
code.push_back(EncodeDs1(8, 0, 7));
result(8, std::max(item.before, item.value));
}
struct CompareCase { uint64_t before, compare, replacement; };
constexpr std::array compare_cases{
CompareCase{0xfedcba9876543210ull, 0xfedcba9876543210ull, 0x0123456789abcdefull},
CompareCase{0xfedcba9876543210ull, 0x0edcba9876543210ull, UINT64_MAX},
CompareCase{0xfedcba9876543210ull, 0xfedcba9876543211ull, UINT64_MAX},
CompareCase{UINT64_MAX, UINT64_MAX, 0}};
AppendVMovU32(&code, 63, 0);
for (const auto &item : compare_cases) {
pair(4, item.before);
code.push_back(EncodeDs0(0x4d, 16));
code.push_back(EncodeDs1(0, 4, 7));
pair(0, item.compare);
pair(2, item.replacement);
// pc 0x1048: DATA compares, DATA2 replaces, VDst aliases DATA2.
code.insert(code.end(), {0xd9c00010u, 0x0202003fu});
result(2, item.before);
code.push_back(EncodeDs0(0x76, 16));
code.push_back(EncodeDs1(8, 0, 7));
result(8, item.before == item.compare ? item.replacement : item.before);
}
pair(4, UINT64_MAX);
AppendVMovU32(&code, 0, 24); // One DWORD remains: ignore the entire atomic.
code.push_back(EncodeDs0(0x68));
code.push_back(EncodeDs1(2, 4, 0));
result(2, 0);
pair(2, 0xabcdef0112345678ull);
AppendVMovU32(&code, 0, 0);
code.push_back(EncodeSMovB32(126, InlineU32(0)));
code.insert(code.end(), {0xd9a00008u, 0x02000400u});
code.push_back(EncodeSMovB32(126, InlineU32(1)));
result(2, 0xabcdef0112345678ull); // Inactive lanes retain both destination DWORDs.
code.push_back(EncodeDs0(0x36, 24));
code.push_back(EncodeDs1(6, 0, 7));
AppendStoreVgpr(&code, 6, static_cast<u32>(test.expected.size()));
test.expected.push_back(0x13579bdfu);
AppendEnd(&code);
test.compute_info.lds_size_dwords = 7;
test.opcodes = {O::V_MOV_B32, O::S_MOV_B32, O::DS_WRITE_B32, O::DS_WRITE_B64,
O::DS_MAX_RTN_U64, O::DS_CMPST_RTN_B64, O::DS_READ_B32, O::DS_READ_B64,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.required_spirv = {"OpAtomicUMax %ulong", "OpAtomicCompareExchange %ulong", "WorkgroupMemoryExplicitLayoutKHR", "Aliased"};
test.forbidden_spirv = {"OpAtomicLoad", "OpLoopMerge"};
return test;
}
TestCase DsCompareStoreReturn32() {
using O = ShaderOpcode;
TestCase test;
test.name = "DsCompareStoreReturn32";
auto &code = test.code;
const auto result = [&](u32 reg, u32 value) {
AppendStoreVgpr(&code, reg, static_cast<u32>(test.expected.size()));
test.expected.push_back(value);
};
struct Case { u32 before, compare, replacement; };
constexpr std::array cases{Case{0x12345678u, 0x12345678u, 0x9abcdef0u},
Case{0x12345678u, 0x9abcdef0u, 0x12345678u},
Case{0x80000000u, 0u, 0x7fffffffu},
Case{UINT32_MAX, UINT32_MAX, 0u}};
AppendVMovU32(&code, 7, 0);
for (const auto &item : cases) {
AppendVMovLiteral(&code, 2, item.before);
code.push_back(EncodeDs0(0x0d, 8));
code.push_back(EncodeDs1(0, 2, 7));
AppendVMovLiteral(&code, 5, item.compare);
AppendVMovLiteral(&code, 19, item.replacement);
AppendVMovU32(&code, 4, 8);
// CS04b812daaacd532f pc 0x3e8: DATA compares, DATA2 replaces, VDst aliases ADDR.
code.insert(code.end(), {0xd8c00000u, 0x04130504u});
result(4, item.before);
code.push_back(EncodeDs0(0x36, 8));
code.push_back(EncodeDs1(8, 0, 7));
result(8, item.before == item.compare ? item.replacement : item.before);
}
AppendVMovU32(&code, 5, 0);
AppendVMovU32(&code, 4, 12); // Past the three LDS DWORDs.
code.insert(code.end(), {0xd8c00000u, 0x04130504u});
result(4, 0);
AppendVMovLiteral(&code, 2, 0x12345678u);
code.push_back(EncodeDs0(0x0d, 8));
code.push_back(EncodeDs1(0, 2, 7));
AppendVMovLiteral(&code, 5, 0x12345678u);
AppendVMovU32(&code, 4, 8);
code.push_back(EncodeSMovB32(126, InlineU32(0)));
code.insert(code.end(), {0xd8c00000u, 0x04130504u});
code.push_back(EncodeSMovB32(126, InlineU32(1)));
result(4, 8); // An inactive lane keeps its destination and leaves memory alone.
code.push_back(EncodeDs0(0x36, 8));
code.push_back(EncodeDs1(8, 0, 7));
result(8, 0x12345678u);
AppendEnd(&code);
test.compute_info.lds_size_dwords = 3;
test.opcodes = {O::V_MOV_B32, O::S_MOV_B32, O::DS_WRITE_B32, O::DS_CMPST_RTN_B32,
O::DS_READ_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.required_spirv = {"OpAtomicCompareExchange %uint"};
test.forbidden_spirv = {"OpLoopMerge"};
return test;
}
TestCase DsAtomic64Contention(bool add, u32 wave_size) {
using O = ShaderOpcode;
TestCase test;
test.name = add ? (wave_size == 32 ? "DsAddU64ContentionWave32" : "DsAddU64ContentionWave64")
: (wave_size == 32 ? "DsOrB64ContentionWave32" : "DsOrB64ContentionWave64");
auto &code = test.code;
AppendVMovU32(&code, 7, 0);
code.push_back(EncodeSop1(0x04, 10, 126));
code.push_back(EncodeVopc(0xc2, InlineU32(0), 0));
code.push_back(EncodeSop1(0x04, 126, 106));
for (u32 offset : {0, 4}) {
code.push_back(EncodeDs0(0x0d, offset));
code.push_back(EncodeDs1(0, 7, 7));
}
code.push_back(EncodeSop1(0x04, 126, 10));
code.push_back(EncodeSopp(0x0a, 0));
if (add) {
AppendVMovLiteral(&code, 3, 0xffffffffu);
AppendVMovU32(&code, 4, 1);
} else {
AppendVMovU32(&code, 5, 1);
AppendVMovU32(&code, 6, 2);
code.push_back(EncodeVop2(0x1a, 3, Vgpr(0), 5));
code.push_back(EncodeVop2(0x1a, 4, Vgpr(0), 6));
}
AppendSMovLiteral(&code, 126, 0x55555555u);
AppendSMovLiteral(&code, 127, 0xaaaaaaaau);
code.insert(code.end(), {add ? 0xd9000000u : 0xd9280000u, 0x00000307u});
code.push_back(EncodeSop1(0x04, 126, 10));
code.push_back(EncodeSopp(0x0a, 0));
for (u32 half = 0; half < 2; ++half) {
code.push_back(EncodeDs0(0x36, half * 4));
code.push_back(EncodeDs1(8, 0, 7));
AppendStoreVgprAtLaneDwordOffset(&code, 8, 0, half * 128);
}
AppendEnd(&code);
test.expected.assign(128, add ? 0xffffffc0u : wave_size == 32 ? 0x55555555u : 0xffffffffu);
test.expected.resize(256, add ? 0x7fu : wave_size == 32 ? 0xaaaaaaaau : 0xfffffffeu);
test.compute_info.threads_num[0] = 128;
test.compute_info.threads_num[1] = test.compute_info.threads_num[2] = 1;
test.compute_info.thread_ids_num = 1;
test.compute_info.wave_size = wave_size;
test.compute_info.lds_size_dwords = 2;
test.has_compute_info = true;
test.opcodes = {O::V_MOV_B32, O::S_MOV_B32, O::S_MOV_B64, O::S_BARRIER,
O::V_CMP_EQ_U32, O::V_ADD_NC_U32,
O::DS_WRITE_B32, add ? O::DS_ADD_U64 : O::DS_OR_B64, O::DS_READ_B32,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
if (!add) {
test.opcodes.push_back(O::V_LSHLREV_B32);
}
test.required_spirv = {add ? "OpAtomicIAdd %ulong" : "OpAtomicOr %ulong",
"WorkgroupMemoryExplicitLayoutKHR", "Aliased"};
return test;
}
TestCase DsBoundedAtomicBoundaries(bool decrement, bool gds) {
using O = ShaderOpcode;
constexpr u32 guard = 0x13579bdfu;
struct Boundary {
u32 before, limit, incremented, decremented;
};
constexpr Boundary boundaries[] = {
{4, 5, 5, 3},
{5, 5, 0, 4},
{6, 5, 0, 5},
{0, 5, 1, 5},
{0, 0, 0, 0},
{1, 0, 0, 0},
{0xffffffffu, 0xffffffffu, 0, 0xfffffffeu},
{0xfffffffeu, 0xffffffffu, 0xffffffffu, 0xfffffffdu},
{0, 0xffffffffu, 1, 0xffffffffu},
{0x80000000u, 0x7fffffffu, 0, 0x7fffffffu},
{0x7fffffffu, 0x80000000u, 0x80000000u, 0x7ffffffeu},
{0xffffffffu, 0x80000000u, 0, 0x80000000u}};
constexpr u32 count = static_cast<u32>(std::size(boundaries));
const u32 opcode = decrement ? 0x24u : 0x23u;
TestCase test;
test.name =
decrement
? (gds ? "DsDecReturnGdsBoundaries" : "DsDecReturnLdsBoundaries")
: (gds ? "DsIncReturnGdsBoundaries" : "DsIncReturnLdsBoundaries");
auto &code = test.code;
AppendVMovU32(&code, 2, 0);
std::vector<u32> memory(68, guard);
for (u32 i = 0; i < count; ++i)
memory[i + 1] = boundaries[i].before;
if (!gds) {
for (u32 i = 0; i < memory.size(); ++i) {
AppendVMovLiteral(&code, 3, memory[i]);
code.push_back(EncodeDs0(0x0d, i * 4));
code.push_back(EncodeDs1(0, 3, 2));
}
}
auto StoreResult = [&](u32 vgpr, u32 expected) {
AppendStoreVgpr(&code, vgpr, static_cast<u32>(test.expected.size()));
test.expected.push_back(expected);
};
for (u32 i = 0; i < count; ++i) {
AppendVMovU32(&code, 2, i * 4);
AppendVMovLiteral(&code, 3, boundaries[i].limit);
// Preserve the captured DATA0/VDST alias and exact increment words in GDS.
code.push_back(decrement ? EncodeDs0(opcode, 4, gds)
: (gds ? 0xd88e0004u : 0xd88c0004u));
code.push_back(0x03000302u);
StoreResult(3, boundaries[i].before);
code.push_back(EncodeDs0(0x36, 4, gds));
code.push_back(EncodeDs1(4, 0, 2));
StoreResult(4, decrement ? boundaries[i].decremented
: boundaries[i].incremented);
}
// EXEC=0 preserves memory and the aliased destination register.
AppendVMovU32(&code, 2, 0);
AppendVMovLiteral(&code, 3, 0x12345678u);
code.push_back(EncodeSop1(0x04, 10, 126));
code.push_back(EncodeSop1(0x04, 126, InlineU32(0)));
code.push_back(EncodeDs0(opcode, 4, gds));
code.push_back(0x03000302u);
code.push_back(EncodeSop1(0x04, 126, 10));
StoreResult(3, 0x12345678u);
// Use both offset bytes and alias VDST with ADDR instead of DATA0.
AppendVMovU32(&code, 2, 4);
AppendVMovLiteral(&code, 3, 0xffffffffu);
code.push_back(EncodeDs0(opcode, 0x104, gds));
code.push_back(EncodeDs1(2, 3, 2));
StoreResult(2, guard);
// Overwriting an unused return must not let DCE discard the atomic update.
AppendVMovU32(&code, 2, 0);
code.push_back(EncodeDs0(opcode, 0x108, gds));
code.push_back(EncodeDs1(4, 3, 2));
AppendVMovU32(&code, 4, 0);
// An active access just outside the resource returns zero.
AppendVMovU32(&code, 2, static_cast<u32>(memory.size() * 4 - 4));
code.push_back(EncodeDs0(opcode, 4, gds));
code.push_back(0x03000302u);
StoreResult(3, 0);
// Read back all words, including guards around both address ranges.
AppendVMovU32(&code, 2, 0);
for (u32 i = 0; i < memory.size(); ++i) {
u32 expected = memory[i];
if (i >= 1 && i <= count) {
expected = decrement ? boundaries[i - 1].decremented
: boundaries[i - 1].incremented;
} else if (i == 66) {
expected = decrement ? guard - 2 : guard + 2;
}
code.push_back(EncodeDs0(0x36, i * 4, gds));
code.push_back(EncodeDs1(4, 0, 2));
StoreResult(4, expected);
if (gds)
test.expected_gds.push_back(expected);
}
AppendEnd(&code);
test.initial.resize(test.expected.size());
test.compute_info.lds_size_dwords = static_cast<u32>(memory.size());
test.opcodes = {
O::S_MOV_B64, O::V_MOV_B32,
O::DS_READ_B32, decrement ? O::DS_DEC_RTN_U32 : O::DS_INC_RTN_U32,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.required_spirv = {"OpAtomicLoad", "OpAtomicCompareExchange",
"OpLoopMerge"};
if (gds)
test.gds_initial = memory;
else
test.opcodes.push_back(O::DS_WRITE_B32);
return test;
}
TestCase DsBoundedAtomicContention(bool decrement, bool gds, u32 wave_size) {
using O = ShaderOpcode;
constexpr u32 guard = 0x13579bdfu;
constexpr u32 limit = 15;
TestCase test;
if (wave_size == 32) {
test.name = decrement ? (gds ? "DsDecReturnGdsContentionWave32"
: "DsDecReturnLdsContentionWave32")
: (gds ? "DsIncReturnGdsContentionWave32"
: "DsIncReturnLdsContentionWave32");
} else {
test.name = decrement ? (gds ? "DsDecReturnGdsContentionWave64"
: "DsDecReturnLdsContentionWave64")
: (gds ? "DsIncReturnGdsContentionWave64"
: "DsIncReturnLdsContentionWave64");
}
auto &code = test.code;
AppendVMovU32(&code, 2, 0);
AppendVMovU32(&code, 3, limit);
AppendVMovU32(&code, 5, 1);
AppendVMovLiteral(&code, 6, guard);
if (!gds) {
// One lane initializes LDS before every wave in the workgroup uses it.
code.push_back(EncodeSop1(0x04, 10, 126));
code.push_back(EncodeVopc(0xc2, InlineU32(0), 0));
code.push_back(EncodeSop1(0x04, 126, 106));
code.push_back(EncodeDs0(0x0d, 4));
code.push_back(EncodeDs1(0, 2, 2));
code.push_back(EncodeSop1(0x04, 126, 10));
code.push_back(EncodeSopp(0x0a, 0));
}
// Distinct masks in the wave64 halves also check inactive return
// preservation.
code.push_back(EncodeSop1(0x04, 10, 126));
AppendSMovLiteral(&code, 126, 0x55555555u);
AppendSMovLiteral(&code, 127, 0xaaaaaaaau);
code.push_back(EncodeDs0(decrement ? 0x24 : 0x23, 4, gds));
code.push_back(EncodeDs1(6, 3, 2));
// Returned old values form a histogram; no lane ordering is assumed.
code.push_back(EncodeVop2(0x1a, 7, InlineU32(2), 6));
AppendBufferStoreOpcode(&code, 0x32, 5, 7);
code.push_back(EncodeSop1(0x04, 126, 10));
AppendVMovLiteral(&code, 7, guard);
code.push_back(EncodeVopc(0xc2, Vgpr(7), 6));
code.push_back(EncodeVop2(0x01, 8, InlineU32(0), 5));
AppendStoreVgprAtLaneDwordOffset(&code, 8, 0, 16);
code.push_back(EncodeSopp(0x0a, 0));
// S_BARRIER orders LDS; use atomic add-zero to read GDS consistently.
code.push_back(EncodeDs0(gds ? 0x20 : 0x36, 4, gds));
code.push_back(EncodeDs1(9, 2, 2));
AppendStoreVgprAtLaneDwordOffset(&code, 9, 0, 144);
AppendEnd(&code);
test.expected.assign(16, 4); // 64 active lanes, four complete wraps.
for (u32 lane = 0; lane < 128; ++lane) {
const uint64_t mask = 0xaaaaaaaa55555555ull;
test.expected.push_back(
(mask & (uint64_t{1} << (lane % wave_size))) == 0 ? 1 : 0);
}
test.expected.resize(272, 0);
test.initial.resize(test.expected.size());
test.compute_info.threads_num[0] = 128;
test.compute_info.threads_num[1] = 1;
test.compute_info.threads_num[2] = 1;
test.compute_info.thread_ids_num = 1;
test.compute_info.wave_size = wave_size;
test.compute_info.lds_size_dwords = 3;
test.has_compute_info = true;
test.opcodes = {O::S_MOV_B32,
O::S_MOV_B64,
O::S_BARRIER,
O::V_MOV_B32,
O::V_LSHLREV_B32,
O::V_ADD_NC_U32,
O::V_CMP_EQ_U32,
O::V_CNDMASK_B32,
decrement ? O::DS_DEC_RTN_U32 : O::DS_INC_RTN_U32,
gds ? O::DS_ADD_RTN_U32 : O::DS_READ_B32,
O::BUFFER_ATOMIC_ADD,
O::BUFFER_STORE_DWORD,
O::S_ENDPGM};
test.required_spirv = {"OpAtomicLoad", "OpAtomicCompareExchange",
"OpLoopMerge"};
if (gds) {
test.gds_initial = {guard, 0, guard};
test.expected_gds = test.gds_initial;
} else
test.opcodes.push_back(O::DS_WRITE_B32);
return test;
}
TestCase DsMiscVariants() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 0, 0);
AppendVMovU32(&code, 1, 0);
AppendVMovLiteral(&code, 2, 0x40800000u);
code.push_back(EncodeDs0(0x0d, 0));
code.push_back(EncodeDs1(0, 2, 1));
AppendVMovLiteral(&code, 3, 0x3f800000u);
code.push_back(EncodeDs0(0x0d, 4));
code.push_back(EncodeDs1(0, 3, 1));
AppendVMovLiteral(&code, 4, 0x40000000u);
code.push_back(EncodeDs0(0x12, 0));
code.push_back(EncodeDs1(0, 4, 1));
AppendVMovLiteral(&code, 5, 0x40400000u);
code.push_back(EncodeDs0(0x13, 4));
code.push_back(EncodeDs1(0, 5, 1));
code.push_back(EncodeDs0(0x36, 0));
code.push_back(EncodeDs1(6, 0, 1));
code.push_back(EncodeDs0(0x36, 4));
code.push_back(EncodeDs1(7, 0, 1));
AppendVMovLiteral(&code, 8, 0x12345678u);
code.push_back(EncodeDs0(0x35, 0x001f));
code.push_back(EncodeDs1(9, 0, 8));
code.push_back(EncodeSMovB32(124, InlineU32(0)));
AppendVMovLiteral(&code, 10, 0xabcdef01u);
code.push_back(EncodeDs0(0xb0, 8));
code.push_back(EncodeDs1(0, 10, 0));
code.push_back(EncodeDs0(0xb1, 8));
code.push_back(EncodeDs1(11, 0, 0));
const u32 results[] = {6, 7, 9, 11};
for (u32 i = 0; i < static_cast<u32>(std::size(results)); i++) {
AppendStoreVgpr(&code, results[i], i);
}
AppendEnd(&code);
TestCase test;
test.name = "DsMiscVariants";
test.code = code;
test.initial = std::vector<u32>(4, 0);
test.expected = {0x40000000u, 0x40400000u, 0x12345678u, 0xabcdef01u};
test.opcodes = {O::V_MOV_B32,
O::S_MOV_B32,
O::DS_WRITE_B32,
O::DS_MIN_F32,
O::DS_MAX_F32,
O::DS_READ_B32,
O::DS_SWIZZLE_B32,
O::DS_WRITE_ADDTID_B32,
O::DS_READ_ADDTID_B32,
O::BUFFER_STORE_DWORD,
O::S_ENDPGM};
test.compute_info.threads_num[0] = 1;
test.compute_info.threads_num[1] = 1;
test.compute_info.threads_num[2] = 1;
test.has_compute_info = true;
return test;
}
TestCase DsFloatMinMaxIgnoresData1() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 1, 0);
AppendVMovLiteral(&code, 2, 0x40800000u);
code.push_back(EncodeDs0(0x0d, 0));
code.push_back(EncodeDs1(0, 2, 1));
AppendVMovLiteral(&code, 3, 0x40800000u);
code.push_back(EncodeDs0(0x0d, 4));
code.push_back(EncodeDs1(0, 3, 1));
AppendVMovLiteral(&code, 4, 0x41100000u);
AppendVMovLiteral(&code, 5, 0x40000000u);
code.push_back(EncodeDs0(0x12, 0));
code.push_back(EncodeDs1Ex(0, 5, 4, 1));
AppendVMovLiteral(&code, 6, 0x3f800000u);
AppendVMovLiteral(&code, 7, 0x40400000u);
code.push_back(EncodeDs0(0x13, 4));
code.push_back(EncodeDs1Ex(0, 7, 6, 1));
code.push_back(EncodeDs0(0x36, 0));
code.push_back(EncodeDs1(8, 0, 1));
code.push_back(EncodeDs0(0x36, 4));
code.push_back(EncodeDs1(9, 0, 1));
AppendStoreVgpr(&code, 8, 0);
AppendStoreVgpr(&code, 9, 1);
AppendEnd(&code);
TestCase test;
test.name = "DsFloatMinMaxIgnoresData1";
test.code = code;
test.initial = std::vector<u32>(2, 0);
test.expected = {0x40800000u, 0x40800000u};
test.opcodes = {O::V_MOV_B32, O::DS_WRITE_B32, O::DS_MIN_F32,
O::DS_MAX_F32, O::DS_READ_B32, O::BUFFER_STORE_DWORD,
O::S_ENDPGM};
test.compute_info.threads_num[0] = 1;
test.compute_info.threads_num[1] = 1;
test.compute_info.threads_num[2] = 1;
test.has_compute_info = true;
return test;
}
TestCase DsFloatMinMaxClasses(bool gds) {
using O = ShaderOpcode;
// Rows are memory, columns source: -inf, -normal, -denorm, -0, +0,
// +denorm, +normal, +inf, qNaN, sNaN. O/N select old/new; Q/R quiet them.
constexpr std::string_view minimum[] = {
"OOOOOOOOOR", "NNOOOOOOOR", "NNNNOOOOOR", "NNNOOOOOOR", "NNNNOOOOOR",
"NNNNNNOOOR", "NNNNNNOOOR", "NNNNNNNOOR", "NNNNNNNNOR", "QQQQQQQQQQ"};
constexpr std::string_view maximum[] = {
"ONNNNNNNOR", "OONNNNNNOR", "OOOONNNNOR", "OOOONNNNOR", "OOOOONNNOR",
"OOOOOONNOR", "OOOOOONNOR", "OOOOOOOOOR", "NNNNNNNNOR", "QQQQQQQQQQ"};
constexpr u32 old_values[] = {
0xff800000u, 0xc0000000u, 0x80000100u, 0x80000000u, 0u,
0x00000100u, 0x40000000u, 0x7f800000u, 0x7fc12345u, 0x7f812345u};
constexpr u32 sources[] = {
0xff800000u, 0xc0400000u, 0x80000001u, 0x80000000u, 0u,
0x00000001u, 0x40400000u, 0x7f800000u, 0xffc54321u, 0xff854321u};
const auto select = [](char choice, u32 old, u32 source) {
const u32 bits = choice == 'O' || choice == 'Q' ? old : source;
return choice == 'Q' || choice == 'R' ? bits | 0x00400000u : bits;
};
TestCase test;
test.name = gds ? "DsFloatMinMaxGdsClasses" : "DsFloatMinMaxLdsClasses";
for (u32 row = 0; row < std::size(old_values); ++row) {
for (u32 column = 0; column < std::size(sources); ++column) {
const auto old = old_values[row];
const auto source = sources[column];
test.initial.insert(test.initial.end(), {old, source, 0u, 0u});
test.expected.insert(test.expected.end(),
{old, source, select(minimum[row][column], old, source),
select(maximum[row][column], old, source)});
}
}
// Reversing positive denorm magnitudes still selects source for min and old for max.
test.initial.insert(test.initial.end(), {1u, 0x100u, 0u, 0u});
test.expected.insert(test.expected.end(), {1u, 0x100u, 0x100u, 1u});
const auto count = static_cast<u32>(test.initial.size() / 4u);
auto &code = test.code;
code.push_back(EncodeVop2(0x1a, 1, InlineU32(3), 0));
code.push_back(EncodeVop2(0x1a, 8, InlineU32(4), 0));
code.push_back(EncodeMubuf0(0x0d));
code.push_back(EncodeMubuf1(2, 0, 8));
for (u32 offset : {0u, 4u}) {
code.push_back(EncodeDs0(0x0d, offset, gds));
code.push_back(EncodeDs1(0, 2, 1));
}
code.push_back(EncodeDs0(0x12, 0, gds));
code.push_back(EncodeDs1Ex(0, 250, 3, 1)); // DATA1 is unused and uninitialized.
code.push_back(EncodeDs0(0x13, 4, gds));
code.push_back(EncodeDs1Ex(0, 251, 3, 1));
AppendVMovLiteral(&code, 6, 0xff800000u);
AppendVMovLiteral(&code, 7, 0x7f800000u);
code.push_back(EncodeSop1(0x04, 4, 126));
code.push_back(EncodeSop1(0x04, 126, InlineU32(0)));
// Inactive updates must leave the table results untouched.
code.push_back(EncodeDs0(0x12, 0, gds));
code.push_back(EncodeDs1(0, 6, 1));
code.push_back(EncodeDs0(0x13, 4, gds));
code.push_back(EncodeDs1(0, 7, 1));
code.push_back(EncodeSop1(0x04, 126, 4));
// Active accesses immediately beyond the allocation must also be discarded.
AppendVMovU32(&code, 9, count * 8u);
code.push_back(EncodeDs0(0x12, 0, gds));
code.push_back(EncodeDs1(0, 6, 9));
code.push_back(EncodeDs0(0x13, 0, gds));
code.push_back(EncodeDs1(0, 7, 9));
for (u32 component = 0; component < 2u; ++component) {
code.push_back(EncodeDs0(0x36, component * 4u, gds));
code.push_back(EncodeDs1(4u + component, 0, 1));
}
code.push_back(EncodeMubuf0(0x1d, 8));
code.push_back(EncodeMubuf1(4, 12, 8));
AppendEnd(&code);
test.compute_info.threads_num[0] = count;
test.compute_info.threads_num[1] = 1;
test.compute_info.threads_num[2] = 1;
test.compute_info.thread_ids_num = 1;
test.compute_info.lds_size_dwords = count * 2u;
test.has_compute_info = true;
test.opcodes = {O::V_MOV_B32, O::V_LSHLREV_B32, O::S_MOV_B64,
O::BUFFER_LOAD_DWORDX2, O::DS_WRITE_B32, O::DS_MIN_F32,
O::DS_MAX_F32, O::DS_READ_B32, O::BUFFER_STORE_DWORDX2,
O::S_ENDPGM};
test.required_spirv = {"OpAtomicLoad", "OpAtomicCompareExchange", "OpLoopMerge"};
test.forbidden_spirv = {"OpFOrdLessThan", "OpFOrdGreaterThan"};
if (gds) {
test.gds_initial.resize(count * 2u);
for (u32 i = 0; i < count; ++i) {
test.expected_gds.push_back(test.expected[i * 4u + 2u]);
test.expected_gds.push_back(test.expected[i * 4u + 3u]);
}
}
return test;
}
TestCase DsSwizzleInvalidSourceLaneZero() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 1, 100);
code.push_back(EncodeDs0(0x35, 0x00e0));
code.push_back(EncodeDs1(2, 0, 1));
code.push_back(EncodeVop2(0x1a, 3, InlineU32(2), 0));
AppendBufferStoreDword(&code, 2, 3);
AppendEnd(&code);
TestCase test;
test.name = "DsSwizzleInvalidSourceLaneZero";
test.code = code;
test.expected = {0, 0, 0, 0};
test.opcodes = {O::V_MOV_B32, O::DS_SWIZZLE_B32, O::V_LSHLREV_B32,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.compute_info.threads_num[0] = 4;
test.compute_info.threads_num[1] = 1;
test.compute_info.threads_num[2] = 1;
test.compute_info.thread_ids_num = 1;
test.has_compute_info = true;
return test;
}
TestCase DsPermuteCapturedExecOffsetAndWrap() {
using O = ShaderOpcode;
std::vector<u32> code;
code.push_back(EncodeVop2(0x1a, 30, InlineU32(2), 0));
AppendBufferLoadDword(&code, 10, 30);
AppendVMovLiteral(&code, 1, 0xdeadbeefu);
AppendVMovU32(&code, 9, 100);
code.push_back(EncodeVop2(0x25, 9, Vgpr(0), 9));
code.push_back(EncodeSop1(0x04, 4, 126));
code.push_back(EncodeVop2(0x1b, 18, InlineU32(1), 0));
code.push_back(EncodeVopc(0xc2, InlineU32(1), 18));
code.push_back(EncodeSop1(0x04, 126, 106));
code.push_back(0xdac80000u);
code.push_back(0x0100090au); // Captured SH2 instruction: ds_permute_b32 v1, v10, v9
code.push_back(EncodeSop1(0x04, 126, 4));
AppendStoreVgprAtLaneDwordOffset(&code, 1, 0, 4);
code.push_back(0xdac80000u);
code.push_back(0x0100090au);
AppendStoreVgprAtLaneDwordOffset(&code, 1, 0, 8);
AppendVMovU32(&code, 10, 127);
code.push_back(EncodeDs0(0xb2, 1));
code.push_back(EncodeDs1(1, 9, 10));
AppendStoreVgprAtLaneDwordOffset(&code, 1, 0, 12);
AppendEnd(&code);
TestCase test;
test.name = "DsPermuteCapturedExecOffsetAndWrap";
test.code = code;
test.initial = {0, 0, 12, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
test.expected = {0, 0, 12, 4, 0xdeadbeefu, 103, 0xdeadbeefu, 0,
101, 103, 0, 102, 103, 0, 0, 0};
test.opcodes = {O::V_LSHLREV_B32, O::BUFFER_LOAD_DWORD, O::V_MOV_B32,
O::V_ADD_NC_U32, O::V_AND_B32, O::V_CMP_EQ_U32, O::S_MOV_B64,
O::DS_PERMUTE_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.required_spirv = {"OpGroupNonUniformShuffle", "OpGroupNonUniformBallot"};
test.forbidden_spirv = {" Workgroup", "OpControlBarrier", "OpMemoryBarrier"};
test.decoded_counts = {{"DS_PERMUTE_B32", 3}};
test.ir_counts = {{"PermuteU32", 3}};
test.compute_info.threads_num[0] = 4;
test.compute_info.threads_num[1] = 1;
test.compute_info.threads_num[2] = 1;
test.compute_info.wave_size = 32;
test.compute_info.thread_ids_num = 1;
test.has_compute_info = true;
return test;
}
TestCase DsPermuteWave64UsesIndependentHalves() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 9, 100);
code.push_back(EncodeVop2(0x25, 9, Vgpr(0), 9));
code.push_back(EncodeVop2(0x1d, 10, InlineU32(31), 0));
code.push_back(EncodeVop2(0x1a, 10, InlineU32(2), 10));
code.push_back(0xdac80000u);
code.push_back(0x0100090au);
AppendStoreVgprAtLaneDwordOffset(&code, 1, 0, 0);
AppendEnd(&code);
TestCase test;
test.name = "DsPermuteWave64UsesIndependentHalves";
test.code = code;
test.initial = std::vector<u32>(64, 0);
for (u32 lane = 0; lane < 64; ++lane) {
test.expected.push_back(100 + (lane ^ 31));
}
test.opcodes = {O::V_MOV_B32, O::V_ADD_NC_U32, O::V_XOR_B32, O::V_LSHLREV_B32,
O::DS_PERMUTE_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.required_spirv = {"OpGroupNonUniformShuffle", "OpGroupNonUniformBallot"};
test.forbidden_spirv = {" Workgroup", "OpControlBarrier", "OpMemoryBarrier"};
test.compute_info.threads_num[0] = 64;
test.compute_info.threads_num[1] = 1;
test.compute_info.threads_num[2] = 1;
test.compute_info.wave_size = 64;
test.compute_info.thread_ids_num = 1;
test.has_compute_info = true;
return test;
}
TestCase DsBpermuteCapturedExecOffsetAndWrap() {
using O = ShaderOpcode;
std::vector<u32> code;
code.push_back(EncodeVop2(0x1a, 30, InlineU32(2), 0));
AppendBufferLoadDword(&code, 17, 30);
AppendVMovLiteral(&code, 1, 0xdeadbeefu);
AppendVMovU32(&code, 3, 100);
code.push_back(EncodeVop2(0x25, 3, Vgpr(0), 3));
code.push_back(EncodeSop1(0x04, 4, 126));
code.push_back(EncodeVop2(0x1b, 18, InlineU32(1), 0));
code.push_back(EncodeVopc(0xc2, InlineU32(1), 18));
code.push_back(EncodeSop1(0x04, 126, 106));
code.push_back(0xdacc0000u);
code.push_back(0x01000311u); // ds_bpermute_b32 v1, v17, v3
code.push_back(EncodeSop1(0x04, 126, 4));
AppendStoreVgprAtLaneDwordOffset(&code, 1, 0, 4);
AppendVMovU32(&code, 17, 124);
code.push_back(EncodeDs0(0xb3, 4));
code.push_back(EncodeDs1(2, 3, 17));
AppendStoreVgprAtLaneDwordOffset(&code, 2, 0, 8);
AppendEnd(&code);
TestCase test;
test.name = "DsBpermuteCapturedExecOffsetAndWrap";
test.code = code;
test.initial = {0, 0, 12, 4, 0, 0, 0, 0, 0, 0, 0, 0};
test.expected = {0, 0, 12, 4,
0xdeadbeefu, 0, 0xdeadbeefu, 101,
100, 100, 100, 100};
test.opcodes = {O::V_LSHLREV_B32, O::BUFFER_LOAD_DWORD,
O::V_MOV_B32, O::V_ADD_NC_U32,
O::V_AND_B32, O::V_CMP_EQ_U32,
O::S_MOV_B64,
O::DS_BPERMUTE_B32, O::BUFFER_STORE_DWORD,
O::S_ENDPGM};
test.required_spirv = {"OpGroupNonUniformShuffle",
"OpGroupNonUniformBallot"};
test.forbidden_spirv = {" Workgroup", "OpControlBarrier",
"OpMemoryBarrier"};
test.decoded_counts = {{"DS_BPERMUTE_B32", 2}};
test.ir_counts = {{"BpermuteU32", 2}};
test.compute_info.threads_num[0] = 4;
test.compute_info.threads_num[1] = 1;
test.compute_info.threads_num[2] = 1;
test.compute_info.wave_size = 32;
test.compute_info.thread_ids_num = 1;
test.has_compute_info = true;
return test;
}
TestCase DsBpermuteWave64UsesIndependentHalves() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 3, 100);
code.push_back(EncodeVop2(0x25, 3, Vgpr(0), 3));
AppendVMovU32(&code, 17, 128);
code.push_back(0xdacc0000u);
code.push_back(0x01000311u); // ds_bpermute_b32 v1, v17, v3
AppendStoreVgprAtLaneDwordOffset(&code, 1, 0, 0);
AppendEnd(&code);
TestCase test;
test.name = "DsBpermuteWave64UsesIndependentHalves";
test.code = code;
test.initial = std::vector<u32>(64, 0);
test.expected = std::vector<u32>(32, 100);
test.expected.insert(test.expected.end(), 32, 132);
test.opcodes = {O::V_MOV_B32, O::V_ADD_NC_U32,
O::DS_BPERMUTE_B32, O::V_LSHLREV_B32,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.required_spirv = {"OpGroupNonUniformShuffle"};
test.compute_info.threads_num[0] = 64;
test.compute_info.threads_num[1] = 1;
test.compute_info.threads_num[2] = 1;
test.compute_info.wave_size = 64;
test.compute_info.thread_ids_num = 1;
test.has_compute_info = true;
return test;
}
TestCase Wave64CrossHalfLaneAndLds() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 1, 100);
code.push_back(EncodeVop2(0x25, 1, Vgpr(0), 1));
AppendVop3(&code, 0x360, 8, Vgpr(1), InlineU32(47));
AppendStoreSgprAtLaneDwordOffset(&code, 8, 0, 0);
code.push_back(EncodeSop1(0x04, 10, 126));
code.push_back(EncodeVop2(0x1b, 2, InlineU32(63), 0));
code.push_back(EncodeVopc(0xc2, InlineU32(40), 2));
code.push_back(EncodeSop1(0x04, 126, 106));
code.push_back(EncodeVop1(0x02, 9, Vgpr(1)));
code.push_back(EncodeSop1(0x04, 126, 10));
AppendStoreSgprAtLaneDwordOffset(&code, 9, 0, 128);
AppendStoreSgprAtLaneDwordOffset(&code, 106, 0, 256);
AppendStoreSgprAtLaneDwordOffset(&code, 107, 0, 384);
code.push_back(EncodeVop2(0x1a, 2, InlineU32(2), 0));
code.push_back(EncodeDs0(0x0d));
code.push_back(EncodeDs1(0, 1, 2));
code.push_back(EncodeSopp(0x0a, 0));
code.push_back(EncodeVop2(0x1d, 3, InlineU32(64), 0));
code.push_back(EncodeVop2(0x1a, 3, InlineU32(2), 3));
code.push_back(EncodeDs0(0x36));
code.push_back(EncodeDs1(4, 0, 3));
AppendStoreVgprAtLaneDwordOffset(&code, 4, 0, 512);
AppendEnd(&code);
TestCase test;
test.name = "Wave64CrossHalfLaneAndLds";
test.code = std::move(code);
test.initial.resize(640);
test.expected.insert(test.expected.end(), 64, 147);
test.expected.insert(test.expected.end(), 64, 211);
test.expected.insert(test.expected.end(), 64, 140);
test.expected.insert(test.expected.end(), 64, 204);
test.expected.insert(test.expected.end(), 128, 0);
test.expected.insert(test.expected.end(), 128, 256);
for (u32 lane = 0; lane < 128; lane++) {
test.expected.push_back(100 + (lane ^ 64));
}
test.opcodes = {O::V_MOV_B32, O::V_ADD_NC_U32, O::V_READLANE_B32,
O::V_READFIRSTLANE_B32, O::V_CMP_EQ_U32, O::S_MOV_B64,
O::V_LSHLREV_B32, O::V_AND_B32, O::V_XOR_B32, O::DS_WRITE_B32,
O::S_BARRIER, O::DS_READ_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.required_spirv = {"OpGroupNonUniformBallot", "OpGroupNonUniformShuffle",
"OpControlBarrier"};
test.ir_counts = {{"Barrier", 1}};
test.compute_info.threads_num[0] = 128;
test.compute_info.threads_num[1] = 1;
test.compute_info.threads_num[2] = 1;
test.compute_info.thread_ids_num = 1;
test.has_compute_info = true;
return test;
}
void CheckWave64WholeWaveResults() {
const auto test = Wave64CrossHalfLaneAndLds();
const auto native = CompileCase(test, 64);
const auto emulated = CompileCase(test, 32);
spvtools::SpirvTools tools(SPV_ENV_VULKAN_1_2);
std::string native_text, emulated_text;
Require(test.name, "whole-wave disassembly",
tools.Disassemble(native.spirv, &native_text) &&
tools.Disassemble(emulated.spirv, &emulated_text),
"failed to disassemble whole-wave operations");
const auto ballots = CountText(native_text, "OpGroupNonUniformBallot ");
Require(test.name, "whole-wave ballot reuse",
ballots != 0 &&
CountText(emulated_text, "OpGroupNonUniformBallot ") == ballots * 2,
"emulated halves repeated a whole-wave ballot");
// READLANE still emits two shuffles per half; READFIRSTLANE needs one shared pair.
Require(test.name, "whole-wave first-lane reuse",
CountText(emulated_text, "OpGroupNonUniformShuffle ") == 6,
"emulated halves repeated the first-active-lane broadcast");
}
TestCase Wave64RawMasksAndScalarBranch() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 1, 7);
AppendSMovLiteral(&code, 8, 33u << 16u);
code.push_back(EncodeSop2(0x29, 126, 193u, 8)); // captured ES BFE EXEC,-1,width
AppendStoreVgprAtLaneDwordOffset(&code, 1, 0, 0);
code.push_back(EncodeSop2(0x1f, 126, InlineU32(1), InlineU32(32)));
const auto exec_branch = code.size();
code.push_back(0);
code.push_back(EncodeSopp(0x0a, 0));
code.push_back(EncodeSop1(0x24, 10, 193u)); // s_and_saveexec_b64 s[10:11],-1
const auto scc_branch = code.size();
code.push_back(0);
AppendVMovU32(&code, 1, 9);
AppendStoreVgprAtLaneDwordOffset(&code, 1, 0, 64);
const auto restore = code.size();
code[exec_branch] = EncodeSopp(0x08, restore - exec_branch - 1);
code[scc_branch] = EncodeSopp(0x04, restore - scc_branch - 1);
code.push_back(EncodeSop1(0x04, 126, 193u));
code.push_back(EncodeVopc(0xc2, Vgpr(0), 0));
code.push_back(EncodeSop1(0x04, 12, 106));
code.push_back(EncodeSMovB32(13, InlineU32(1)));
AppendVop3(&code, 0x101, 2, InlineU32(0), InlineU32(11), 12);
AppendStoreVgprAtLaneDwordOffset(&code, 2, 0, 128);
code.push_back(EncodeSMovB32(127, InlineU32(1)));
AppendVMovU32(&code, 3, 13);
AppendStoreVgprAtLaneDwordOffset(&code, 3, 0, 192);
code.push_back(EncodeSop1(0x04, 126, InlineU32(1)));
AppendStoreVgprAtLaneDwordOffset(&code, 3, 0, 256);
code.push_back(EncodeSMovB32(107, InlineU32(2)));
code.push_back(EncodeSMovB32(127, 107));
AppendStoreVgprAtLaneDwordOffset(&code, 1, 0, 320);
AppendEnd(&code);
TestCase test;
test.name = "Wave64RawMasksAndScalarBranch";
test.code = std::move(code);
test.initial.assign(384, 0xdeadbeef);
test.expected = test.initial;
for (u32 lane = 0; lane < 64; lane++) {
if (lane < 33) {
test.expected[lane] = 7;
test.expected[192 + lane] = 13;
}
test.expected[128 + lane] = lane < 33 ? 11 : 0;
}
test.expected[64 + 32] = 9;
test.expected[256] = 13;
test.expected[320] = 7;
test.expected[320 + 33] = 7;
test.opcodes = {O::V_MOV_B32, O::S_MOV_B32, O::S_BFE_U64, O::S_LSHL_B64,
O::S_CBRANCH_EXECZ, O::S_AND_SAVEEXEC_B64, O::S_CBRANCH_SCC0,
O::S_BARRIER, O::S_MOV_B64, O::V_CMP_EQ_U32, O::V_CNDMASK_B32,
O::V_LSHLREV_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.compute_info.threads_num[0] = 64;
test.compute_info.threads_num[1] = 1;
test.compute_info.threads_num[2] = 1;
test.compute_info.thread_ids_num = 1;
test.has_compute_info = true;
return test;
}
TestCase Wave64PartialMultidimensionalWorkgroup() {
using O = ShaderOpcode;
std::vector<u32> code;
code.push_back(EncodeVop2(0x25, 3, Vgpr(2), 2));
code.push_back(EncodeVop2(0x25, 3, Vgpr(2), 3));
code.push_back(EncodeVop2(0x25, 3, Vgpr(1), 3));
code.push_back(EncodeVop2(0x1a, 3, InlineU32(3), 3));
code.push_back(EncodeVop2(0x25, 3, Vgpr(0), 3));
code.push_back(EncodeVop2(0x1a, 4, InlineU32(2), 3));
AppendVMovU32(&code, 5, 1);
AppendBufferStoreOpcode(&code, 0x32, 5, 4, false);
AppendEnd(&code);
TestCase test;
test.name = "Wave64PartialMultidimensionalWorkgroup";
test.code = std::move(code);
test.initial.resize(128);
test.expected.assign(96, 1);
test.expected.resize(128);
test.opcodes = {O::V_ADD_NC_U32, O::V_LSHLREV_B32, O::V_MOV_B32,
O::BUFFER_ATOMIC_ADD, O::S_ENDPGM};
test.compute_info.threads_num[0] = 8;
test.compute_info.threads_num[1] = 3;
test.compute_info.threads_num[2] = 4;
test.compute_info.thread_ids_num = 3;
test.has_compute_info = true;
return test;
}
TestCase Wave64AppendConsumeHighHalf() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendSMovLiteral(&code, 124, 4);
code.push_back(EncodeSop2(0x1f, 126, 193u, InlineU32(32)));
code.push_back(EncodeDs0(0x3e, 0, true));
code.push_back(EncodeDs1(1, 0, 0));
AppendStoreVgprAtLaneDwordOffset(&code, 1, 0, 0);
code.push_back(EncodeDs0(0x3d, 0, true));
code.push_back(EncodeDs1(2, 0, 0));
AppendStoreVgprAtLaneDwordOffset(&code, 2, 0, 64);
AppendEnd(&code);
TestCase test;
test.name = "Wave64AppendConsumeHighHalf";
test.code = std::move(code);
test.initial.assign(128, 0xdeadbeef);
test.expected = test.initial;
for (u32 lane = 32; lane < 64; lane++) {
test.expected[lane] = 10;
test.expected[64 + lane] = 42;
}
test.gds_initial = {10};
test.expected_gds = {10};
test.opcodes = {O::S_MOV_B32, O::S_LSHL_B64, O::DS_APPEND, O::DS_CONSUME,
O::V_LSHLREV_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.compute_info.threads_num[0] = 64;
test.compute_info.threads_num[1] = 1;
test.compute_info.threads_num[2] = 1;
test.compute_info.thread_ids_num = 1;
test.has_compute_info = true;
return test;
}
TestCase BufferWorkgroupPublication(u32 wave_size, bool dlc_only = false,
bool store_completion = false) {
using O = ShaderOpcode;
constexpr u32 groups = 31;
TestCase test;
test.name = store_completion ? (wave_size == 64 ? "Wave64StoreCompletionPublication"
: "Wave32StoreCompletionPublication")
: dlc_only ? "BufferWorkgroupPublicationDlc"
: wave_size == 64 ? "BufferWorkgroupPublicationWave64"
: "BufferWorkgroupPublicationWave32";
auto &code = test.code;
// Allocate work in execution order so every polled predecessor has started.
// Only one lane per workgroup owns a ticket and publishes its prefix total.
code.push_back(EncodeSop1(0x04, 126, InlineU32(1)));
AppendVMovU32(&code, 1, 1);
AppendVMovU32(&code, 20, 0);
AppendBufferStoreOpcode(&code, 0x32, 1, 20, true);
code.push_back(EncodeVop1(0x02, 20, Vgpr(1)));
code.push_back(EncodeVop2(0x1a, 20, InlineU32(2), 1));
AppendVMovU32(&code, 2, 0);
code.push_back(EncodeSMovB32(21, InlineU32(0)));
AppendSMovLiteral(&code, 22, 1000000);
if (store_completion) AppendSMovLiteral(&code, 26, groups * 4);
code.push_back(EncodeSopc(0x06, 20, InlineU32(0)));
const auto first_ticket = code.size();
code.push_back(0);
const auto poll = code.size();
if (store_completion) {
code.push_back(EncodeVop2(0x25, 4, 26, 20));
AppendVMovU32(&code, 3, 0);
AppendBufferStoreOpcode(&code, 0x32, 3, 4, true);
}
code.push_back(EncodeMubuf0(0x0c, 0, false, true, !dlc_only) | (1u << 15u));
code.push_back(EncodeMubuf1(2, 12, 20));
code.push_back(EncodeSopp(0x0c, 0)); // S_WAITCNT vmcnt(0).
code.push_back(EncodeVop1(0x02, 23, Vgpr(store_completion ? 3 : 2)));
code.push_back(EncodeSopc(0x07, 23, InlineU32(0)));
const auto ready = code.size();
code.push_back(0);
code.push_back(EncodeSop2(0x00, 21, 21, InlineU32(1)));
code.push_back(EncodeSopc(0x0a, 21, 22));
code.push_back(EncodeSopp(0x05, static_cast<u32>(poll - code.size() - 1)));
// A broken poll reports an incorrect result instead of hanging the test GPU.
AppendVMovLiteral(&code, 2, 0x80000000u);
const auto publish = code.size();
code[first_ticket] = EncodeSopp(0x05, publish - first_ticket - 1);
code[ready] = EncodeSopp(0x05, publish - ready - 1);
code.push_back(EncodeVop2(0x25, 2, Vgpr(1), 2));
code.push_back(EncodeVop2(0x25, 2, InlineU32(1), 2));
code.push_back(EncodeVop2(0x25, 20, InlineU32(4), 20));
if (store_completion) {
// The last lane writes the payload; lane zero publishes readiness after the
// native wave-wide store completion, including both emulated wave64 halves.
code.push_back(EncodeVop1(0x02, 24, Vgpr(2)));
code.push_back(EncodeVop1(0x02, 25, Vgpr(20)));
code.push_back(EncodeSop1(0x04, 126, 193u));
code.push_back(EncodeVop1(0x01, 2, 24));
code.push_back(EncodeVop1(0x01, 20, 25));
code.push_back(EncodeSop2(0x1f, 126, InlineU32(1), InlineU32(wave_size - 1)));
}
// RDNA2 stores publish to L2 even without GLC/DLC on the producer.
AppendBufferStoreDword(&code, 2, 20);
if (store_completion) {
code.push_back(0xbbfd0000u); // Captured S_WAITCNT_VSCNT null, 0 at 5b10:0x2464.
code.push_back(EncodeSop1(0x04, 126, InlineU32(1)));
code.push_back(EncodeVop2(0x25, 4, 26, 20));
AppendVMovU32(&code, 3, 1);
AppendBufferStoreOpcode(&code, 0x32, 3, 4, true);
}
AppendEnd(&code);
test.initial.assign(groups * (store_completion ? 2 : 1) + 1, 0);
test.expected = {groups};
for (u32 ticket = 0; ticket < groups; ++ticket) {
test.expected.push_back((ticket + 1) * (ticket + 2) / 2);
}
if (store_completion) test.expected.resize(groups * 2 + 1, 1);
test.opcodes = {O::S_MOV_B64, O::S_MOV_B32, O::V_MOV_B32,
O::BUFFER_ATOMIC_ADD, O::V_READFIRSTLANE_B32,
O::V_LSHLREV_B32, O::S_CMP_EQ_U32, O::S_CMP_LG_U32,
O::S_CMP_LT_U32, O::S_CBRANCH_SCC1, O::S_ADD_U32,
O::S_WAITCNT, O::BUFFER_LOAD_DWORD, O::V_ADD_NC_U32,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.required_spirv = {"Coherent", "Volatile", "OpAtomicIAdd", "OpLoopMerge"};
if (store_completion) {
test.opcodes.push_back(O::S_WAITCNT_VSCNT);
test.opcodes.push_back(O::S_LSHL_B64);
test.required_spirv.push_back("OpControlBarrier %uint_3 %uint_1 %uint_2116");
}
test.compute_info.threads_num[0] = wave_size;
test.compute_info.threads_num[1] = 1;
test.compute_info.threads_num[2] = 1;
test.compute_info.thread_ids_num = 1;
test.compute_info.wave_size = wave_size;
test.has_compute_info = true;
test.dispatch_x = groups;
// DLC alone does not bypass RDNA2's L0, so only check its translated policy.
test.compile_only = dlc_only;
return test;
}
TestCase BufferAtomicVariants() {
using O = ShaderOpcode;
std::vector<u32> code;
const u32 values[] = {100, 5, 3, 5, 5, 5, 20, 0x0ff0u, 0x0f00u, 0x00ffu};
const u32 ops[] = {0x30, 0x32, 0x33, 0x35, 0x36,
0x37, 0x38, 0x39, 0x3a, 0x3b};
for (u32 i = 0; i < static_cast<u32>(std::size(values)); i++) {
AppendVMovU32(&code, 20, i * 4u);
AppendVMovU32(&code, i, values[i]);
AppendBufferStoreOpcode(&code, ops[i], i, 20, true);
}
for (u32 i = 0; i < static_cast<u32>(std::size(values)); i++) {
AppendStoreVgpr(&code, i, i + 10u);
}
AppendEnd(&code);
return {
"BufferAtomicVariants",
code,
{10, 10, 10, 0xfffffff0u, 10, 0xfffffff0u, 10, 0xf0f0u, 0xf000u, 0xf00fu,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0},
{100, 15, 7, 0xfffffff0u, 5, 5, 20,
0x00f0u, 0xff00u, 0xf0f0u, 10, 10, 10, 0xfffffff0u,
10, 0xfffffff0u, 10, 0xf0f0u, 0xf000u, 0xf00fu},
{O::V_MOV_B32, O::BUFFER_ATOMIC_SWAP, O::BUFFER_ATOMIC_ADD,
O::BUFFER_ATOMIC_SUB, O::BUFFER_ATOMIC_SMIN, O::BUFFER_ATOMIC_UMIN,
O::BUFFER_ATOMIC_SMAX, O::BUFFER_ATOMIC_UMAX, O::BUFFER_ATOMIC_AND,
O::BUFFER_ATOMIC_OR, O::BUFFER_ATOMIC_XOR, O::BUFFER_STORE_DWORD,
O::S_ENDPGM}};
}
TestCase BufferAtomicAndX2GlcAndExec() {
using O = ShaderOpcode;
TestCase test;
test.name = "BufferAtomicAndX2GlcAndExec";
auto &code = test.code;
code.push_back(EncodeSop1(0x04, 12, 126));
AppendVMovU32(&code, 20, 0);
const u32 values[][2] = {{0x0ff00ff0u, 0xf00ff00fu},
{0xff00ff00u, 0x00ff00ffu},
{0, 0}};
for (u32 i = 0; i < std::size(values); ++i) {
AppendVMovLiteral(&code, 2, values[i][0]);
AppendVMovLiteral(&code, 3, values[i][1]);
if (i == 2) {
code.push_back(EncodeSop1(0x04, 126, InlineU32(0)));
}
AppendBufferStoreOpcode(&code, 0x59, 2, 20, i != 1);
code.push_back(EncodeSop1(0x04, 126, 12));
AppendStoreVgpr(&code, 2, 2 + i * 2);
AppendStoreVgpr(&code, 3, 3 + i * 2);
}
AppendEnd(&code);
test.initial = {0x12345678u, 0x87654321u, 0, 0, 0, 0, 0, 0};
test.expected = {0x02000600u, 0x00050001u, 0x12345678u, 0x87654321u,
0xff00ff00u, 0x00ff00ffu, 0, 0};
test.opcodes = {O::V_MOV_B32, O::S_MOV_B64, O::BUFFER_ATOMIC_AND_X2,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.required_spirv = {"OpAtomicAnd %ulong", "Int64Atomics"};
return test;
}
TestCase BufferAtomicAddX2GlcAndExec() {
using O = ShaderOpcode;
TestCase test;
test.name = "BufferAtomicAddX2GlcAndExec";
auto &code = test.code;
code.push_back(EncodeSop1(0x04, 12, 126));
AppendVMovU32(&code, 20, 0);
const u32 values[][2] = {{0x0ff00ff0u, 0xf00ff00fu},
{0xff00ff00u, 0x00ff00ffu},
{0x76543210u, 0xfedcba98u}};
for (u32 i = 0; i < std::size(values); ++i) {
AppendVMovLiteral(&code, 2, values[i][0]);
AppendVMovLiteral(&code, 3, values[i][1]);
if (i == 2) {
code.push_back(EncodeSop1(0x04, 126, InlineU32(0)));
}
AppendBufferStoreOpcode(&code, 0x52, 2, 20, i != 1);
code.push_back(EncodeSop1(0x04, 126, 12));
AppendStoreVgpr(&code, 2, 2 + i * 2);
AppendStoreVgpr(&code, 3, 3 + i * 2);
}
AppendEnd(&code);
test.initial = {0x12345678u, 0x87654321u, 0, 0, 0, 0, 0, 0};
// The first add wraps at 64 bits; the second carries from the low dword.
// GLC returns the first old value, preserves the second operand, and EXEC = 0
// suppresses the third add and its writeback despite its nonzero operand.
test.expected = {0x21256568u, 0x78743430u, 0x12345678u, 0x87654321u,
0xff00ff00u, 0x00ff00ffu, 0x76543210u, 0xfedcba98u};
test.opcodes = {O::V_MOV_B32, O::S_MOV_B64, O::BUFFER_ATOMIC_ADD_X2,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.required_spirv = {"OpAtomicIAdd %ulong", "Int64Atomics"};
return test;
}
TestCase BufferAtomicCmpSwapExactRaw() {
using O = ShaderOpcode;
std::vector<u32> code;
const auto Store = [&](u32 value, u32 index) {
AppendVMovU32(&code, 31, index * sizeof(u32));
code.push_back(EncodeMubuf0(0x1cu));
code.push_back(EncodeMubuf1(value, 8, 31));
};
AppendVMovU32(&code, 4, 1);
AppendVMovLiteral(&code, 1, 0x22222222u);
AppendVMovLiteral(&code, 2, 0x11111111u);
code.push_back(0xe0c46000u);
code.push_back(0x80080104u);
Store(1, 2);
AppendVMovLiteral(&code, 1, 0x33333333u);
AppendVMovLiteral(&code, 2, 0x44444444u);
code.push_back(0xe0c46000u);
code.push_back(0x80080104u);
Store(1, 3);
Store(2, 4);
AppendVMovLiteral(&code, 1, 0x55555555u);
AppendVMovLiteral(&code, 2, 0x22222222u);
code.push_back(EncodeMubuf0(0x31u, 0, true, false));
code.push_back(0x80080104u);
Store(1, 5);
AppendEnd(&code);
TestCase test;
test.name = "BufferAtomicCmpSwapExactRaw";
test.code = std::move(code);
test.initial = {0xaaaaaaaau, 0x11111111u, 0, 0, 0, 0};
test.expected = {0xaaaaaaaau, 0x55555555u, 0x11111111u,
0x22222222u, 0x44444444u, 0x55555555u};
test.opcodes = {O::V_MOV_B32, O::BUFFER_ATOMIC_CMPSWAP,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.required_spirv = {"OpAtomicCompareExchange"};
const auto descriptor = MakeStructuredStorageBufferData(
sizeof(u32), static_cast<u32>(test.initial.size()));
std::copy_n(descriptor.begin(), 4, test.user_data.begin() + 32);
test.has_user_data = true;
return test;
}
TestCase BufferAtomicGlc0DoesNotReturnOldValue() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 20, 0);
AppendVMovU32(&code, 0, 5);
AppendBufferStoreOpcode(&code, 0x32, 0, 20);
AppendStoreVgpr(&code, 0, 1);
AppendEnd(&code);
return {
"BufferAtomicGlc0DoesNotReturnOldValue",
code,
{10, 0},
{15, 5},
{O::V_MOV_B32, O::BUFFER_ATOMIC_ADD, O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}
TestCase BufferAtomicFMinExactRawGlcModes() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovLiteral(&code, 0, 0x40000000u); // 2.0
code.push_back(0xe0fc0000u);
code.push_back(0x80010000u); // exact failing buffer_atomic_fmin, GLC=0
AppendStoreVgpr(&code, 0, 1);
AppendVMovLiteral(&code, 0, 0x3f800000u); // 1.0
code.push_back(0xe0fc4000u);
code.push_back(0x80010000u); // same instruction with GLC=1
AppendStoreVgpr(&code, 0, 2);
AppendEnd(&code);
TestCase test;
test.name = "BufferAtomicFMinExactRawGlcModes";
test.code = code;
test.initial = {0x40800000u, 0, 0}; // 4.0
test.expected = {0x3f800000u, 0x40000000u, 0x40000000u};
test.opcodes = {O::V_MOV_B32, O::BUFFER_ATOMIC_FMIN, O::BUFFER_STORE_DWORD,
O::S_ENDPGM};
const auto descriptor = MakeStructuredStorageBufferData(
0, static_cast<u32>(test.initial.size() * sizeof(u32)));
std::copy_n(descriptor.begin(), 4, test.user_data.begin() + 4);
test.user_data[50] = 1u << 20u;
test.has_user_data = true;
return test;
}
TestCase BufferAtomicFMinSpecialValues() {
using O = ShaderOpcode;
const u32 values[] = {
0x40000000u, // 2.0
0x7f800000u, // +infinity
0xff800000u, // -infinity
0x3f800000u, // 1.0
0x7fc00000u, // quiet NaN
0x00000000u, // +0.0
0x80000000u, // -0.0
0x00000000u, // +0.0
0x80000001u, // smallest negative denorm
};
std::vector<u32> code;
for (u32 i = 0; i < static_cast<u32>(std::size(values)); i++) {
AppendVMovU32(&code, 20, i * 4u);
AppendVMovLiteral(&code, i, values[i]);
AppendBufferStoreOpcode(&code, 0x3f, i, 20, true);
}
for (u32 i = 0; i < static_cast<u32>(std::size(values)); i++) {
AppendStoreVgpr(&code, i, i + static_cast<u32>(std::size(values)));
}
AppendEnd(&code);
return {"BufferAtomicFMinSpecialValues",
code,
{0x40800000u, 0xbf800000u, 0x7f800000u, 0x7fc00000u, 0x3f800000u,
0x80000000u, 0x00000000u, 0x00000001u, 0x00000000u, 0, 0, 0, 0, 0, 0,
0, 0, 0},
{0x40000000u, 0xbf800000u, 0xff800000u, 0x7fc00000u, 0x3f800000u,
0x80000000u, 0x00000000u, 0x00000000u, 0x80000001u, 0x40800000u,
0xbf800000u, 0x7f800000u, 0x7fc00000u, 0x3f800000u, 0x80000000u,
0x00000000u, 0x00000001u, 0x00000000u},
{O::V_MOV_B32, O::BUFFER_ATOMIC_FMIN, O::BUFFER_STORE_DWORD,
O::S_ENDPGM}};
}
TestCase BufferAtomicFMinContendedWorkgroup() {
using O = ShaderOpcode;
std::vector<u32> code;
code.push_back(EncodeVop1(0x06, 1, Vgpr(0))); // v_cvt_f32_u32 v1, thread_id.x
AppendVMovU32(&code, 20, 0);
AppendBufferStoreOpcode(&code, 0x3f, 1, 20);
AppendEnd(&code);
TestCase test;
test.name = "BufferAtomicFMinContendedWorkgroup";
test.code = code;
test.initial = {0x42c80000u}; // 100.0
test.expected = {0x00000000u}; // min(100.0, 0.0 .. 63.0)
test.opcodes = {O::V_CVT_F32_U32, O::V_MOV_B32, O::BUFFER_ATOMIC_FMIN,
O::S_ENDPGM};
test.compute_info.threads_num[0] = 64;
test.compute_info.threads_num[1] = 1;
test.compute_info.threads_num[2] = 1;
test.compute_info.thread_ids_num = 1;
test.has_compute_info = true;
return test;
}
TestCase BufferAtomicFMaxExactRawGlcModes() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovLiteral(&code, 3, 0x40000000u); // 2.0
code.push_back(0xe100000cu);
code.push_back(
0x80010300u); // exact failing buffer_atomic_fmax v3, offset 12, GLC=0
AppendStoreVgpr(&code, 3, 4);
AppendVMovLiteral(&code, 0, 0x40800000u); // 4.0
code.push_back(0xe100400cu);
code.push_back(0x80010000u); // same address with GLC=1
AppendStoreVgpr(&code, 0, 5);
AppendEnd(&code);
TestCase test;
test.name = "BufferAtomicFMaxExactRawGlcModes";
test.code = code;
test.initial = {0, 0, 0, 0x3f800000u, 0, 0}; // memory[3] = 1.0
test.expected = {0, 0, 0, 0x40800000u, 0x40000000u, 0x40000000u};
test.opcodes = {O::V_MOV_B32, O::BUFFER_ATOMIC_FMAX, O::BUFFER_STORE_DWORD,
O::S_ENDPGM};
const auto descriptor = MakeStructuredStorageBufferData(
0, static_cast<u32>(test.initial.size() * sizeof(u32)));
std::copy_n(descriptor.begin(), 4, test.user_data.begin() + 4);
test.user_data[50] = 1u << 20u;
test.has_user_data = true;
return test;
}
TestCase BufferAtomicFMaxSpecialValues() {
using O = ShaderOpcode;
const u32 values[] = {
0x40000000u, // 2.0, finite update
0x40000000u, // 2.0, finite no-update
0x7f800000u, // incoming +infinity
0xff800000u, // incoming -infinity
0x3f800000u, // finite against old +infinity
0x3f800000u, // finite against old -infinity
0x3f800000u, // finite against old quiet NaN
0x7fcabcdeu, // incoming quiet NaN
0x3f800000u, // finite against old signaling NaN
0x7faabcdeu, // incoming signaling NaN
0x00000000u, // +0.0 against old -0.0
0x80000000u, // -0.0 against old +0.0
0x80000000u, // -0.0 against a negative denorm
0x80000001u, // negative denorm against -0.0
0x00000001u, // positive denorm against +0.0
0x00000000u, // +0.0 against a positive denorm
};
std::vector<u32> code;
for (u32 i = 0; i < static_cast<u32>(std::size(values)); i++) {
AppendVMovU32(&code, 20, i * 4u);
AppendVMovLiteral(&code, i, values[i]);
AppendBufferStoreOpcode(&code, 0x40, i, 20, true);
}
for (u32 i = 0; i < static_cast<u32>(std::size(values)); i++) {
AppendStoreVgpr(&code, i, i + static_cast<u32>(std::size(values)));
}
AppendEnd(&code);
return {"BufferAtomicFMaxSpecialValues",
code,
{0x3f800000u, 0x40800000u, 0xbf800000u, 0x3f800000u,
0x7f800000u, 0xff800000u, 0x7fc12345u, 0x3f800000u,
0x7fa54321u, 0x3f800000u, 0x80000000u, 0x00000000u,
0x80000001u, 0x80000000u, 0x00000000u, 0x00000001u,
0, 0, 0, 0,
0, 0, 0, 0,
0, 0, 0, 0,
0, 0, 0, 0},
{0x40000000u, 0x40800000u, 0x7f800000u, 0x3f800000u, 0x7f800000u,
0x3f800000u, 0x7fc12345u, 0x3f800000u, 0x7fa54321u, 0x3f800000u,
0x80000000u, 0x00000000u, 0x80000000u, 0x80000000u, 0x00000001u,
0x00000001u, 0x3f800000u, 0x40800000u, 0xbf800000u, 0x3f800000u,
0x7f800000u, 0xff800000u, 0x7fc12345u, 0x3f800000u, 0x7fa54321u,
0x3f800000u, 0x80000000u, 0x00000000u, 0x80000001u, 0x80000000u,
0x00000000u, 0x00000001u},
{O::V_MOV_B32, O::BUFFER_ATOMIC_FMAX, O::BUFFER_STORE_DWORD,
O::S_ENDPGM}};
}
TestCase BufferAtomicFMaxContendedWorkgroup() {
using O = ShaderOpcode;
std::vector<u32> code;
code.push_back(EncodeVop1(0x06, 1, Vgpr(0))); // v_cvt_f32_u32 v1, thread_id.x
AppendVMovU32(&code, 20, 0);
AppendBufferStoreOpcode(&code, 0x40, 1, 20);
AppendEnd(&code);
TestCase test;
test.name = "BufferAtomicFMaxContendedWorkgroup";
test.code = code;
test.initial = {0xc2c80000u}; // -100.0
test.expected = {0x427c0000u}; // max(-100.0, 0.0 .. 63.0)
test.opcodes = {O::V_CVT_F32_U32, O::V_MOV_B32, O::BUFFER_ATOMIC_FMAX,
O::S_ENDPGM};
test.compute_info.threads_num[0] = 64;
test.compute_info.threads_num[1] = 1;
test.compute_info.threads_num[2] = 1;
test.compute_info.thread_ids_num = 1;
test.has_compute_info = true;
return test;
}
std::vector<u32> MakeRgbaImage(u32 width, u32 height, u32 value = 0) {
return std::vector<u32>(static_cast<size_t>(width) * height * 4u, value);
}
void SetRgbaPixel(std::vector<u32> *image, u32 width, u32 x, u32 y, u32 r,
u32 g, u32 b, u32 a) {
const auto base = static_cast<size_t>((y * width + x) * 4u);
(*image)[base + 0] = r;
(*image)[base + 1] = g;
(*image)[base + 2] = b;
(*image)[base + 3] = a;
}
TestCase ImageLoadVariants() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 20, 2);
AppendVMovU32(&code, 21, 1);
AppendVMovU32(&code, 22, 0);
AppendVMovU32(&code, 23, 0);
code.push_back(EncodeMimg0(0x00, 0xf));
code.push_back(EncodeMimg1(0, 20));
code.push_back(EncodeMimg0(0x01, 0xf));
code.push_back(EncodeMimg1(4, 20));
AppendVMovU32(&code, 24, 0);
code.push_back(EncodeMimg0(0x0e, 0x1));
code.push_back(EncodeMimg1(8, 24));
for (u32 i = 0; i < 9u; i++) {
AppendStoreVgpr(&code, i, i);
}
AppendEnd(&code);
auto image = MakeRgbaImage(4, 4);
SetRgbaPixel(&image, 4, 2, 1, 0x3f800000u, 0x40000000u, 0x40400000u,
0x40800000u);
TestCase test;
test.name = "ImageLoadVariants";
test.code = code;
test.expected = {0x3f800000u, 0x40000000u, 0x40400000u,
0x40800000u, 0x3f800000u, 0x40000000u,
0x40400000u, 0x40800000u, 4};
test.opcodes = {O::V_MOV_B32, O::IMAGE_LOAD,
O::IMAGE_LOAD_MIP, O::IMAGE_GET_RESINFO,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.sampled_image_rgba = image;
return test;
}
TestCase DsAppendConsumeUsesEncodedLdsSelector() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendSMovLiteral(&code, 124, 0xffff0000u);
AppendVMovU32(&code, 1, 0);
AppendVMovU32(&code, 2, 10);
code.push_back(EncodeDs0(0x0d, 0));
code.push_back(EncodeDs1(0, 2, 1));
code.push_back(EncodeDs0(0x3e, 1));
code.push_back(EncodeDs1(3, 0, 0));
code.push_back(EncodeDs0(0x3d, 2));
code.push_back(EncodeDs1(4, 0, 0));
AppendSMovLiteral(&code, 124, 0);
code.push_back(EncodeDs0(0x3e, 3));
code.push_back(EncodeDs1(5, 0, 0));
code.push_back(EncodeDs0(0x36, 0));
code.push_back(EncodeDs1(6, 0, 1));
AppendStoreVgpr(&code, 3, 0);
AppendStoreVgpr(&code, 4, 1);
AppendStoreVgpr(&code, 5, 2);
AppendStoreVgpr(&code, 6, 3);
AppendEnd(&code);
return {"DsAppendConsumeLdsSelector",
code,
{},
{10, 11, 10, 11},
{O::S_MOV_B32, O::V_MOV_B32, O::DS_WRITE_B32, O::DS_READ_B32,
O::DS_APPEND, O::DS_CONSUME, O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
}
TestCase DsAppendAllocatesAcrossWaves(u32 wave_size) {
using O = ShaderOpcode;
TestCase test;
test.name = wave_size == 64 ? "DsAppendAllocatesAcrossWaves64"
: "DsAppendAllocatesAcrossWaves32";
auto &code = test.code;
AppendVMovU32(&code, 2, 0);
AppendVMovU32(&code, 5, 1);
// Only workgroup lane zero initializes the shared allocation counter.
code.push_back(EncodeSop1(0x04, 10, 126));
code.push_back(EncodeVopc(0xc2, InlineU32(0), 0));
code.push_back(EncodeSop1(0x04, 126, 106));
code.push_back(EncodeDs0(0x0d, 0));
code.push_back(EncodeDs1(0, 2, 2));
code.push_back(EncodeSop1(0x04, 126, 10));
code.push_back(EncodeSopp(0x0a, 0));
for (u32 batch = 0; batch < 2; ++batch) {
// Captured CS ea0aceac sets M0=0 and uses OFFSET=1. LDS ignores M0
// and aligns OFFSET before allocating once for every active wave.
AppendSMovLiteral(&code, 124, batch == 0 ? 0 : 0xffff0000u);
code.push_back(EncodeDs0(0x3e, batch == 0 ? 1 : 3));
code.push_back(EncodeDs1(3, 0, 0));
code.push_back(EncodeVop2(0x24, 6, 127, 3));
code.push_back(EncodeVop2(0x23, 6, 126, 6));
code.push_back(EncodeVop2(0x1a, 7, InlineU32(2), 6));
// A histogram detects duplicate allocations without assuming wave order.
AppendBufferStoreOpcode(&code, 0x32, 5, 7);
}
code.push_back(EncodeSopp(0x0a, 0));
code.push_back(EncodeDs0(0x36, 0));
code.push_back(EncodeDs1(8, 0, 2));
AppendStoreVgprAtLaneDwordOffset(&code, 8, 0, 512);
AppendEnd(&code);
test.expected.assign(512, 1);
test.expected.resize(768, 512);
test.initial.resize(test.expected.size());
test.compute_info.threads_num[0] = 256;
test.compute_info.threads_num[1] = 1;
test.compute_info.threads_num[2] = 1;
test.compute_info.thread_ids_num = 1;
test.compute_info.wave_size = wave_size;
test.compute_info.lds_size_dwords = 1;
test.has_compute_info = true;
test.forbidden_spirv = {"\"gds\""};
test.opcodes = {O::S_MOV_B32, O::S_MOV_B64, O::S_BARRIER,
O::V_MOV_B32, O::V_CMP_EQ_U32, O::DS_WRITE_B32,
O::DS_APPEND, O::V_MBCNT_HI_U32_B32,
O::V_MBCNT_LO_U32_B32, O::V_LSHLREV_B32,
O::BUFFER_ATOMIC_ADD, O::DS_READ_B32,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
return test;
}
TestCase DsAppendUsesEncodedGdsSelector() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendSMovLiteral(&code, 124, 0x00000008u);
code.push_back(EncodeDs0(0x3e, 0, true));
code.push_back(EncodeDs1(0, 0, 0));
code.push_back(EncodeDs0(0x3d, 0, true));
code.push_back(EncodeDs1(1, 0, 0));
code.push_back(EncodeDs0(0x3e, 4, true));
code.push_back(EncodeDs1(2, 0, 0));
code.push_back(EncodeDs0(0x3d, 4, true));
code.push_back(EncodeDs1(3, 0, 0));
AppendSMovLiteral(&code, 124, 0x00080008u);
code.push_back(EncodeDs0(0x3e, 4, true));
code.push_back(EncodeDs1(4, 0, 0));
code.push_back(EncodeDs0(0x3d, 4, true));
code.push_back(EncodeDs1(5, 0, 0));
AppendStoreVgpr(&code, 0, 0);
AppendStoreVgpr(&code, 1, 1);
AppendStoreVgpr(&code, 2, 2);
AppendStoreVgpr(&code, 3, 3);
AppendStoreVgpr(&code, 4, 4);
AppendStoreVgpr(&code, 5, 5);
AppendEnd(&code);
TestCase test{"DsAppendGdsSelector",
code,
{},
{10, 11, 20, 21, 40, 41},
{O::S_MOV_B32, O::DS_APPEND, O::DS_CONSUME,
O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
test.gds_initial = {10, 20, 30, 40};
test.expected_gds = {10, 20, 30, 40};
return test;
}
TestCase DsOrderedCountAddressAndExec(u32 wave_size, bool pixel_counter) {
using O = ShaderOpcode;
TestCase test;
test.name = pixel_counter
? (wave_size == 64 ? "DsOrderedCountPixelAddressWave64"
: "DsOrderedCountPixelAddressWave32")
: (wave_size == 64 ? "DsOrderedCountAddressAndExecWave64"
: "DsOrderedCountAddressAndExecWave32");
auto &code = test.code;
// Both the base and OFFSET0 are aligned independently; M0's low bits are
// a wave index, or a pixel packer ID, rather than a byte address or size.
AppendSMovLiteral(&code, 124, 0x01030003u);
code.push_back(EncodeSop1(0x04, 10, 126)); // Save the full EXEC mask.
code.push_back(EncodeVop2(0x25, 2, InlineU32(1), 0)); // ADDR value = lane + 1.
AppendVMovU32(&code, 9, 99); // DATA0 must not supply the count.
const u32 selected_lane = wave_size == 64 ? 47u : 7u;
const std::array<uint64_t, 3> masks = {
wave_size == 64 ? UINT64_MAX : uint64_t{UINT32_MAX},
uint64_t{1} << selected_lane, 0};
u32 counter = 1000;
for (bool exchange : {false, true}) {
for (size_t mode = 0; mode < masks.size(); ++mode) {
AppendVMovU32(&code, 3, 0xdeadbeefu);
AppendSMovLiteral(&code, 126, static_cast<u32>(masks[mode]));
AppendSMovLiteral(&code, 127, static_cast<u32>(masks[mode] >> 32u));
const u32 release_done = exchange && mode == 2 ? 3u : 0u;
const u32 offset1 = release_done | (pixel_counter ? 4u : 0u) |
(exchange ? 0x10u : 0u);
code.push_back(EncodeDs0(0x3f, (offset1 << 8u) | 7u, true));
code.push_back(EncodeDs1(3, 9, 2));
code.push_back(EncodeSopp(0x0c, 0)); // Wait for the returned GDS value.
code.push_back(EncodeSop1(0x04, 126, 10)); // Restore EXEC before reading all lanes.
AppendStoreVgprAtLaneDwordOffset(&code, 3, 0,
static_cast<u32>(test.expected.size()));
test.expected.insert(test.expected.end(), wave_size, counter);
if (mode != 2) {
const u32 value = mode == 0 ? 1u : selected_lane + 1u;
counter = exchange ? value : counter + value;
}
}
}
AppendEnd(&code);
test.initial.assign(test.expected.size(), 0xdeadbeefu);
const u32 counter_address = pixel_counter ? 0x110u : 0x104u;
test.gds_initial.assign(0x120u / sizeof(u32), 100);
test.gds_initial[counter_address / sizeof(u32)] = 1000;
test.expected_gds = test.gds_initial;
test.expected_gds[counter_address / sizeof(u32)] = counter;
test.compute_info.threads_num[0] = wave_size;
test.compute_info.threads_num[1] = 1;
test.compute_info.threads_num[2] = 1;
test.compute_info.thread_ids_num = 1;
test.compute_info.wave_size = wave_size;
test.has_compute_info = true;
test.opcodes = {O::S_MOV_B32, O::S_MOV_B64, O::V_ADD_NC_U32, O::V_MOV_B32,
O::DS_ORDERED_COUNT, O::S_WAITCNT, O::V_LSHLREV_B32,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
return test;
}
TestCase DsAppendConsumeGdsRegionBounds() {
using O = ShaderOpcode;
struct Access {
u32 m0, offset;
bool valid;
};
constexpr Access accesses[] = {
{0x00000004u, 3, true},
{0x00000004u, 4, false},
{0x00000003u, 0, false},
{0xbffc0004u, 3, true},
{0xbffc0008u, 0, false},
{0xc0000004u, 0, false},
{0x0000c000u, 0xbfff, true},
{0x0000c000u, 0xc000, false},
};
TestCase test;
test.name = "DsAppendConsumeGdsRegionBounds";
auto &code = test.code;
for (const auto &access : accesses) {
AppendSMovLiteral(&code, 124, access.m0);
code.push_back(EncodeDs0(0x3e, access.offset, true));
code.push_back(EncodeDs1(0, 0, 0));
code.push_back(EncodeDs0(0x3d, access.offset, true));
code.push_back(EncodeDs1(1, 0, 0));
AppendStoreVgpr(&code, 0, static_cast<u32>(test.expected.size()));
test.expected.push_back(access.valid ? 10 : 0);
AppendStoreVgpr(&code, 1, static_cast<u32>(test.expected.size()));
test.expected.push_back(access.valid ? 11 : 0);
}
AppendEnd(&code);
test.initial.resize(test.expected.size());
// Backing beyond the guest partition must not make an invalid region valid.
test.gds_initial.assign(0x10000u / sizeof(u32), 10);
test.expected_gds = test.gds_initial;
test.opcodes = {O::S_MOV_B32, O::DS_APPEND, O::DS_CONSUME,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
return test;
}
TestCase DsGdsSubdwordAndAtomicWrites() {
using O = ShaderOpcode;
std::vector<u32> code;
code.push_back(EncodeVop1(0x01, 1, Vgpr(0)));
code.push_back(EncodeVop2(0x25, 2, InlineU32(1), 0));
code.push_back(EncodeDs0(0x1e, 0, true));
code.push_back(EncodeDs1(0, 2, 1));
code.push_back(EncodeVop2(0x1a, 8, InlineU32(1), 0));
code.push_back(EncodeVop2(0x25, 8, InlineU32(12), 8));
code.push_back(EncodeDs0(0x1f, 0, true));
code.push_back(EncodeDs1(0, 2, 8));
AppendVMovU32(&code, 3, 4);
AppendVMovU32(&code, 4, 1);
code.push_back(EncodeDs0(0x00, 0, true));
code.push_back(EncodeDs1(0, 4, 3));
AppendVMovU32(&code, 5, 8);
AppendVMovLiteral(&code, 6, 0x40a00000u);
AppendVMovLiteral(&code, 7, 0x40a00000u);
code.push_back(EncodeDs0(0x12, 0, true));
code.push_back(EncodeDs1Ex(0, 7, 6, 5));
AppendEnd(&code);
TestCase test;
test.name = "DsGdsSubdwordAndAtomics";
test.code = code;
test.opcodes = {O::V_MOV_B32, O::V_ADD_NC_U32, O::V_LSHLREV_B32,
O::DS_WRITE_B8, O::DS_WRITE_B16, O::DS_ADD_U32,
O::DS_MIN_F32, O::S_ENDPGM};
test.compute_info.threads_num[0] = 4;
test.compute_info.threads_num[1] = 1;
test.compute_info.threads_num[2] = 1;
test.compute_info.thread_ids_num = 1;
test.has_compute_info = true;
test.gds_initial = {0, 0, 0x42c80000u, 0, 0};
test.expected_gds = {0x04030201u, 4, 0x40a00000u, 0x00020001u, 0x00040003u};
return test;
}
TestCase ImageLoadR32UintUsesIntegerSampledImage() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 20, 2);
AppendVMovU32(&code, 21, 1);
code.push_back(EncodeMimg0(0x00, 0x1));
code.push_back(EncodeMimg1(0, 20));
AppendStoreVgpr(&code, 0, 0);
AppendEnd(&code);
TestCase test;
test.name = "ImageLoadR32UintUsesIntegerSampledImage";
test.code = code;
test.expected = {0xdeadbeefu};
test.opcodes = {O::V_MOV_B32, O::IMAGE_LOAD, O::BUFFER_STORE_DWORD,
O::S_ENDPGM};
test.sampled_image_rgba.resize(16);
test.sampled_image_rgba[6] = 0xdeadbeefu;
test.sampled_image_format = vk::Format::eR32Uint;
test.sampled_image_dwords_per_pixel = 1;
test.user_data = MakeSampledTextureData(Prospero::BufferFormat::k32UInt);
test.has_user_data = true;
test.required_spirv = {"image_10", "OpTypeImage %uint",
"OpImageFetch %v4uint"};
return test;
}
TestCase ImageLoadFmaskUsesNativeSampleMapping() {
auto test = ImageLoadR32UintUsesIntegerSampledImage();
test.name = "ImageLoadFmaskUsesNativeSampleMapping";
test.expected = {0x76543210u};
test.user_data = {0x303ac300u, 0xca100000u, 0x021bc3bfu, 0x91800004u,
0u, 0x00700000u, 0u, 0u};
test.image_descriptor_swizzle = DstSel(4, 0, 0, 0);
test.sampled_image_rgba.clear();
test.required_spirv = {"OpConstant %uint 1985229328"};
test.forbidden_spirv = {"OpTypeImage", "OpImageFetch"};
return test;
}
template <bool byte> TestCase ImageLoadSintUsesSignedSampledImage() {
using O = ShaderOpcode;
std::vector<u32> code;
for (u32 pixel = 0; pixel < (byte ? 4u : 1u); ++pixel) {
AppendVMovU32(&code, 20, byte ? pixel : 2u);
AppendVMovU32(&code, 21, 1);
code.push_back(EncodeMimg0(0x00, 0x1));
code.push_back(EncodeMimg1(0, 20));
AppendStoreVgpr(&code, 0, pixel);
}
AppendEnd(&code);
TestCase test;
test.name = byte ? "ImageLoadR8SintUsesSignedSampledImage"
: "ImageLoadR32SintUsesSignedSampledImage";
test.code = code;
if constexpr (byte) test.expected = {0xffffff80u, 0xffffffffu, 0u, 127u};
else test.expected = {0xffffff80u};
test.opcodes = {O::V_MOV_B32, O::IMAGE_LOAD, O::BUFFER_STORE_DWORD,
O::S_ENDPGM};
test.sampled_image_rgba.resize(byte ? 4u : 16u);
if constexpr (byte) test.sampled_image_rgba[1] = 0x7f00ff80u;
else test.sampled_image_rgba[6] = 0xffffff80u;
test.sampled_image_format = byte ? vk::Format::eR8Sint : vk::Format::eR32Sint;
test.sampled_image_dwords_per_pixel = 1;
constexpr auto format = byte ? Prospero::BufferFormat::k8SInt : Prospero::BufferFormat::k32SInt;
const auto surface = TextureGetSurfaceFormatInfo(format);
Require(test.name, "signed format metadata",
Prospero::NumBytesPerElement(format) == (byte ? 1u : 4u) &&
Prospero::SampledTextureNumericClass(format) == Prospero::TextureNumericClass::Sint &&
surface.vk_format == test.sampled_image_format &&
surface.conversion_format == Prospero::BufferFormat::kInvalid,
"signed sampled image lost its byte width, native mapping or numeric class");
test.user_data = MakeSampledTextureData(format);
test.has_user_data = true;
test.required_spirv = {"image_17", "OpTypeImage %int", "OpImageFetch %v4int",
"OpBitcast %v4uint"};
return test;
}
TestCase ImageLoadPackedUintUnpacksAndSwizzles() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 20, 2);
AppendVMovU32(&code, 21, 1);
code.push_back(EncodeMimg0(0x00, 0xf));
code.push_back(EncodeMimg1(0, 20));
for (u32 component = 0; component < 4; component++) {
AppendStoreVgpr(&code, component, component);
}
AppendEnd(&code);
TestCase test;
test.name = "ImageLoadPackedUintUnpacksAndSwizzles";
test.code = std::move(code);
test.expected = {0x2abu, 0x456u, 0x321u, 0x321u};
test.opcodes = {O::V_MOV_B32, O::IMAGE_LOAD, O::BUFFER_STORE_DWORD,
O::S_ENDPGM};
test.sampled_image_rgba.resize(16);
test.sampled_image_rgba[6] = 0xaae2b321u;
test.sampled_image_format = vk::Format::eR32Uint;
test.sampled_image_dwords_per_pixel = 1;
test.user_data =
MakeSampledTextureData(Prospero::BufferFormat::k11_11_10UInt);
test.has_user_data = true;
test.image_descriptor_swizzle = DstSel(6, 5, 4, 7);
test.required_spirv = {"OpImageFetch %v4uint", "OpBitFieldUExtract"};
return test;
}
template <bool d16, bool sparse> TestCase ImageLoadPackedUnorm() {
TestCase test;
test.opcodes = {ShaderOpcode::V_MOV_B32, ShaderOpcode::IMAGE_LOAD,
ShaderOpcode::BUFFER_STORE_DWORD, ShaderOpcode::S_ENDPGM};
test.name = d16 ? (sparse ? "ImageLoadPackedUnormD16Sparse" : "ImageLoadPackedUnormD16")
: (sparse ? "ImageLoadPackedUnormSparse" : "ImageLoadPackedUnorm");
AppendVMovU32(&test.code, 20, 2u);
AppendVMovU32(&test.code, 21, 1u);
test.code.push_back(EncodeMimg0(0x00u, sparse ? 0x5u : 0xfu));
test.code.push_back(EncodeMimg1(0, 20, 0, 0, false, d16));
constexpr u32 words = (sparse ? 2u : 4u) / (d16 ? 2u : 1u);
for (u32 word = 0; word < words; ++word) AppendStoreVgpr(&test.code, word, word);
AppendEnd(&test.code);
test.sampled_image_rgba.assign(16, 0u);
test.sampled_image_rgba[6] = (1024u << 11u) | (1023u << 22u);
test.sampled_image_format = vk::Format::eR32Uint;
test.sampled_image_dwords_per_pixel = 1;
test.user_data = MakeSampledTextureData(Prospero::BufferFormat::k11_11_10UNorm);
test.has_user_data = true;
test.image_descriptor_swizzle = sparse ? DstSel(1, 4, 5, 0) : DstSel(6, 5, 4, 7);
const u32 green = std::bit_cast<u32>(1024.0f * (1.0f / 2047.0f));
test.expected = d16 ? std::vector<u32>{0x38013c00u, 0u}
: std::vector<u32>{0x3f800000u, green, 0u, 0u};
test.expected.resize(words);
test.required_spirv = {"OpImageFetch %v4uint", "OpBitFieldUExtract", "OpConvertUToF"};
if constexpr (d16) test.required_spirv.emplace_back("PackHalf2x16");
test.forbidden_spirv = {"OpTypeSampler", "OpImageGather"};
return test;
}
TestCase ImageSamplePackedUintConvertsSampleAndGather() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovLiteral(&code, 20, 0x3f200000u);
AppendVMovLiteral(&code, 21, 0x3ec00000u);
code.push_back(EncodeMimg0(0x20, 0xf));
code.push_back(EncodeMimg1(0, 20));
code.push_back(EncodeMimg0(0x47, 0x2));
code.push_back(EncodeMimg1(4, 20));
for (u32 component = 0; component < 8; component++) {
AppendStoreVgpr(&code, component, component);
}
AppendEnd(&code);
TestCase test;
test.name = "ImageSamplePackedUintConvertsSampleAndGather";
test.code = std::move(code);
test.expected = {0x2abu, 0x456u, 0x321u, 0x321u,
0x456u, 0x456u, 0x456u, 0x456u};
test.opcodes = {O::V_MOV_B32, O::IMAGE_SAMPLE, O::IMAGE_GATHER4_LZ,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.sampled_image_rgba = std::vector<u32>(16, 0xaae2b321u);
test.sampled_image_format = vk::Format::eR32Uint;
test.sampled_image_dwords_per_pixel = 1;
test.user_data =
MakeSampledTextureData(Prospero::BufferFormat::k11_11_10UInt);
test.has_user_data = true;
test.image_descriptor_swizzle = DstSel(6, 5, 4, 7);
test.expected_force_point_sampler = true;
test.required_spirv = {"OpImageSampleExplicitLod", "OpImageGather",
"OpBitFieldUExtract"};
return test;
}
template <bool sparse> TestCase ImageSamplePackedUnormBilinear() {
using O = ShaderOpcode;
namespace D = ShaderRecompiler::Decoder;
const std::vector<u32> unnormalized = {EncodeMimg0(0x27, 0x7) | (1u << 12u),
EncodeMimg1(0, 20, 0, 2)};
D::Instruction decoded;
D::DecodeInstruction(unnormalized, 0, decoded);
Require("ImageSamplePackedUnormBilinear", "coordinate mode",
decoded.image_sample_flags ==
(D::ImageSampleFlagLevelZero | D::ImageSampleFlagUnnormalized),
"unnormalized instruction coordinates lost their sampling restriction");
constexpr u32 channels[4][3] = {{0, 0, 0}, {2047, 0, 512},
{0, 2047, 1023}, {2047, 2047, 256}};
constexpr float coordinates[][2] = {{0.5f, 0.5f}, {0.375f, 0.625f},
{-0.25f, 1.25f}, {0.25f, 0.25f},
{0.75f, 0.25f}, {0.0f, 0.5f}};
TestCase test;
test.name = sparse ? "ImageSamplePackedUnormBilinearSparse" : "ImageSamplePackedUnormBilinear";
test.image_width = test.image_height = 2;
test.sampled_image_format = vk::Format::eR32Uint;
test.sampled_image_dwords_per_pixel = 1;
for (const auto &pixel : channels) {
test.sampled_image_rgba.push_back(pixel[0] | (pixel[1] << 11u) | (pixel[2] << 22u));
}
test.user_data = MakeSampledTextureData(Prospero::BufferFormat::k11_11_10UNorm);
test.user_data[1] |= 1u << 30u;
test.user_data[2] = 1u << 14u;
test.user_data[8] = 0x92u;
test.user_data[9] = 0x00fff000u;
test.user_data[10] = 0x0a500000u; // Captured linear guest sampler; gather uses integer texels.
test.has_user_data = true;
test.image_descriptor_swizzle = DstSel(6, 4, 5, 1);
test.expected_force_point_sampler = true;
test.expected_float_tolerance = 0.000005f;
auto &code = test.code;
code.push_back(EncodeSop1(0x04, 12, 126));
for (const auto &uv : coordinates) {
AppendVMovLiteral(&code, 20, std::bit_cast<u32>(uv[0]));
AppendVMovLiteral(&code, 21, std::bit_cast<u32>(uv[1]));
code.push_back(EncodeMimg0(0x27, sparse ? 0x5 : 0xf));
code.push_back(EncodeMimg1(0, 20, 0, 2));
const float x = uv[0] * 2.0f - 0.5f, y = uv[1] * 2.0f - 0.5f;
const int ix = static_cast<int>(std::floor(x)), iy = static_cast<int>(std::floor(y));
const float fx = x - std::floor(x), fy = y - std::floor(y);
float rgba[4] = {0, 0, 0, 1};
for (u32 channel = 0; channel < 3; ++channel) {
for (int dy = 0; dy < 2; ++dy) for (int dx = 0; dx < 2; ++dx) {
const auto pixel = std::clamp(iy + dy, 0, 1) * 2 + std::clamp(ix + dx, 0, 1);
rgba[channel] += channels[pixel][channel] / (channel == 2 ? 1023.0f : 2047.0f) *
(dx ? fx : 1.0f - fx) * (dy ? fy : 1.0f - fy);
}
}
constexpr u32 swizzle[4] = {2, 0, 1, 3};
u32 reg = 0;
for (u32 component = 0; component < 4; ++component) {
if (sparse && component != 0 && component != 2) continue;
AppendStoreVgpr(&code, reg++, test.expected.size());
test.expected.push_back(std::bit_cast<u32>(rgba[swizzle[component]]));
}
}
AppendVMovLiteral(&code, 0, 0x3f400000u);
code.push_back(EncodeSop1(0x04, 126, InlineU32(0)));
code.push_back(EncodeMimg0(0x27, 0x1));
code.push_back(EncodeMimg1(0, 20, 0, 2));
code.push_back(EncodeSop1(0x04, 126, 12));
AppendStoreVgpr(&code, 0, test.expected.size());
test.expected.push_back(0x3f400000u);
AppendEnd(&code);
test.user_data[50] = test.expected.size() * sizeof(u32);
test.user_data[51] = 3u << 28u;
test.opcodes = {O::V_MOV_B32, O::S_MOV_B64, O::IMAGE_SAMPLE,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.required_spirv = {"OpImageGather", "OpBitFieldUExtract", "OpConvertUToF"};
test.forbidden_spirv = {"OpImageFetch", "OpImageSample", "OpImageRead", "OpImageWrite"};
return test;
}
template <bool rg, bool snorm = false> TestCase ImageSamplePacked8() {
using O = ShaderOpcode;
TestCase test;
test.name = snorm ? "ImageSampleR8SNorm"
: rg ? "ImageSampleRG8UScaled" : "ImageSampleR8UScaled";
AppendVMovLiteral(&test.code, 20, std::bit_cast<u32>(0.375f));
AppendVMovLiteral(&test.code, 21, std::bit_cast<u32>(0.375f));
test.code.push_back(EncodeMimg0(0x20, 0xf));
test.code.push_back(EncodeMimg1(0, 20));
test.code.push_back(EncodeMimg0(0x47, 0x1));
test.code.push_back(EncodeMimg1(4, 20));
test.code.push_back(EncodeMimg0(0x47, 0x8));
test.code.push_back(EncodeMimg1(8, 20));
for (u32 component = 0; component < 12; ++component) {
AppendStoreVgpr(&test.code, component, component);
}
AppendEnd(&test.code);
test.image_width = test.image_height = 2;
test.sampled_image_rgba =
snorm ? std::vector<u32>{0x7fc08180u}
: rg ? std::vector<u32>{0xffffff00u, 0x00ff0000u}
: std::vector<u32>{0xff00ff00u};
test.sampled_image_format =
snorm ? vk::Format::eR8Snorm
: rg ? vk::Format::eR8G8Unorm : vk::Format::eR8Unorm;
test.sampled_image_dwords_per_pixel = 1;
test.sampler_filter = snorm ? vk::Filter::eNearest : vk::Filter::eLinear;
test.expected_float_tolerance = snorm ? 0.00002f : 0.01f;
test.user_data = MakeSampledTextureData(
snorm ? Prospero::BufferFormat::k8SNorm
: rg ? Prospero::BufferFormat::k8_8UScaled
: Prospero::BufferFormat::k8UScaled);
test.user_data[1] |= 1u << 30u;
test.user_data[2] = 1u << 14u;
test.user_data[50] = 12u * sizeof(u32);
test.user_data[51] = 3u << 28u;
test.has_user_data = true;
test.image_descriptor_swizzle = snorm ? DstSel(4, 5, 6, 7)
: DstSel(rg ? 5 : 4, 4, 0, 1);
constexpr float midpoint = -64.0f / 127.0f;
const std::array<float, 12> expected = snorm
// -128 and -127 both clamp to -1; the interior signed value normalizes by 127.
? std::array<float, 12>{-1, 0, 0, 1, midpoint, 1, -1, -1, 1, 1, 1, 1}
// R varies horizontally; G varies vertically in the opposite direction.
: std::array<float, 12>{rg ? 191.25f : 63.75f, 63.75f, 0.0f, 1.0f,
0.0f, rg ? 0.0f : 255.0f, 255.0f, rg ? 255.0f : 0.0f,
1.0f, 1.0f, 1.0f, 1.0f};
for (float value : expected) {
test.expected.push_back(std::bit_cast<u32>(value));
}
test.opcodes = {O::V_MOV_B32, O::IMAGE_SAMPLE, O::IMAGE_GATHER4_LZ,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
return test;
}
TestCase ImageLoadR128IgnoresAdjacentMaskSgprs() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 20, 2);
AppendVMovU32(&code, 21, 1);
code.push_back(EncodeSop1(0x04, 4, 126)); // s_mov_b64 s[4:5], exec
code.push_back(EncodeMimg0(0x00, 0x1, 0, false, 1, true));
code.push_back(EncodeMimg1(0, 20));
AppendStoreVgpr(&code, 0, 0);
AppendEnd(&code);
TestCase test;
test.name = "ImageLoadR128IgnoresAdjacentMaskSgprs";
test.code = code;
test.expected = {0xdeadbeefu};
test.opcodes = {O::V_MOV_B32, O::S_MOV_B64, O::IMAGE_LOAD,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.sampled_image_rgba.resize(16);
test.sampled_image_rgba[6] = 0xdeadbeefu;
test.sampled_image_format = vk::Format::eR32Uint;
test.sampled_image_dwords_per_pixel = 1;
test.user_data = MakeSampledTextureData(Prospero::BufferFormat::k32UInt);
test.has_user_data = true;
test.decoded_counts = {{"r128=1", 1}};
return test;
}
TestCase ImageLoad1DUsesScalarCoordinate() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 20, 2);
code.push_back(EncodeMimg0(0x00, 0x1, 0, false, 0));
code.push_back(EncodeMimg1(0, 20));
AppendStoreVgpr(&code, 0, 0);
AppendEnd(&code);
TestCase test;
test.name = "ImageLoad1DUsesScalarCoordinate";
test.code = code;
test.expected = {0xdeadbeefu};
test.opcodes = {O::V_MOV_B32, O::IMAGE_LOAD, O::BUFFER_STORE_DWORD,
O::S_ENDPGM};
test.image_width = 4;
test.image_height = 1;
test.sampled_image_rgba = {0, 0, 0xdeadbeefu, 0};
test.sampled_image_format = vk::Format::eR32Uint;
test.sampled_image_dwords_per_pixel = 1;
test.sampled_image_type = vk::ImageType::e1D;
test.sampled_image_view_type = vk::ImageViewType::e1D;
test.user_data = MakeSampledTextureData(Prospero::BufferFormat::k32UInt);
test.user_data[3] = static_cast<uint32_t>(Prospero::ImageType::kColor1D)
<< 28u;
test.has_user_data = true;
test.required_spirv = {"image_8"};
return test;
}
TestCase ImageGather2DInstructionWith1DDescriptor() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovLiteral(&code, 20, 0x3f000000u);
AppendVMovLiteral(&code, 21, 0x3f000000u);
code.push_back(EncodeMimg0(0x47, 0x1));
code.push_back(EncodeMimg1(0, 20));
for (u32 i = 0; i < 4u; i++) {
AppendStoreVgpr(&code, i, i);
}
AppendEnd(&code);
auto image = MakeRgbaImage(4, 1);
SetRgbaPixel(&image, 4, 1, 0, 0x40000000u, 0, 0, 0);
SetRgbaPixel(&image, 4, 2, 0, 0x40400000u, 0, 0, 0);
TestCase test;
test.name = "ImageGather2DInstructionWith1DDescriptor";
test.code = code;
test.expected = {0x40000000u, 0x40400000u, 0x40400000u, 0x40000000u};
test.opcodes = {O::V_MOV_B32, O::IMAGE_GATHER4_LZ, O::BUFFER_STORE_DWORD,
O::S_ENDPGM};
test.image_width = 4;
test.image_height = 1;
test.sampled_image_rgba = image;
test.sampled_image_type = vk::ImageType::e1D;
test.sampled_image_view_type = vk::ImageViewType::e1D;
test.user_data = MakeSampledTextureData(Prospero::BufferFormat::k32Float);
test.user_data[3] = static_cast<uint32_t>(Prospero::ImageType::kColor1D)
<< 28u;
test.has_user_data = true;
test.required_spirv = {"image_1", "OpImageQuerySizeLod",
"OpImageSampleExplicitLod", "Floor"};
test.forbidden_spirv = {"OpImageGather"};
return test;
}
TestCase ImageLoad1DArrayUsesLayerCoordinate() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 20, 2);
AppendVMovU32(&code, 21, 1);
code.push_back(EncodeMimg0(0x00, 0x1, 0, false, 4));
code.push_back(EncodeMimg1(0, 20));
AppendStoreVgpr(&code, 0, 0);
AppendEnd(&code);
TestCase test;
test.name = "ImageLoad1DArrayUsesLayerCoordinate";
test.code = code;
test.expected = {0xcafebabeu};
test.opcodes = {O::V_MOV_B32, O::IMAGE_LOAD, O::BUFFER_STORE_DWORD,
O::S_ENDPGM};
test.image_width = 4;
test.image_height = 1;
test.sampled_image_rgba = {0, 0, 0, 0, 0, 0, 0xcafebabeu, 0};
test.sampled_image_format = vk::Format::eR32Uint;
test.sampled_image_dwords_per_pixel = 1;
test.sampled_image_type = vk::ImageType::e1D;
test.sampled_image_view_type = vk::ImageViewType::e1DArray;
test.sampled_image_layers = 2;
test.user_data = MakeSampledTextureData(Prospero::BufferFormat::k32UInt);
test.user_data[3] = static_cast<uint32_t>(Prospero::ImageType::kColor1DArray)
<< 28u;
test.has_user_data = true;
test.required_spirv = {"image_9"};
return test;
}
TestCase ImageLoad1DArrayDescriptorUsesSelectedLayer() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 20, 2);
code.push_back(EncodeMimg0(0x00, 0x1, 0, false, 0));
code.push_back(EncodeMimg1(0, 20));
AppendStoreVgpr(&code, 0, 0);
AppendEnd(&code);
TestCase test;
test.name = "ImageLoad1DArrayDescriptorUsesSelectedLayer";
test.code = code;
test.expected = {0xcafebabeu};
test.opcodes = {O::V_MOV_B32, O::IMAGE_LOAD, O::BUFFER_STORE_DWORD,
O::S_ENDPGM};
test.image_width = 4;
test.image_height = 1;
test.sampled_image_rgba = {
0, 0, 0xdeadbeefu, 0, 0, 0, 0xcafebabeu, 0,
};
test.sampled_image_format = vk::Format::eR32Uint;
test.sampled_image_dwords_per_pixel = 1;
test.sampled_image_type = vk::ImageType::e1D;
test.sampled_image_view_type = vk::ImageViewType::e1D;
test.sampled_image_layers = 2;
test.sampled_image_view_base_layer = 1;
test.sampled_image_view_layers = 1;
test.user_data = MakeSampledTextureData(Prospero::BufferFormat::k32UInt);
test.user_data[3] = static_cast<uint32_t>(Prospero::ImageType::kColor1DArray)
<< 28u;
test.user_data[4] = 1u | (1u << 16u);
test.has_user_data = true;
test.required_spirv = {"image_8"};
return test;
}
TestCase ImageLoadMipUsesVaddr2Lod2D() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 20, 1);
AppendVMovU32(&code, 21, 1);
AppendVMovU32(&code, 22, 1);
AppendVMovU32(&code, 23, 0);
code.push_back(EncodeMimg0(0x01, 0x1));
code.push_back(EncodeMimg1(0, 20));
AppendStoreVgpr(&code, 0, 0);
AppendEnd(&code);
auto base = MakeRgbaImage(4, 4);
SetRgbaPixel(&base, 4, 1, 1, 0x3f800000u, 0, 0, 0);
auto mip1 = MakeRgbaImage(2, 2);
SetRgbaPixel(&mip1, 2, 1, 1, 0x40000000u, 0, 0, 0);
TestCase test;
test.name = "ImageLoadMipUsesVaddr2Lod2D";
test.code = code;
test.expected = {0x40000000u};
test.opcodes = {O::V_MOV_B32, O::IMAGE_LOAD_MIP, O::BUFFER_STORE_DWORD,
O::S_ENDPGM};
test.sampled_image_rgba_mips = {base, mip1};
return test;
}
TestCase ImageLoadMipNsaUsesSelectedAddressVgprs() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 20, 1);
AppendVMovU32(&code, 21, 0);
AppendVMovU32(&code, 22, 0);
AppendVMovU32(&code, 30, 1);
AppendVMovU32(&code, 31, 1);
code.push_back(EncodeMimg0(0x01, 0x1, 1));
code.push_back(EncodeMimg1(0, 20));
code.push_back((30u << 0u) | (31u << 8u));
AppendStoreVgpr(&code, 0, 0);
AppendEnd(&code);
auto base = MakeRgbaImage(4, 4);
SetRgbaPixel(&base, 4, 1, 0, 0x3f800000u, 0, 0, 0);
auto mip1 = MakeRgbaImage(2, 2);
SetRgbaPixel(&mip1, 2, 1, 1, 0x40a00000u, 0, 0, 0);
TestCase test;
test.name = "ImageLoadMipNsaUsesSelectedAddressVgprs";
test.code = code;
test.expected = {0x40a00000u};
test.opcodes = {O::V_MOV_B32, O::IMAGE_LOAD_MIP, O::BUFFER_STORE_DWORD,
O::S_ENDPGM};
test.sampled_image_rgba_mips = {base, mip1};
return test;
}
TestCase ImageLoadA16UintCoordsOnGpu() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovLiteral(&code, 20, 0x00010002u); // x=2, y=1 packed as u16.
AppendVMovU32(&code, 21, 0);
code.push_back(EncodeMimg0(0x00, 0xf));
code.push_back(EncodeMimg1(0, 20, 0, 0, true));
for (u32 i = 0; i < 4u; i++) {
AppendStoreVgpr(&code, i, i);
}
AppendEnd(&code);
auto image = MakeRgbaImage(4, 4);
SetRgbaPixel(&image, 4, 2, 1, 0x3f800000u, 0x40000000u, 0x40400000u,
0x40800000u);
TestCase test;
test.name = "ImageLoadA16UintCoordsOnGpu";
test.code = code;
test.expected = {0x3f800000u, 0x40000000u, 0x40400000u, 0x40800000u};
test.opcodes = {O::V_MOV_B32, O::IMAGE_LOAD, O::BUFFER_STORE_DWORD,
O::S_ENDPGM};
test.sampled_image_rgba = image;
test.required_spirv = {"OpShiftRightLogical", "OpBitwiseAnd"};
test.forbidden_spirv = {"UnpackHalf2x16"};
return test;
}
TestCase ImageGetResinfoDmaskWidthHeight() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 24, 0);
code.push_back(EncodeMimg0(0x0e, 0x3));
code.push_back(EncodeMimg1(0, 24));
AppendStoreVgpr(&code, 0, 0);
AppendStoreVgpr(&code, 1, 1);
AppendEnd(&code);
TestCase test;
test.name = "ImageGetResinfoDmaskWidthHeight";
test.code = code;
test.expected = {4, 2};
test.opcodes = {O::V_MOV_B32, O::IMAGE_GET_RESINFO, O::BUFFER_STORE_DWORD,
O::S_ENDPGM};
test.image_width = 4;
test.image_height = 2;
test.sampled_image_rgba = MakeRgbaImage(test.image_width, test.image_height);
return test;
}
TestCase ImageGetResinfoDmaskMipLevels() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 24, 0);
code.push_back(EncodeMimg0(0x0e, 0x8));
code.push_back(EncodeMimg1(0, 24));
AppendStoreVgpr(&code, 0, 0);
AppendEnd(&code);
TestCase test;
test.name = "ImageGetResinfoDmaskMipLevels";
test.code = code;
test.expected = {2};
test.opcodes = {O::V_MOV_B32, O::IMAGE_GET_RESINFO, O::BUFFER_STORE_DWORD,
O::S_ENDPGM};
auto base = MakeRgbaImage(4, 4);
auto mip1 = MakeRgbaImage(2, 2);
test.sampled_image_rgba_mips = {base, mip1};
return test;
}
TestCase ImageSampleAndGather() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovLiteral(&code, 20, 0x3f200000u);
AppendVMovLiteral(&code, 21, 0x3ec00000u);
AppendVMovU32(&code, 22, 0);
code.push_back(EncodeMimg0(0x20, 0xf));
code.push_back(EncodeMimg1(0, 20));
code.push_back(EncodeMimg0(0x47, 0x1));
code.push_back(EncodeMimg1(4, 20));
AppendVMovU32(&code, 24, 0);
AppendVMovLiteral(&code, 25, 0x3f200000u);
AppendVMovLiteral(&code, 26, 0x3ec00000u);
AppendVMovU32(&code, 27, 0);
code.push_back(EncodeMimg0(0x57, 0x1));
code.push_back(EncodeMimg1(8, 24));
code.push_back(EncodeMimg0(0x60, 0x1));
code.push_back(EncodeMimg1(12, 20));
for (u32 i = 0; i < 13u; i++) {
AppendStoreVgpr(&code, i, i);
}
AppendEnd(&code);
auto image = MakeRgbaImage(4, 4);
for (u32 y = 0; y < 4u; y++) {
for (u32 x = 0; x < 4u; x++) {
SetRgbaPixel(&image, 4, x, y, 0x3f800000u, 0, 0, 0);
}
}
SetRgbaPixel(&image, 4, 2, 1, 0x3f800000u, 0x40000000u, 0x40400000u,
0x40800000u);
TestCase test;
test.name = "ImageSampleAndGather";
test.code = code;
test.expected = {0x3f800000u,
0x40000000u,
0x40400000u,
0x40800000u,
0x3f800000u,
0x3f800000u,
0x3f800000u,
0x3f800000u,
0x3f800000u,
0x3f800000u,
0x3f800000u,
0x3f800000u,
0};
test.opcodes = {
O::V_MOV_B32, O::IMAGE_SAMPLE, O::IMAGE_GET_LOD,
O::IMAGE_GATHER4_LZ, O::IMAGE_GATHER4_LZ_O, O::BUFFER_STORE_DWORD,
O::S_ENDPGM};
test.sampled_image_rgba = image;
return test;
}
TestCase ImageCubeGradientsPreserveDerivatives() {
using O = ShaderOpcode;
TestCase test;
test.name = "ImageCubeGradientsPreserveDerivatives";
for (u32 component = 0; component < 7u; component++) {
constexpr std::array values{0.125f, 0.0f, 0.0f, 0.125f,
1.375f, 1.625f, 10.0f};
AppendVMovLiteral(&test.code, 20u + component,
std::bit_cast<u32>(values[component]));
}
test.code.push_back(EncodeMimg0(0x22, 0x1, 0, false, 3));
test.code.push_back(EncodeMimg1(0, 20));
AppendStoreVgpr(&test.code, 0, 0);
AppendEnd(&test.code);
test.opcodes = {O::V_MOV_B32, O::IMAGE_SAMPLE, O::BUFFER_STORE_DWORD,
O::S_ENDPGM};
test.user_data = MakeSampledTextureData(Prospero::BufferFormat::k32_32_32_32Float);
test.user_data[1] |= 3u << 30u;
test.user_data[2] = 3u << 14u;
test.user_data[3] =
(static_cast<u32>(Prospero::ImageType::kCube) << 28u) | (2u << 16u);
test.user_data[4] = 11u;
test.user_data[5] = 2u << 4u;
test.user_data[50] = sizeof(u32);
test.has_user_data = true;
test.sampled_image_view_type = vk::ImageViewType::e2DArray;
test.sampled_image_layers = 12;
for (u32 mip = 0; mip < 3u; mip++) {
const auto layer_dwords = (4u >> mip) * (4u >> mip) * 4u;
auto &pixels = test.sampled_image_rgba_mips.emplace_back(layer_dwords * 12u);
for (u32 layer = 0; layer < 12u; layer++) {
std::fill_n(pixels.begin() + layer * layer_dwords, layer_dwords,
std::bit_cast<u32>(static_cast<float>(1u + layer + 100u * mip)));
}
}
// Guest face 10 is host layer 8. Unmodified gradients select mip 0;
// applying the cube coordinate bias to gradients instead selects mip 2.
test.expected = {std::bit_cast<u32>(9.0f)};
test.required_spirv = {"OpImageSampleExplicitLod", "Grad"};
return test;
}
TestCase ImageGatherExplicitLod() {
using O = ShaderOpcode;
std::vector<u32> code;
// PPSA29343 uses NSA coordinates v81/v84 with its explicit LOD in v78.
AppendVMovLiteral(&code, 81, 0x3f000000u);
AppendVMovLiteral(&code, 84, 0x3f000000u);
AppendVMovLiteral(&code, 78, 0x3f800000u);
code.push_back(EncodeMimg0(0x44, 0x1, 1));
code.push_back(EncodeMimg1(9, 81, 0, 2));
code.push_back(84u | (78u << 8u));
code.push_back(EncodeMimg0(0x24, 0x1, 1));
code.push_back(EncodeMimg1(13, 81, 0, 2));
code.push_back(84u | (78u << 8u));
for (u32 i = 0; i < 5u; i++) {
AppendStoreVgpr(&code, i + 9, i);
}
AppendEnd(&code);
TestCase test;
test.name = "ImageGatherExplicitLod";
test.code = std::move(code);
test.opcodes = {O::V_MOV_B32, O::IMAGE_GATHER4_L, O::IMAGE_SAMPLE,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
// A gather's single-mip descriptors must not replace the ordinary sample's
// full mip chain when both instructions use the same guest descriptor.
test.expected = std::vector<u32>(5, 0x41000000u);
test.sampled_image_rgba_mips = {std::vector<u32>(4 * 4 * 4, 0x3f800000u),
std::vector<u32>(2 * 2 * 4, 0x41000000u)};
test.user_data = MakeSampledTextureData(Prospero::BufferFormat::k32_32_32_32Float);
test.user_data[1] |= 3u << 30u;
test.user_data[2] = 3u << 14u;
test.user_data[3] |= 1u << 16u;
test.user_data[8] = 0x92u;
test.user_data[9] = 0xfff000u;
test.user_data[10] = 0x05000000u; // Captured point sampler, preclamp disabled.
test.user_data[50] = test.expected.size() * sizeof(u32);
test.user_data[51] = 3u << 28u;
test.has_user_data = true;
test.expected_mip_descriptors = 2;
test.required_spirv = {"OpImageGather", "OpImageSampleExplicitLod"};
return test;
}
template <u32 mode> TestCase ImageGatherLodPerLane() {
using O = ShaderOpcode;
constexpr std::array names{
"ImageGatherLodPerLane", "ImageGatherLodBaseMip",
"ImageGatherLodSamplerBounds", "ImageGatherLodTextureMinimum",
"ImageGatherLodMipNone", "ImageGatherLodPointPostclampMaximum",
"ImageGatherLodPointPreclampMaximum", "ImageGatherLodPointPostclampMinimum",
"ImageGatherLodPointPreclampMinimum"};
constexpr std::array lods = mode < 5
? std::array{-1.0f, 0.0f, 1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 9.0f}
: std::array{-1.0f, 0.0f, 0.25f, 0.75f, 1.25f, 1.75f, 2.75f, 9.0f};
constexpr std::array<std::array<u32, 8>, 9> selected_mips{{
{0, 0, 1, 2, 3, 4, 4, 4},
{1, 1, 2, 3, 4, 4, 4, 4},
{2, 2, 2, 3, 3, 3, 3, 3},
{3, 3, 3, 3, 3, 3, 3, 3},
{1, 1, 1, 1, 1, 1, 1, 1},
{0, 0, 0, 1, 1, 2, 2, 2},
{0, 0, 0, 1, 1, 1, 1, 1},
{2, 2, 2, 2, 2, 2, 3, 4},
{1, 1, 1, 1, 1, 2, 3, 4},
}};
constexpr u32 lanes = lods.size();
constexpr u32 sampler_min = mode == 2 ? 256 : mode >= 7 ? 448 : 0;
constexpr u32 sampler_max = mode == 2 ? 512 : mode == 3 ? 256
: mode == 5 || mode == 6 ? 448 : 1024;
TestCase test;
test.name = names[mode];
test.image_width = test.image_height = 16;
test.sampled_image_view_base_mip = mode > 0 && mode < 5 ? 1 : 0;
for (u32 mip = 0; mip < 5; mip++) {
test.sampled_image_rgba_mips.emplace_back(
(16u >> mip) * (16u >> mip) * 4u,
std::bit_cast<u32>(static_cast<float>(mip + 1)));
}
for (const auto lod : lods) {
test.initial.push_back(std::bit_cast<u32>(lod));
}
test.expected = test.initial;
for (u32 component = 0; component < 4; component++) {
for (const auto mip : selected_mips[mode]) {
test.expected.push_back(std::bit_cast<u32>(static_cast<float>(mip + 1)));
}
}
auto &code = test.code;
code.push_back(EncodeVop1(0x01, 16, Vgpr(0)));
code.push_back(EncodeVop2(0x1a, 31, InlineU32(2), 16));
code.push_back(EncodeMubuf0(0x0c));
code.push_back(EncodeMubuf1(22, 4, 31));
AppendVMovLiteral(&code, 20, 0x3f000000u);
AppendVMovLiteral(&code, 21, 0x3f000000u);
code.push_back(EncodeMimg0(0x44, 0x1));
code.push_back(EncodeMimg1(0, 20, 0, 2));
for (u32 component = 0; component < 4; component++) {
AppendStoreVgprAtLaneDwordOffset(&code, component, 16, lanes * (component + 1));
}
AppendEnd(&code);
test.opcodes = {O::V_MOV_B32, O::V_LSHLREV_B32, O::BUFFER_LOAD_DWORD,
O::IMAGE_GATHER4_L, O::V_ADD_NC_U32, O::BUFFER_STORE_DWORD,
O::S_ENDPGM};
test.user_data = MakeSampledTextureData(Prospero::BufferFormat::k32_32_32_32Float);
test.user_data[1] |= (3u << 30u) | (mode == 3 ? 768u << 8u : 0u);
test.user_data[2] = 3u | (15u << 14u);
test.user_data[3] |= (test.sampled_image_view_base_mip << 12u) | (4u << 16u);
test.user_data[8] = 0x92u;
test.user_data[9] = sampler_min | (sampler_max << 12u);
// Point preclamp selects whether the half-level rounding adjustment precedes
// or follows the clamps. The streaming T# minimum follows the sampler bounds.
test.user_data[10] = mode == 4 ? 0x4001u : (1u << 26u);
if constexpr (mode == 6 || mode == 8) {
test.user_data[10] |= 1u << 28u;
}
test.user_data[18] = lanes * sizeof(u32);
test.user_data[19] = 3u << 28u;
test.user_data[50] = test.expected.size() * sizeof(u32);
test.user_data[51] = 3u << 28u;
test.has_user_data = true;
test.compute_info.threads_num[0] = lanes;
test.compute_info.threads_num[1] = test.compute_info.threads_num[2] = 1;
test.compute_info.thread_ids_num = 1;
test.has_compute_info = true;
test.expected_mip_descriptors = 5 - test.sampled_image_view_base_mip;
test.required_spirv = {"OpImageGather"};
return test;
}
TestCase ImageD16GatherPacksHalfPairs() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovLiteral(&code, 20, 0x3f000000u);
AppendVMovLiteral(&code, 21, 0x3f000000u);
AppendVMovU32(&code, 22, 0);
code.push_back(EncodeMimg0(0x47, 0x1));
code.push_back(EncodeMimg1(0, 20, 0, 0, false, true));
AppendStoreVgpr(&code, 0, 0);
AppendStoreVgpr(&code, 1, 1);
AppendEnd(&code);
auto image = MakeRgbaImage(4, 4);
for (u32 y = 0; y < 4u; y++) {
for (u32 x = 0; x < 4u; x++) {
SetRgbaPixel(&image, 4, x, y, 0x3f000000u, 0, 0, 0);
}
}
TestCase test;
test.name = "ImageD16GatherPacksHalfPairs";
test.code = code;
test.expected = {0x38003800u, 0x38003800u};
test.opcodes = {O::V_MOV_B32, O::IMAGE_GATHER4_LZ, O::BUFFER_STORE_DWORD,
O::S_ENDPGM};
test.sampled_image_rgba = image;
test.decoded_counts = {{"d16=1", 1}};
test.required_spirv = {"OpImageGather", "PackHalf2x16"};
return test;
}
TestCase ImageSampleA16SamplerCoordsOnGpu() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovLiteral(&code, 20, 0x36003900u); // x=0.625, y=0.375 packed as f16.
AppendVMovU32(&code, 21, 0);
code.push_back(EncodeMimg0(0x20, 0xf));
code.push_back(EncodeMimg1(0, 20, 0, 0, true));
for (u32 i = 0; i < 4u; i++) {
AppendStoreVgpr(&code, i, i);
}
AppendEnd(&code);
auto image = MakeRgbaImage(4, 4);
SetRgbaPixel(&image, 4, 2, 1, 0x3f800000u, 0x40000000u, 0x40400000u,
0x40800000u);
TestCase test;
test.name = "ImageSampleA16SamplerCoordsOnGpu";
test.code = code;
test.expected = {0x3f800000u, 0x40000000u, 0x40400000u, 0x40800000u};
test.opcodes = {O::V_MOV_B32, O::IMAGE_SAMPLE, O::BUFFER_STORE_DWORD,
O::S_ENDPGM};
test.sampled_image_rgba = image;
test.required_spirv = {"UnpackHalf2x16"};
return test;
}
TestCase ImageSampleOpcodeAliasUsesNormalCoords() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovLiteral(&code, 2, 0x3f000000u);
AppendVMovLiteral(&code, 3, 0x3f000000u);
code.push_back(0xf0800109u); // observed image_sample_a v6, v2, s0, s24
code.push_back(0x00c00602u);
AppendStoreVgpr(&code, 6, 0);
AppendEnd(&code);
TestCase test;
test.name = "ImageSampleOpcodeAliasUsesNormalCoords";
test.code = code;
test.opcodes = {O::V_MOV_B32, O::IMAGE_SAMPLE, O::BUFFER_STORE_DWORD,
O::S_ENDPGM};
test.required_spirv = {"OpImageSampleExplicitLod"};
test.forbidden_spirv = {"UnpackHalf2x16"};
test.compile_only = true;
return test;
}
template <u32 wave_size> TestCase ImageSampleA16OffsetKeepsTexelOffset32BitOnGpu() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 20, 1); // +1 X remains a full dword with A16 coordinates.
AppendVMovLiteral(&code, 21, 0x36003900u); // x=0.625, y=0.375 packed as f16.
AppendVMovU32(&code, 22, 0);
code.push_back(EncodeMimg0(0x30, 0xf));
code.push_back(EncodeMimg1(0, 20, 0, 0, true));
for (u32 i = 0; i < 4u; i++) {
AppendStoreVgpr(&code, i, i);
}
AppendEnd(&code);
auto image = MakeRgbaImage(4, 4);
SetRgbaPixel(&image, 4, 2, 1, 0x3f000000u, 0, 0, 0);
SetRgbaPixel(&image, 4, 3, 1, 0x3f800000u, 0x40000000u, 0x40400000u,
0x40800000u);
TestCase test;
test.name = wave_size == 32 ? "ImageSampleA16OffsetKeepsTexelOffset32BitOnGpu32"
: "ImageSampleA16OffsetKeepsTexelOffset32BitOnGpu64";
test.code = code;
test.compute_info.wave_size = wave_size;
test.has_compute_info = true;
test.expected = {0x3f800000u, 0x40000000u, 0x40400000u, 0x40800000u};
test.opcodes = {O::V_MOV_B32, O::IMAGE_SAMPLE, O::BUFFER_STORE_DWORD,
O::S_ENDPGM};
test.sampled_image_rgba = image;
test.required_spirv = {"UnpackHalf2x16", "Lod|ConstOffset"};
test.forbidden_spirv = {"OpBitFieldSExtract", "OpImageQuerySizeLod", "OpFDiv"};
return test;
}
template <u32 wave_size> TestCase ImageSampleOffsetsReadFourNeighbors() {
using O = ShaderOpcode;
TestCase test;
test.name = wave_size == 32 ? "ImageSampleOffsetsReadFourNeighbors32"
: "ImageSampleOffsetsReadFourNeighbors64";
test.compute_info.wave_size = wave_size;
test.has_compute_info = true;
constexpr std::array offsets{0x01u, 0x3fu, 0x3f00u, 0x0100u};
constexpr std::array opcodes{0x37u, 0x37u, 0x34u, 0x32u};
for (u32 sample = 0; sample < offsets.size(); ++sample) {
AppendVMovLiteral(&test.code, 20, offsets[sample]);
u32 coordinate = 21;
if (opcodes[sample] == 0x32u) {
for (float gradient : {0.25f, 0.f, 0.f, 0.25f}) {
AppendVMovLiteral(&test.code, coordinate++, std::bit_cast<u32>(gradient));
}
}
// All instructions address the center of texel (2, 1) before applying the offset.
AppendVMovLiteral(&test.code, coordinate, std::bit_cast<u32>(0.625f));
AppendVMovLiteral(&test.code, coordinate + 1, std::bit_cast<u32>(0.375f));
if (opcodes[sample] == 0x34u) AppendVMovU32(&test.code, coordinate + 2, 0);
test.code.push_back(EncodeMimg0(opcodes[sample], 0x1));
test.code.push_back(EncodeMimg1(sample, 20));
AppendStoreVgpr(&test.code, sample, sample);
}
AppendEnd(&test.code);
test.sampled_image_rgba = MakeRgbaImage(4, 4);
SetRgbaPixel(&test.sampled_image_rgba, 4, 2, 1, std::bit_cast<u32>(16.f), 0, 0, 0);
constexpr std::array<std::array<u32, 2>, 4> neighbors{{{3, 1}, {1, 1}, {2, 0}, {2, 2}}};
for (u32 sample = 0; sample < neighbors.size(); ++sample) {
const auto value = std::bit_cast<u32>(float(sample + 1));
SetRgbaPixel(&test.sampled_image_rgba, 4, neighbors[sample][0], neighbors[sample][1],
value, 0, 0, 0);
test.expected.push_back(value);
}
test.opcodes = {O::V_MOV_B32, O::IMAGE_SAMPLE, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.required_spirv = {"Lod|ConstOffset", "Grad|ConstOffset"};
test.forbidden_spirv = {"OpImageQuerySizeLod", "OpFDiv", "OpBitFieldSExtract", "OpSwitch"};
return test;
}
void CheckDynamicSampleOffsets(VulkanHarness &vulkan, u32 wave_size) {
using O = ShaderOpcode;
TestCase test;
test.name = wave_size == 32u ? "DynamicSampleOffsets32" : "DynamicSampleOffsets64";
test.has_compute_info = test.has_user_data = true;
test.compute_info.wave_size = wave_size;
test.compute_info.host_subgroup_size = vulkan.SubgroupSize();
test.compute_info.threads_num[0] = wave_size;
test.compute_info.threads_num[1] = test.compute_info.threads_num[2] = 1u;
test.compute_info.thread_ids_num = 1;
test.user_data = MakeSampledTextureData(Prospero::BufferFormat::k32_32_32_32Float);
test.user_data[2] = 7u | (7u << 14u);
test.user_data[3] |= 1u << 16u;
test.user_data[5] = 1u << 4u;
test.user_data[50] = wave_size * sizeof(u32);
// Read the offset selector from a texture, as the native shader does.
test.code = {EncodeVop2(0x1b, 20, InlineU32(7), 0),
EncodeVop2(0x16, 21, InlineU32(3), 0),
EncodeVop2(0x1b, 21, InlineU32(1), 21),
EncodeMimg0(0x00, 0x1), EncodeMimg1(4, 20),
EncodeVop2(0x1b, 20, InlineU32(3), 4),
EncodeVop2(0x16, 22, InlineU32(2), 4),
EncodeVop2(0x1b, 22, InlineU32(3), 22)};
for (u32 reg : {20u, 22u}) {
test.code.push_back(EncodeVop2(0x25, reg, InlineU32(63), reg));
test.code.push_back(EncodeVop2(0x1b, reg, InlineU32(63), reg));
}
test.code.push_back(EncodeVop2(0x1a, 22, InlineU32(8), 22));
test.code.push_back(EncodeVop2(0x1c, 20, Vgpr(22), 20));
AppendVMovLiteral(&test.code, 21, std::bit_cast<u32>(0.375f));
AppendVMovLiteral(&test.code, 22, std::bit_cast<u32>(0.375f));
AppendVMovLiteral(&test.code, 23, std::bit_cast<u32>(1.0f));
test.code.push_back(EncodeMimg0(0x34, 0x1)); // SAMPLE_L_O: offset is applied at mip 1.
test.code.push_back(EncodeMimg1(5, 20));
AppendStoreVgprAtLaneDwordOffset(&test.code, 5, 0, 0);
AppendEnd(&test.code);
test.opcodes = {O::V_AND_B32, O::V_LSHRREV_B32, O::V_ADD_NC_U32, O::V_LSHLREV_B32,
O::V_OR_B32, O::V_MOV_B32, O::IMAGE_LOAD, O::IMAGE_SAMPLE,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.required_spirv = {"ImageGatherExtended", "Lod|Offset", "OpBitFieldSExtract %int"};
test.forbidden_spirv = {"OpSwitch", "OpImageQuerySizeLod", "OpFDiv"};
test.initial.assign(wave_size, 0xdeadbeefu);
for (u32 lane = 0; lane < wave_size; ++lane)
test.expected.push_back(std::bit_cast<u32>(float(100u + (lane & 15u))));
auto compiled = CompileCase(test, vulkan.SubgroupSize());
spvtools::SpirvTools tools(SPV_ENV_VULKAN_1_2);
std::string text;
const auto halves = wave_size > vulkan.SubgroupSize() ? 2u : 1u;
Require(test.name, "direct dynamic offset", tools.Disassemble(compiled.spirv, &text) &&
CountText(text, "OpImageSampleExplicitLod") == halves &&
CountText(text, "OpBitFieldSExtract %int") == 2u * halves,
"dynamic offsets expanded samples or lost signed six-bit components");
#if KYTY_PLATFORM != KYTY_PLATFORM_WINDOWS
if (wave_size == 32u) {
Require(test.name, "default feature gate", !ShaderRecompiler::CompileOptions{}.maintenance8_enabled,
"maintenance8 must reflect the enabled device feature");
ExpectFatal("DynamicSampleOffsetsRequireMaintenance8", [&] {
ShaderRecompiler::Spirv::EmitProgram(compiled.program, {.compute = &test.compute_info});
});
}
#endif
auto base = MakeRgbaImage(8, 8);
auto mip = MakeRgbaImage(4, 4);
for (u32 selector = 0; selector < 16u; ++selector) {
SetRgbaPixel(&base, 8, selector & 7u, selector >> 3u, 0x3f800000u | selector, 0, 0, 0);
SetRgbaPixel(&mip, 4, selector & 3u, selector >> 2u,
std::bit_cast<u32>(float(100u + selector)), 0, 0, 0);
}
auto texture = vulkan.CreateImageMips(test.name, 8, 8, vk::Format::eR32G32B32A32Sfloat,
vk::ImageUsageFlagBits::eSampled, {base, mip}, 4u, vk::ImageLayout::eShaderReadOnlyOptimal,
vk::ImageType::e2D, vk::ImageViewType::e2D, 1u);
const auto sampler = vulkan.CreateSampler(test.name, 2u);
auto output = vulkan.CreateStorageBuffer(test.name, test.initial, test.initial.size());
vulkan.Dispatch(test, compiled, output, nullptr, &texture, nullptr, nullptr, sampler);
CompareWords(test, "image-derived offsets -1..2 at mip 1", test.expected,
vulkan.ReadBuffer(test.name, output, test.expected.size()));
vulkan.DestroyBuffer(&output);
vulkan.Device().destroySampler(sampler);
vulkan.DestroyImage(&texture);
std::printf("[compute] %-32s ok\n", test.name);
}
TestCase ImageSampleA16CompareBiasRdna2AddressOrder() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovLiteral(&code, 20, 0x00003800u); // bias=0.5 as low f16.
AppendVMovLiteral(&code, 21, 0x3f000000u); // PCF reference stays 32-bit.
AppendVMovLiteral(&code, 22, 0x36003900u); // x=0.625, y=0.375 packed as f16.
code.push_back(EncodeMimg0(0x2d, 0x1));
code.push_back(EncodeMimg1(0, 20, 0, 0, true));
AppendStoreVgpr(&code, 0, 0);
// Ordinary and comparison reads of one sharp require distinct SPIR-V image types.
code.push_back(EncodeMimg0(0x27, 0x1));
code.push_back(EncodeMimg1(1, 22, 0, 0, true));
AppendStoreVgpr(&code, 1, 1);
AppendEnd(&code);
TestCase test;
test.name = "ImageSampleA16CompareBiasRdna2AddressOrder";
test.code = code;
test.opcodes = {O::V_MOV_B32, O::IMAGE_SAMPLE, O::BUFFER_STORE_DWORD,
O::S_ENDPGM};
test.required_spirv = {"OpTypeImage %float 2D 0", "OpTypeImage %float 2D 1",
"OpImageSampleDrefExplicitLod", "OpImageSampleExplicitLod",
"UnpackHalf2x16"};
test.compile_only = true;
return test;
}
TestCase ImageGatherCompareOpcodes() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 20, 0);
AppendVMovLiteral(&code, 21, 0x3f000000u);
AppendVMovLiteral(&code, 22, 0x3f200000u);
AppendVMovLiteral(&code, 23, 0x3ec00000u);
AppendVMovLiteral(&code, 24, 0x3f000000u);
AppendVMovLiteral(&code, 25, 0x3f200000u);
AppendVMovLiteral(&code, 26, 0x3ec00000u);
AppendVMovLiteral(&code, 28, 0x3f000000u);
AppendVMovLiteral(&code, 29, 0x3f200000u);
AppendVMovLiteral(&code, 30, 0x3ec00000u);
AppendVMovU32(&code, 32, 0);
AppendVMovLiteral(&code, 33, 0x3f000000u);
AppendVMovLiteral(&code, 34, 0x3f200000u);
AppendVMovLiteral(&code, 35, 0x3ec00000u);
code.push_back(EncodeMimg0(0x48, 0x1));
code.push_back(EncodeMimg1(4, 24));
code.push_back(EncodeMimg0(0x4f, 0x1));
code.push_back(EncodeMimg1(8, 28));
code.push_back(EncodeMimg0(0x58, 0x1));
code.push_back(EncodeMimg1(0, 20));
code.push_back(EncodeMimg0(0x5f, 0x1));
code.push_back(EncodeMimg1(12, 32));
AppendStoreVgpr(&code, 4, 0);
AppendStoreVgpr(&code, 8, 1);
AppendStoreVgpr(&code, 0, 2);
AppendStoreVgpr(&code, 12, 3);
AppendEnd(&code);
TestCase test;
test.name = "ImageGatherCompareOpcodes";
test.code = code;
test.opcodes = {
O::V_MOV_B32, O::IMAGE_GATHER4_C, O::IMAGE_GATHER4_C_LZ,
O::IMAGE_GATHER4_C_O, O::IMAGE_GATHER4_C_LZ_O, O::BUFFER_STORE_DWORD,
O::S_ENDPGM};
test.required_spirv = {"OpImageDrefGather", "ConstOffset"};
test.forbidden_spirv = {"OpBitFieldSExtract"};
test.compile_only = true;
return test;
}
TestCase ImageStoreVariants() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 20, 1);
AppendVMovU32(&code, 21, 2);
AppendVMovU32(&code, 22, 0);
AppendVMovLiteral(&code, 0, 0x3f800000u);
AppendVMovLiteral(&code, 1, 0x40000000u);
AppendVMovLiteral(&code, 2, 0x40400000u);
AppendVMovLiteral(&code, 3, 0x40800000u);
code.push_back(EncodeMimg0(0x08, 0xf));
code.push_back(EncodeMimg1(0, 20));
AppendVMovU32(&code, 24, 3);
AppendVMovU32(&code, 25, 2);
AppendVMovU32(&code, 26, 0);
AppendVMovU32(&code, 27, 0);
AppendVMovLiteral(&code, 4, 0x40a00000u);
AppendVMovLiteral(&code, 5, 0x40c00000u);
AppendVMovLiteral(&code, 6, 0x40e00000u);
AppendVMovLiteral(&code, 7, 0x41000000u);
code.push_back(EncodeMimg0(0x09, 0xf));
code.push_back(EncodeMimg1(4, 24));
AppendVMovLiteral(&code, 8, 0x12345678u);
AppendStoreVgpr(&code, 8, 0);
AppendEnd(&code);
auto expected_image = MakeRgbaImage(4, 4);
SetRgbaPixel(&expected_image, 4, 1, 2, 0x3f800000u, 0x40000000u, 0x40400000u,
0x40800000u);
SetRgbaPixel(&expected_image, 4, 3, 2, 0x40a00000u, 0x40c00000u, 0x40e00000u,
0x41000000u);
TestCase test;
test.name = "ImageStoreVariants";
test.code = code;
test.expected = {0x12345678u};
test.opcodes = {O::V_MOV_B32, O::IMAGE_STORE, O::IMAGE_STORE_MIP,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.storage_image_rgba = MakeRgbaImage(4, 4);
test.storage_image_r32ui = std::vector<u32>(16, 0);
test.expected_storage_image_rgba = expected_image;
return test;
}
TestCase ImageD16StoreUnpacksHalfPairs() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 20, 1);
AppendVMovU32(&code, 21, 2);
AppendVMovU32(&code, 22, 0);
AppendVMovLiteral(&code, 0, 0x38003400u);
AppendVMovLiteral(&code, 1, 0x3c003a00u);
code.push_back(EncodeMimg0(0x08, 0xf));
code.push_back(EncodeMimg1(0, 20, 0, 0, false, true));
AppendEnd(&code);
auto expected_image = MakeRgbaImage(4, 4);
SetRgbaPixel(&expected_image, 4, 1, 2, 0x3e800000u, 0x3f000000u, 0x3f400000u,
0x3f800000u);
TestCase test;
test.name = "ImageD16StoreUnpacksHalfPairs";
test.code = code;
test.opcodes = {O::V_MOV_B32, O::IMAGE_STORE, O::S_ENDPGM};
test.storage_image_rgba = MakeRgbaImage(4, 4);
test.expected_storage_image_rgba = expected_image;
test.decoded_counts = {{"d16=1", 1}};
test.required_spirv = {"UnpackHalf2x16", "OpImageWrite"};
return test;
}
void CheckComputeThreadDimensions(VulkanHarness &vulkan) {
using O = ShaderOpcode;
constexpr u32 grid_size = 16 * 8 * 4;
const std::array extents = {std::array{9u, 5u, 3u}, std::array{13u, 7u, 4u},
std::array{3u, 2u, 1u}, std::array{16u, 8u, 4u}};
for (const u32 wave_size : {32u, 64u}) {
for (const bool thread_dimensions : {false, true}) {
TestCase test;
test.name = thread_dimensions ? (wave_size == 32 ? "ThreadDimensions32" : "ThreadDimensions64")
: (wave_size == 32 ? "GroupDimensions32" : "GroupDimensions64");
auto &code = test.code;
code.push_back(EncodeSop1(0x04, 20, 126)); // Save the SPI-provided initial EXEC.
AppendSMovLiteral(&code, 24, grid_size * 4);
for (u32 axis = 0; axis < 3; ++axis) {
code.push_back(EncodeVop1(0x01, 3 + axis, 4 + axis));
code.push_back(EncodeVop2(0x1a, 3 + axis, InlineU32(3 - axis), 3 + axis));
code.push_back(EncodeVop2(0x25, 3 + axis, Vgpr(axis), 3 + axis));
}
code.push_back(EncodeVop2(0x1a, 4, InlineU32(4), 4));
code.push_back(EncodeVop2(0x1a, 5, InlineU32(7), 5));
code.push_back(EncodeVop2(0x25, 3, Vgpr(4), 3));
code.push_back(EncodeVop2(0x25, 3, Vgpr(5), 3));
code.push_back(EncodeVop2(0x25, 20, InlineU32(1), 3));
code.push_back(EncodeVop2(0x1a, 20, InlineU32(2), 20));
AppendVMovU32(&code, 6, 1);
AppendVMovU32(&code, 7, 0);
AppendBufferStoreDword(&code, 6, 20);
AppendBufferStoreOpcode(&code, 0x32, 6, 7);
code.push_back(EncodeSop2(0x0f, 126, 20, InlineU32(1)));
AppendVMovU32(&code, 6, 2); // Divergence changes only lane zero of each wave.
code.push_back(EncodeSop1(0x04, 126, 20));
code.push_back(EncodeVop2(0x25, 20, 24, 20));
AppendBufferStoreDword(&code, 6, 20);
AppendVMovU32(&code, 8, 1);
AppendBufferStoreOpcode(&code, 0x32, 8, 7);
AppendEnd(&code);
test.initial.resize(1 + 2 * grid_size);
test.opcodes = {O::S_MOV_B64, O::S_MOV_B32, O::S_AND_B64, O::V_MOV_B32,
O::V_LSHLREV_B32, O::V_ADD_NC_U32, O::BUFFER_STORE_DWORD,
O::BUFFER_ATOMIC_ADD, O::S_ENDPGM};
test.compute_info.wave_size = wave_size;
test.compute_info.host_subgroup_size = vulkan.SubgroupSize();
test.compute_info.threads_num[0] = 8;
test.compute_info.threads_num[1] = 4;
test.compute_info.threads_num[2] = 2;
std::fill_n(test.compute_info.group_id, 3, true);
test.compute_info.thread_ids_num = 3;
test.compute_info.workgroup_register = 4;
test.compute_info.dispatch_thread_dimensions = thread_dimensions;
test.has_compute_info = true;
test.expand_shader_data_storage = thread_dimensions && wave_size == 32;
const auto compiled = CompileCase(test, vulkan.SubgroupSize());
std::vector<u32> compiled_key;
BuildStageStaticKey(test.compute_info, compiled_key);
for (const auto &extent : extents) {
std::copy(extent.begin(), extent.end(), test.compute_info.dispatch_threads_num);
std::vector<u32> runtime_key;
BuildStageStaticKey(test.compute_info, runtime_key);
Require(test.name, "runtime shader reuse", compiled_key == runtime_key,
"dispatch thread counts changed the compiled shader cache key");
test.dispatch_x = thread_dimensions ? (extent[0] + 7) / 8 : 2;
test.dispatch_y = thread_dimensions ? (extent[1] + 3) / 4 : 2;
test.dispatch_z = thread_dimensions ? (extent[2] + 1) / 2 : 2;
test.expected.assign(test.initial.size(), 0);
for (u32 z = 0; z < 4; ++z) {
for (u32 y = 0; y < 8; ++y) {
for (u32 x = 0; x < 16; ++x) {
if (thread_dimensions && (x >= extent[0] || y >= extent[1] || z >= extent[2])) continue;
const auto index = x + 16 * y + 128 * z;
const auto local_index = x % 8 + 8 * (y % 4) + 32 * (z % 2);
test.expected[0] += 2;
test.expected[1 + index] = 1;
test.expected[1 + grid_size + index] = local_index % wave_size == 0 ? 2 : 1;
}
}
}
auto output = vulkan.CreateStorageBuffer(test.name, test.initial, test.initial.size());
vulkan.Dispatch(test, compiled, output);
const auto actual = vulkan.ReadBuffer(test.name, output, test.expected.size());
vulkan.DestroyBuffer(&output);
CompareWords(test, "initial EXEC, restored EXEC and padded-lane atomics", test.expected, actual);
if (!thread_dimensions) break;
}
std::printf("[compute] %-32s ok\n", test.name);
}
}
}
void CheckRuntimeBufferRecords(VulkanHarness &vulkan) {
using namespace ShaderRecompiler::IR;
TestCase test;
test.name = "RuntimeBufferRecords";
// Captured b71ef7c579835a0d: read an input count, divide it into chunks,
// construct a structured descriptor, then clear each live record's last DWORD.
test.code = {
0xbfa00003u, 0xd7460000u, 0x04010c0eu, 0x8805ff0du, 0x00040000u, 0xbe84030cu,
0xbe860381u, 0xbe8703ffu, 0x00016204u, 0xb06a0100u, 0xe0300000u, 0x80010400u,
0x92049281u, 0x7e0a026au, 0x996a9480u, 0xb0062000u, 0xb00a0080u, 0x380212f9u,
0x8686066au, 0x7e040208u, 0x7e0602ffu, 0x00016204u, 0xbf8c3f70u, 0x7d8808ffu,
0x00000200u, 0x7d8808f9u, 0x06868404u, 0x7d8808f9u, 0x06868606u, 0x020d80f9u,
0x8686060au, 0xd5010005u, 0x00120affu, 0x00000200u, 0xd5010006u, 0x001a0d05u,
0x7e0a0d06u, 0x7e0c0506u, 0xd76d0004u, 0x041a08c1u, 0x7e0a5705u, 0x81ea0680u,
0x7e080504u, 0x100a0affu, 0x4f7ffffeu, 0x7e0a0f05u, 0x7ed60505u, 0x936a6b6au,
0x9aea6b6au, 0x816a6a6bu, 0x9a87046au, 0x936a0607u, 0x4e08086au, 0x4e0a0806u,
0x7d8608f9u, 0x06868406u, 0x7d860a06u, 0x87ea046au, 0x020902f9u, 0x86860680u,
0xd5286a04u, 0x00120807u, 0xbeea047eu, 0x7da40080u, 0xbf880003u, 0x7e0a0280u,
0xe0700000u, 0x80000500u, 0xbefe046au, 0x7e0a0280u, 0x7e000502u, 0x7e020501u,
0x7e040504u, 0x7e060503u, 0xe0702010u, 0x80000500u, 0xbf810000u,
};
constexpr u32 count_address = 4;
test.user_data[0] = 16;
test.user_data[1] = 4u << 16;
test.user_data[2] = 1;
test.user_data[3] = 0x5204;
test.user_data[8] = 20;
test.user_data[12] = count_address;
test.has_user_data = true;
test.storage_buffer_offsets = {count_address, 16, 20};
test.initial.assign(5 + 5 * 130, 0xdeadbeefu);
test.initial[1] = 72;
test.has_compute_info = true;
test.compute_info.threads_num[0] = 64;
test.compute_info.threads_num[1] = test.compute_info.threads_num[2] = 1;
test.compute_info.lds_size_dwords = 0;
test.compute_info.wave_size = 64;
test.compute_info.float_mode = 0xc0;
test.compute_info.group_id[0] = true;
test.compute_info.thread_ids_num = 1;
test.compute_info.workgroup_register = 14;
test.compute_info.dispatch_thread_dimensions = true;
test.compute_info.dispatch_threads_num[0] = 2;
test.compute_info.dispatch_threads_num[1] = test.compute_info.dispatch_threads_num[2] = 1;
test.forbidden_spirv = {"get_bda_pointer", "flattened_srt", "OpFDiv", "OpUMulExtended"};
const auto compiled = CompileCase(test, vulkan.SubgroupSize());
Require(test.name, "runtime record count",
compiled.program.info.buffers.size() == 3 &&
compiled.resources.buffers[2].dwords[2] == 2 &&
compiled.resources.specialization_reads ==
std::vector<std::pair<uint64_t, uint64_t>>{{count_address, 4}},
"descriptor capacity lost its exact clean input read");
ShaderRecompiler::CompileOptions options;
options.stage = ShaderType::Compute;
options.wave_size = 64;
options.user_data = test.user_data;
options.input_info.compute = &test.compute_info;
auto translated = ShaderRecompiler::TranslateProgram(test.code, options);
auto plan = ExtractResourcePlan(translated.program);
struct CountMemory { u32 value = 72; u32 reads = 0; bool clean = true; } memory;
SrtRuntime runtime{.user_data = test.user_data, .userdata = &memory};
ResourceSnapshot snapshot;
ResourceSpecialization specialization;
Require(test.name, "strict reader required",
!MaterializeResources(plan, runtime, snapshot, specialization),
"vector descriptor input bypassed the clean-memory reader");
runtime.read_specialization_memory = [](void *data, uint64_t address, std::span<u32> words) {
auto &memory = *static_cast<CountMemory *>(data);
++memory.reads;
if (!memory.clean || address != count_address || words.size() != 1) return false;
words[0] = memory.value;
return true;
};
memory.clean = false;
Require(test.name, "dirty input rejected",
!MaterializeResources(plan, runtime, snapshot, specialization),
"GPU-dirty count was accepted for host descriptor evaluation");
memory.clean = true;
ResourceSpecialization initial_specialization;
// Reuse the same resource plan and compiled shader as the count and coverage change.
for (const u32 count : {0u, 72u, 512u, 8192u, 262144u}) {
memory.value = count;
memory.reads = 0;
const u32 chunk = count < 512 ? 64 : count < 8192 ? 128 : count < 262144 ? 256 : 512;
const u32 records = (count + chunk - 1) / chunk;
Require(test.name, "cached descriptor refresh",
MaterializeResources(plan, runtime, snapshot, specialization) && memory.reads == 1 &&
snapshot.buffers[2].dwords[2] == records &&
snapshot.specialization_reads ==
std::vector<std::pair<uint64_t, uint64_t>>{{count_address, 4}},
"count refresh repeated a memory read or retained an old descriptor capacity");
if (count == 0) initial_specialization = specialization;
Require(test.name, "capacity is not specialization",
specialization == initial_specialization,
"changing record capacity would recompile the cached shader");
test.initial.assign(5 + 5 * records, 0xdeadbeefu);
for (const u32 threads : {2u, 3u, 65u}) {
test.initial[1] = count;
test.compute_info.dispatch_threads_num[0] = threads;
test.dispatch_x = (threads + 63) / 64;
test.expected = test.initial;
test.expected[4] = 0;
for (u32 index = 0; index < std::min(threads, records); ++index) {
test.expected[9 + index * 5] = 0;
}
auto output = vulkan.CreateStorageBuffer(test.name, test.initial, test.initial.size());
vulkan.Dispatch(test, compiled, output);
const auto actual = vulkan.ReadBuffer(test.name, output, test.expected.size());
vulkan.DestroyBuffer(&output);
CompareWords(test, "record capacity and padded invocation bounds", test.expected, actual);
}
}
std::printf("[compute] %-32s ok\n", test.name);
}
void CheckBoundedFormattedBufferCopy(VulkanHarness &vulkan) {
using namespace ShaderRecompiler::IR;
constexpr std::array<u32, 8> lengths{16, 1, 4, 32, 48, 4, 32, 48};
constexpr std::array<u32, 8> source_records{16, 1, 2, 32, 40, 0, 32, 48};
constexpr std::array<u32, 8> destination_records{16, 1, 4, 24, 48, 4, 32, 44};
const auto alignment = static_cast<u32>(vulkan.RuntimeContext().GetPhysicalDeviceProperties()
.limits.minStorageBufferOffsetAlignment);
const auto slot_bytes = (512u + alignment - 1u) / alignment * alignment;
for (const u32 wave : {32u, 64u}) {
for (const u32 count : {0u, 1u, 8u}) {
TestCase test;
test.name = wave == 32 ? "BoundedFormattedBufferCopy32" : "BoundedFormattedBufferCopy64";
// Captured b3ade16b0485b3bd: uniform loop over two V# arrays; copy only lanes
// below each row's count. Scalar+vector guards and register reuse are intact.
test.code = {0xbfa00001u, 0xd7460000u, 0x04010c02u, 0xbeeb0380u,
0xf4000080u, 0xfa000120u, 0xbf8cc07fu, 0xbf0a026bu, 0xbf840015u,
0x97eaff6bu, 0x00000100u, 0xf40000c0u, 0xd4000000u, 0xbf8cc07fu,
0x7da80003u, 0xbf88000bu, 0x8f6a846bu, 0xf4080100u, 0xd4000000u,
0xbf8cc07fu, 0xe0002000u, 0x80010100u, 0xf4080200u, 0xd4000080u,
0xbf8c0070u, 0xe0102000u, 0x80020100u, 0x816b816bu, 0xbefe04c1u,
0xbf82ffe8u, 0xbf810000u};
constexpr u32 table_base = 256u;
test.initial.assign(slot_bytes * 19u / 4u, 0xdeadbeefu);
test.user_data[0] = table_base;
test.has_user_data = true;
test.has_compute_info = true;
test.compute_info.wave_size = wave;
test.compute_info.threads_num[0] = wave;
test.compute_info.threads_num[1] = test.compute_info.threads_num[2] = 1;
test.compute_info.lds_size_dwords = 0;
test.compute_info.group_id[0] = true;
test.compute_info.workgroup_register = 2;
test.compute_info.thread_ids_num = 1;
test.bda_mappings = {{0u, 0u}};
for (u32 key = 0; key < 8u; ++key) {
const auto src = slot_bytes * (key + 2u);
const auto dst = slot_bytes * (key + 10u);
for (const auto [row, address, records] :
{std::tuple{key * 4u, src, source_records[key]},
std::tuple{32u + key * 4u, dst, destination_records[key]}}) {
const auto word = table_base / 4u + row;
test.initial[word] = address;
test.initial[word + 1u] = 4u << 16u;
test.initial[word + 2u] = records;
test.initial[word + 3u] =
static_cast<u32>(Prospero::BufferFormat::k32UInt) << 12u | 4u;
}
test.initial[table_base / 4u + 64u + key] = lengths[key];
for (u32 lane = 0; lane < 64u; ++lane)
test.initial[src / 4u + lane] = 0x80000001u + key * 0x10000u + lane;
}
test.initial[table_base / 4u + 72u] = count;
test.expected = test.initial;
for (u32 key = 0; key < count; ++key)
for (u32 lane = 0; lane < std::min({wave, lengths[key], destination_records[key]}); ++lane)
test.expected[slot_bytes * (key + 10u) / 4u + lane] =
lane < source_records[key] ? 0x80000001u + key * 0x10000u + lane : 0u;
auto compiled = CompileCase(test, vulkan.SubgroupSize());
if (count == 0u) {
u32 formatted_reads = 0;
for (const auto* block : compiled.program.blocks) {
for (const auto& inst : *block) {
if (inst.GetOpcode() != ValueOpcode::LoadBufferU32) continue;
const auto& memory = compiled.program.memory_info[inst.Flags<MemoryFlags>().index];
formatted_reads += memory.kind == ResourceKind::IndirectBuffer && memory.formatted;
}
}
Require(test.name, "runtime zero loop bound", formatted_reads == 1u,
"runtime loop count must retain the GPU formatted source read");
}
Require(test.name, "finite writable descriptors",
compiled.resources.buffers.size() == std::max(1u, count),
"the bounded loop lost or overprobed a destination");
test.storage_buffer_ranges.resize(compiled.resources.buffers.size());
for (u32 resource = 0; resource < compiled.resources.buffers.size(); ++resource) {
const auto& descriptor = compiled.resources.buffers[resource].dwords;
test.storage_buffer_ranges[resource] =
{descriptor[0], std::max(4u, descriptor[2] * 4u)};
}
if (count == 8u) {
std::set<u32> nonuniform;
std::vector<u32> stores;
u32 bda_function = 0, conversion_function = 0, geometry_function = 0, function = 0;
u32 bda_calls = 0, conversion_calls = 0, conversion_definitions = 0;
u32 geometry_calls = 0, geometry_definitions = 0, geometry_switches = 0;
bool geometry_pure = true, geometry_cases = true;
for (size_t offset = 5; offset < compiled.spirv.size();) {
const auto words = std::span(compiled.spirv).subspan(offset, compiled.spirv[offset] >> 16u);
const auto opcode = static_cast<spv::Op>(words[0] & 0xffffu);
if (opcode == spv::OpName) {
const std::string_view name(reinterpret_cast<const char *>(words.data() + 2));
if (name == "get_bda_pointer") bda_function = words[1];
if (name == "convert_indirect_buffer_format") conversion_function = words[1];
if (name == "indirect_buffer_format_geometry") geometry_function = words[1];
}
if (opcode == spv::OpDecorate && words[2] == spv::DecorationNonUniform)
nonuniform.insert(words[1]);
if (opcode == spv::OpStore) stores.push_back(words[1]);
if (opcode == spv::OpFunction) {
function = words[2];
conversion_definitions += function == conversion_function;
geometry_definitions += function == geometry_function;
}
if (function == geometry_function && function != 0u) {
geometry_pure &= opcode != spv::OpLoad && opcode != spv::OpStore &&
opcode != spv::OpFunctionCall;
if (opcode == spv::OpSwitch) {
++geometry_switches;
constexpr u32 formats = u32(Prospero::BufferFormat::k32_32_32_32Float) + 1u;
geometry_cases &= words.size() == 3u + 2u * formats;
if (words.size() == 3u + 2u * formats) {
geometry_cases &= words[2] == words[4]; // Invalid formats use format zero.
for (u32 format = 0; format < formats; ++format)
geometry_cases &= words[3u + 2u * format] == format;
}
}
}
if (opcode == spv::OpFunctionCall) {
if (words[3] == bda_function) ++bda_calls;
if (words[3] == conversion_function) ++conversion_calls;
if (words[3] == geometry_function) ++geometry_calls;
}
if (opcode == spv::OpFunctionEnd) function = 0;
offset += words.size();
}
Require(test.name, "one shared formatted conversion",
conversion_definitions == 1u &&
conversion_calls == (wave > vulkan.SubgroupSize() ? 2u : 1u),
"formatted loads must call one conversion function across virtual halves");
Require(test.name, "one shared pure format geometry",
geometry_definitions == 1u && geometry_calls == conversion_calls &&
geometry_switches == 1u && geometry_cases && geometry_pure,
"format geometry duplicated, lost a format/default or acquired memory accesses");
Require(test.name, "one shared indirect store per virtual half",
std::ranges::count_if(stores, [&](u32 pointer) { return nonuniform.contains(pointer); }) ==
(wave > vulkan.SubgroupSize() ? 2 : 1),
"formatted identity lowering duplicated the store per descriptor");
Require(test.name, "shared formatted payload read", bda_calls <= 12u,
"formatted source load duplicated BDA reads per format or component");
std::printf("[compute] BoundedFormattedBufferCopy%u: %zu SPIR-V words, %u BDA calls\n",
wave, compiled.spirv.size(), bda_calls);
}
auto output = vulkan.CreateStorageBuffer(test.name, test.initial, test.initial.size(), true);
vulkan.Dispatch(test, compiled, output);
const auto actual = vulkan.ReadBuffer(test.name, output, test.expected.size());
vulkan.DestroyBuffer(&output);
CompareWords(test, "bounded rows, formatted raw bits, source OOB zero and destination canaries",
test.expected, actual);
}
}
std::printf("[compute] %-32s ok\n", "BoundedFormattedBufferCopy");
}
void CheckBoundedRecordBufferAccess(VulkanHarness &vulkan) {
using namespace ShaderRecompiler::IR;
using F = Prospero::BufferFormat;
const u32 alignment = vulkan.RuntimeContext().GetPhysicalDeviceProperties()
.limits.minStorageBufferOffsetAlignment;
const u32 slot = (4096u + alignment - 1u) / alignment * alignment;
for (const u32 wave : {32u, 64u}) {
for (const u32 count : {0u, 2u, 3u, 6u}) {
TestCase test;
test.name = wave == 32 ? "BoundedRecordBufferAccess32" : "BoundedRecordBufferAccess64";
constexpr u32 table = 256u;
test.initial.assign((slot * 13u + 6u * wave * 16u) / 4u, 0xdeadbeefu);
test.has_user_data = test.has_compute_info = true;
test.user_data[0] = table;
test.user_data[1] = 8u << 16u;
test.user_data[2] = 6u * 192u / 8u;
test.user_data[3] = 0x5204u;
test.user_data[4] = count;
const std::array<u32, 4> output{slot * 13u, 16u << 16u, 6u * wave, 0x5204u};
std::copy(output.begin(), output.end(), test.user_data.begin() + 16u);
test.compute_info.wave_size = wave;
test.compute_info.threads_num[0] = wave;
test.compute_info.threads_num[1] = test.compute_info.threads_num[2] = 1u;
test.compute_info.thread_ids_num = 1;
test.compute_info.lds_size_dwords = 0;
auto &code = test.code;
code.insert(code.end(), {EncodeSMovB32(32, InlineU32(0)), EncodeSop1(0x04, 36, 126)});
AppendVMovLiteral(&code, 13, 0x80000000u);
code.insert(code.end(), {EncodeVopc(0xc2, InlineU32(6), 0),
EncodeVop2(0x01, 12, Vgpr(0), 13)});
AppendVMovLiteral(&code, 14, 0x55555556u);
code.insert(code.end(), {EncodeVopc(0xc2, InlineU32(6), 0),
EncodeVop2(0x01, 15, Vgpr(0), 14)});
AppendVMovLiteral(&code, 10, 0xa5c39b72u);
const auto header = code.size();
code.push_back(EncodeSopc(0x04, 32, 4));
const auto exit_branch = code.size();
code.push_back(0);
code.insert(code.end(), {EncodeSop2(0x26, 33, 32, 255), 192u,
EncodeSmem0(0x0a, 8, 0), EncodeSmem1(160u, 33),
EncodeSmem0(0x0a, 12, 0), EncodeSmem1(176u, 33), EncodeSopp(0x0c, 0),
EncodeMubuf0(0x83, 0, true, false), EncodeMubuf1(4, 2, 12),
EncodeMubuf0(0x83, 0, true, false, true), EncodeMubuf1(6, 2, 12),
EncodeSopp(0x0c, 0), EncodeSop2(0x1e, 34, 32, InlineU32(std::countr_zero(wave))),
EncodeVop2(0x25, 20, 34, 0),
EncodeMubuf0(0x1e, 0, true, false), EncodeMubuf1(4, 4, 20)});
AppendSMovLiteral(&code, 126, 0x55555555u);
AppendSMovLiteral(&code, 127, wave == 64 ? 0x55555555u : 0u);
code.insert(code.end(), {EncodeMubuf0(0x19, 0, true, false, true), EncodeMubuf1(10, 3, 15),
EncodeSop1(0x04, 126, 36), EncodeSop2(0x02, 32, 32, InlineU32(1))});
code.push_back(EncodeSopp(0x02, header - code.size() - 1u));
code[exit_branch] = EncodeSopp(0x04, code.size() - exit_branch - 1u);
AppendEnd(&code);
for (u32 row = 0; row < 6u; ++row) {
const u32 src = slot * (row + 1u) + (row + 1u) * 4u;
const u32 dst = slot * (row + 7u) + row + 1u;
const auto format = count == 6u && row == 0u ? F::k32_32_32_32Float :
count == 6u && row == 1u ? F::k8_8_8_8UNorm : F::k16_16_16_16Float;
const bool empty_one = row == (count == 3u ? 0u : 2u);
const u32 swizzle = empty_one ? DstSel(1, 4, 1, 4) : DstSel(4, 5, 6, 7);
std::array<u32, 4> input{empty_one ? 0u : src, 16u << 16u,
empty_one ? 0u : 2u, static_cast<u32>(format) << 12u | swizzle};
if (row == 3u) input.fill(0u);
const std::array<u32, 4> destination{dst, 3u << 16u, 16u, 0x5204u};
std::copy(input.begin(), input.end(), test.initial.begin() + table / 4u + row * 48u + 40u);
std::copy(destination.begin(), destination.end(), test.initial.begin() + table / 4u + row * 48u + 44u);
for (u32 record = 0; record < 2u; ++record) {
const auto word = src / 4u + record * 4u;
if (format == F::k32_32_32_32Float) {
const std::array<u32, 4> data{0x3f800000u, 0x3f000000u, 0u, 0xc0000000u};
std::copy(data.begin(), data.end(), test.initial.begin() + word);
} else if (format == F::k8_8_8_8UNorm) test.initial[word] = 0xffffffffu;
else {
test.initial[word] = 0x38003c00u;
test.initial[word + 1u] = 0xc0000000u;
}
}
}
test.expected = test.initial;
auto expected_bytes = std::as_writable_bytes(std::span(test.expected));
for (u32 row = 0; row < count; ++row) {
for (u32 lane = 0; lane < wave; ++lane) {
std::array<u32, 2> value{};
if (row == (count == 3u ? 0u : 2u)) value = {0x3c00u, 0x3c00u};
else if (row != 3u && lane < 2u && !(row == 5u && lane == 1u))
value = count == 6u && row == 1u ? std::array<u32, 2>{0x3c003c00u, 0x3c003c00u}
: std::array<u32, 2>{0x38003c00u, 0xc0000000u};
const u32 out = slot * 13u / 4u + (row * wave + lane) * 4u;
for (u32 word = 0; word < 4u; ++word) test.expected[out + word] = value[word % 2u];
if (lane < 16u && lane != 6u && (lane & 1u) == 0u)
expected_bytes[slot * (row + 7u) + row + 1u + lane * 3u] = std::byte{0xc3};
}
}
if (count == 2u || count == 3u) {
test.required_spirv = {"StorageBufferArrayNonUniformIndexing"};
test.forbidden_spirv = {"OpSwitch", "PhysicalStorageBuffer", "convert_indirect_buffer_format",
"indirect_buffer_format_geometry"};
} else if (count == 6u) test.required_spirv = {"OpSwitch"};
auto compiled = CompileCase(test, vulkan.SubgroupSize());
Require(test.name, "mixed native aliases", count != 6u ||
(compiled.program.info.buffer_u8 && compiled.program.info.buffer_u16),
"mixed candidates lost an 8-bit or 16-bit native binding");
u32 selections = 0;
for (auto *block : compiled.program.blocks)
for (const auto &inst : *block) selections += inst.GetOpcode() == ValueOpcode::GetBufferResourceIndex;
Require(test.name, "shared loop selection", count == 0u ? selections <= 2u : selections == 2u,
"repeated formatted loads repeated the descriptor lookup");
for (u32 resource = 0; resource < compiled.resources.buffers.size(); ++resource) {
ShaderBufferResource descriptor{};
std::copy_n(compiled.resources.buffers[resource].dwords.begin(), 4u, descriptor.fields);
const u32 address = descriptor.Base48();
const u32 prefix = address % alignment;
u32 bytes = std::max(4u, static_cast<u32>(descriptor.GetSize()));
if (address == slot * 5u + 20u) bytes = 24u;
if (address == slot * 6u + 24u) bytes = 23u;
test.storage_buffer_ranges.emplace_back(address - prefix, bytes + prefix);
const u32 binding = compiled.program.info.buffers[resource].descriptor_index;
if (binding != UINT32_MAX)
compiled.packed_user_data[compiled.program.bindings.memory_offset_dword + binding / 4u] |=
prefix << ((binding % 4u) * 8u);
}
auto output_buffer = vulkan.CreateStorageBuffer(test.name, test.initial, test.initial.size());
vulkan.Dispatch(test, compiled, output_buffer);
CompareWords(test, "row keys, mixed formats, empty-One, partial payloads, prefixes and byte canaries",
test.expected, vulkan.ReadBuffer(test.name, output_buffer, test.expected.size()));
vulkan.DestroyBuffer(&output_buffer);
}
}
std::printf("[compute] %-32s ok\n", "BoundedRecordBufferAccess");
}
void CheckGpuSelectedRecordBufferAccess(VulkanHarness &vulkan) {
const u32 alignment = vulkan.RuntimeContext().GetPhysicalDeviceProperties()
.limits.minStorageBufferOffsetAlignment;
const u32 slot = (4096u + alignment - 1u) / alignment * alignment;
constexpr u32 selectors = 256u, table = 512u;
// Multiplication by 192 wraps: these keys reach congruence slots 1,4,0,3,2,1,OOB,7,5.
constexpr std::array keys{0u, 1u, 0x15555555u, 0x15555556u, 0x2aaaaaabu,
0x40000000u, UINT32_MAX, 2u, 0x2aaaaaacu};
for (const u32 components : {0u, 2u, 3u, 4u})
for (const u32 wave : {32u, 64u}) {
for (const u32 table_bytes : {0u, 536u, 544u}) {
TestCase test;
test.name = wave == 32u ? "GpuSelectedRecordBufferAccess32" : "GpuSelectedRecordBufferAccess64";
test.initial.assign((slot * 13u + keys.size() * wave * 16u) / 4u, 0xdeadbeefu);
test.has_user_data = test.has_compute_info = true;
const std::array<u32, 8> descriptors{selectors, 12u << 16u, keys.size(), 0x5204u,
table, 0u, table_bytes, 0x5204u};
std::copy(descriptors.begin(), descriptors.end(), test.user_data.begin());
const std::array<u32, 4> output{slot * 13u, 16u << 16u, u32(keys.size()) * wave, 0x5204u};
std::copy(output.begin(), output.end(), test.user_data.begin() + 16u);
test.compute_info.wave_size = wave;
test.compute_info.threads_num[0] = wave;
test.compute_info.threads_num[1] = test.compute_info.threads_num[2] = 1u;
test.compute_info.thread_ids_num = 1;
test.compute_info.group_id[0] = true;
test.compute_info.workgroup_register = 24u;
test.compute_info.lds_size_dwords = 0;
test.dispatch_x = keys.size();
test.bda_mappings = {{0u, 0u}};
auto &code = test.code;
// A vector load keeps this regression on the unbounded affine-table path.
code.insert(code.end(), {EncodeVop1(0x01, 2, 24),
EncodeMubuf0(0x0c, 0, true, false), EncodeMubuf1(3, 0, 2),
EncodeSopp(0x0c, 0), EncodeVop1(0x02, 32, Vgpr(3)),
EncodeSop2(0x26, 33, 32, 255), 192u,
EncodeSop2(0x02, 33, 33, 255), 96u});
if (components == 0u)
code.insert(code.end(), {EncodeSmem0(0x0a, 36, 2), EncodeSmem1(144u, 33)});
code.insert(code.end(), {
EncodeSmem0(0x0a, 8, 2), EncodeSmem1(160u, 33),
EncodeSmem0(0x0a, 12, 2), EncodeSmem1(176u, 33), EncodeSopp(0x0c, 0)});
if (components == 0u) {
code.insert(code.end(), {
EncodeMubuf0(0x83, 0, true, false), EncodeMubuf1(4, 2, 0),
EncodeMubuf0(0x0a, 0, true, false), EncodeMubuf1(6, 9, 0),
EncodeMubuf0(0x00, 0, true, false), EncodeMubuf1(7, 9, 0)});
} else {
for (u32 component = components; component < 4u; ++component)
AppendVMovLiteral(&code, 4u + component, 0x13579bdfu);
code.insert(code.end(), {EncodeMubuf0(components - 1u, 0, true, false, true),
EncodeMubuf1(4, 2, 0)});
}
code.insert(code.end(), {
EncodeSopp(0x0c, 0), EncodeSop2(0x1e, 34, 24, InlineU32(std::countr_zero(wave))),
EncodeVop2(0x25, 20, 34, 0),
EncodeMubuf0(0x1e, 0, true, false), EncodeMubuf1(4, 4, 20)});
AppendVMovLiteral(&code, 10, 0xa5c39b72u);
AppendSMovLiteral(&code, 126, 0x55555555u);
AppendSMovLiteral(&code, 127, wave == 64u ? 0x55555555u : 0u);
code.insert(code.end(), {EncodeMubuf0(0x19, 0, true, false, true), EncodeMubuf1(10, 3, 20)});
AppendEnd(&code);
for (u32 row = 0; row < 6u; ++row) {
std::array<u32, 8> record{slot * (row + 1u), 8u << 16u, 1u,
static_cast<u32>(Prospero::BufferFormat::k16_16_16_16Float) << 12u | DstSel(4, 5, 6, 7),
slot * (row + 7u), 1u << 16u, u32(keys.size()) * wave, 0x5204u};
if (components != 0u) {
constexpr std::array formats{Prospero::BufferFormat::k16_16Float,
Prospero::BufferFormat::k16_16_16_16SNorm, Prospero::BufferFormat::k10_10_10_2SNorm};
const std::array swizzles{DstSel(4, 5, 0, 1), DstSel(7, 4, 1, 0), DstSel(5, 4, 7, 6)};
record[3] = (u32(formats[row % 3u]) << 12u) | swizzles[row % 3u];
if (row == 3u) std::fill_n(record.begin(), 4u, 0u);
}
std::copy(record.begin(), record.end(), test.initial.begin() + (table + 192u + row * 64u) / 4u);
// BDA loads use a separate tuple, as the captured shader's record members do.
if (components == 0u) {
std::copy_n(record.begin(), 4u, test.initial.begin() + (table + 176u + row * 64u) / 4u);
test.initial[slot * (row + 1u) / 4u] = ((0x3800u + row) << 16u) | (0x3c00u + row);
test.initial[slot * (row + 1u) / 4u + 1u] = ((0xc000u + row) << 16u) | (0x4000u + row);
} else {
constexpr std::array<std::array<u32, 2>, 3> payload{{
{0x3800bc00u, 0u}, {0x7fff8000u, 0x80010000u}, {0xc007fe00u, 0u}}};
std::copy(payload[row % 3u].begin(), payload[row % 3u].end(),
test.initial.begin() + slot * (row + 1u) / 4u);
}
}
// Compile against poison selectors. A separate GPU dispatch writes the actual keys.
auto compiled = CompileCase(test, vulkan.SubgroupSize());
if (table_bytes == 0u || components != 0u)
test.forbidden_spirv = {"convert_indirect_buffer_format", "load_indirect_buffer_",
"indirect_buffer_format_geometry"};
else
test.required_spirv = {"get_bda_pointer", "convert_indirect_buffer_format"};
CheckSpirvEmission(test, compiled.spirv);
u32 bda_function = 0, bda_calls = 0;
for (size_t offset = 5; offset < compiled.spirv.size();) {
const auto words = std::span(compiled.spirv).subspan(offset, compiled.spirv[offset] >> 16u);
const auto opcode = static_cast<spv::Op>(words[0] & 0xffffu);
if (opcode == spv::OpName &&
std::string_view(reinterpret_cast<const char *>(words.data() + 2)) == "get_bda_pointer")
bda_function = words[1];
if (opcode == spv::OpFunctionCall && words[3] == bda_function) ++bda_calls;
offset += words.size();
}
Require(test.name, "empty descriptor table specialization", (bda_calls == 0u) == (table_bytes == 0u || components != 0u),
"empty table retained BDA reads or nonempty table lost GPU descriptor loads");
for (const auto &descriptor : compiled.resources.buffers) {
ShaderBufferResource resource{};
std::copy_n(descriptor.dwords.begin(), 4u, resource.fields);
test.storage_buffer_ranges.emplace_back(resource.Base48(),
components != 0u && resource.Base48() == slot * 5u ? 6u : std::max(4u, u32(resource.GetSize())));
}
TestCase producer;
producer.name = "GpuRecordSelectorProducer";
producer.initial = test.initial;
producer.has_compute_info = true;
producer.compute_info.threads_num[0] = producer.compute_info.threads_num[1] =
producer.compute_info.threads_num[2] = 1u;
producer.compute_info.wave_size = 32u;
producer.compute_info.lds_size_dwords = 0;
for (u32 group = 0; group < keys.size(); ++group) {
AppendVMovLiteral(&producer.code, 1, keys[group]);
AppendVMovLiteral(&producer.code, 2, selectors + group * 12u);
AppendBufferStoreDword(&producer.code, 1, 2);
}
AppendEnd(&producer.code);
const auto producer_shader = CompileCase(producer, vulkan.SubgroupSize());
test.expected = test.initial;
auto bytes = std::as_writable_bytes(std::span(test.expected));
for (u32 group = 0; group < keys.size(); ++group) {
test.expected[selectors / 4u + group * 3u] = keys[group];
const u32 dynamic = keys[group] * 192u + 96u;
const u32 row = dynamic / 64u;
for (u32 lane = 0; lane < wave; ++lane) {
const u32 out = slot * 13u / 4u + (group * wave + lane) * 4u;
const bool source = uint64_t{dynamic} + 160u + 16u <= table_bytes && lane == 0u;
if (components == 0u) {
test.expected[out] = source ? test.initial[slot * (row + 1u) / 4u] : 0u;
test.expected[out + 1u] = source ? test.initial[slot * (row + 1u) / 4u + 1u] : 0u;
const bool bda_source = uint64_t{dynamic} + 144u + 16u <= table_bytes && lane == 0u;
test.expected[out + 2u] = bda_source ? 0x3c00u + row : 0u;
test.expected[out + 3u] = bda_source ? 0x3f800000u + (row << 13u) : 0u;
} else {
constexpr std::array<std::array<u32, 4>, 3> values{{
{0xbf800000u, 0x3f000000u, 0u, 0x3f800000u},
{0xbf800000u, 0xbf800000u, 0x3f800000u, 0u},
{0x3f800000u, 0xbf800000u, 0xbf800000u, 0u}}};
constexpr std::array<std::array<u32, 4>, 3> defaults{{
{0u, 0u, 0u, 0x3f800000u}, {0u, 0u, 0x3f800000u, 0u}, {0u, 0u, 0u, 0u}}};
const bool descriptor = uint64_t{dynamic} + 160u + 16u <= table_bytes && row != 3u;
for (u32 component = 0; component < 4u; ++component)
test.expected[out + component] = component >= components ? 0x13579bdfu :
!descriptor ? 0u : source && row != 4u ? values[row % 3u][component] : defaults[row % 3u][component];
}
if (uint64_t{dynamic} + 176u + 16u <= table_bytes && (lane & 1u) == 0u)
bytes[slot * (row + 7u) + group * wave + lane] = std::byte{0xc3};
}
}
auto buffer = vulkan.CreateStorageBuffer(test.name, test.initial, test.initial.size(), true);
vulkan.Dispatch(producer, producer_shader, buffer);
vulkan.Dispatch(test, compiled, buffer);
CompareWords(test, "GPU selectors, wrapped offsets, partial descriptors, OOB and masked byte stores",
test.expected, vulkan.ReadBuffer(test.name, buffer, test.expected.size()));
vulkan.DestroyBuffer(&buffer);
}
}
std::printf("[compute] %-32s ok\n", "GpuSelectedRecordBufferAccess");
}
void CheckGuardedWaterfallBufferRecords(VulkanHarness &vulkan) {
const u32 alignment = vulkan.RuntimeContext().GetPhysicalDeviceProperties()
.limits.minStorageBufferOffsetAlignment;
const u32 slot = (4096u + alignment - 1u) / alignment * alignment;
constexpr u32 selectors = 256u, table = 512u;
enum class Guard { AndNot, DirectGt, NarrowingLoop };
for (const auto guard : {Guard::AndNot, Guard::DirectGt, Guard::NarrowingLoop})
for (const auto [wave, upper_only] : {std::pair{32u, false}, std::pair{64u, false},
std::pair{64u, true}}) {
const bool direct_gt = guard == Guard::DirectGt;
const bool outer_loop = guard == Guard::NarrowingLoop;
const u32 iterations = outer_loop ? 2u : 1u;
const std::string name = std::string(direct_gt ? "GuardedWaterfallGtRecords" : outer_loop
? "GuardedWaterfallLoopRecords" : "GuardedWaterfallRecords") +
(upper_only ? "Upper64" : std::to_string(wave));
TestCase test;
test.name = name.c_str();
test.initial.assign((slot * 3u + wave * iterations * 16u + 16u) / 4u, 0xdeadbeefu);
std::fill_n(test.initial.begin() + table / 4u, 976u / 4u, 0u);
constexpr std::array poison{0x012b6890u, 0x501502f9u, 0x493ba30eu, 0x3ea2f352u};
std::copy(poison.begin(), poison.end(), test.initial.begin() + (table + 480u) / 4u);
test.has_user_data = test.has_compute_info = true;
const std::array<u32, 8> descriptors{table, 488u << 16u, 2u, 0x5204u,
selectors, 4u << 16u, wave, 0x5204u};
std::copy(descriptors.begin(), descriptors.end(), test.user_data.begin());
const std::array output{slot * 3u, 16u << 16u, wave * iterations, 0x5204u};
std::copy(output.begin(), output.end(), test.user_data.begin() + 16u);
for (u32 row = 0; row < 2u; ++row) {
const std::array descriptor{slot * (row + 1u), 12u << 16u, wave,
(u32(Prospero::BufferFormat::k32_32_32Float) << 12u) | DstSel(4, 5, 6, 1)};
std::copy(descriptor.begin(), descriptor.end(),
test.initial.begin() + (table + row * 488u + 216u) / 4u);
for (u32 lane = 0; lane < wave; ++lane)
for (u32 component = 0; component < 3u; ++component)
test.initial[slot * (row + 1u) / 4u + lane * 3u + component] =
0x3f800000u + (row << 20u) + (lane << 12u) + (component << 8u);
}
test.compute_info.wave_size = wave;
test.compute_info.threads_num[0] = wave;
test.compute_info.threads_num[1] = test.compute_info.threads_num[2] = 1u;
test.compute_info.thread_ids_num = 1;
test.compute_info.lds_size_dwords = 0;
auto &code = test.code;
code.push_back(EncodeVop1(0x01, 30, Vgpr(0)));
if (upper_only) {
AppendSMovLiteral(&code, 126, 0u);
AppendSMovLiteral(&code, 127, UINT32_MAX);
}
if (outer_loop) code.insert(code.end(), {EncodeVop2(0x1b, 10, InlineU32(8), 30),
EncodeVopc(0xd2, InlineU32(0), 10)});
code.insert(code.end(), {EncodeSop1(0x04, 20, 126),
EncodeMubuf0(0x0c, 0, true, false), EncodeMubuf1(5, 1, 30), EncodeSopp(0x0c, 0)});
if (outer_loop) {
// Save the initial-EXEC comparison before the loop, as in native 9d4c.
code.insert(code.end(), {EncodeVopc(0xc3, 2, 5), EncodeSop1(0x04, 52, 106)});
AppendSMovLiteral(&code, 36, 0u);
}
const auto outer = code.size();
if (direct_gt) code.push_back(EncodeVopc(0xd4, 2, 5)); // Keep N > local key in EXEC.
else if (outer_loop) code.insert(code.end(), {
EncodeSop2(0x1e, 37, 36, InlineU32(std::countr_zero(wave))),
EncodeVop2(0x25, 31, 37, 30), EncodeSop1(0x37, 50, 52)});
else code.insert(code.end(), {EncodeVopc(0xc3, 2, 5),
EncodeSop1(0x37, 52, 106)}); // Exclude N <= local key.
const auto empty = code.size();
code.push_back(0u);
code.push_back(EncodeSop1(0x04, 54, 126));
const auto loop = code.size();
code.push_back(EncodeSop1(0x14, 34, 54));
AppendVop3(&code, 0x360, 32, Vgpr(5), 34);
code.insert(code.end(), {EncodeVopc(0xc2, 32, 5), EncodeSop1(0x24, 58, 106)});
const auto skip = code.size();
code.push_back(0u);
code.insert(code.end(), {EncodeSop2(0x26, 33, 32, 255), 488u,
EncodeSmem0(0x0a, 12, 0), EncodeSmem1(216u, 33), EncodeSopp(0x0c, 0),
EncodeMubuf0(0x02, 0, true, false, true), EncodeMubuf1(6, 3, 30),
EncodeSopp(0x0c, 0), EncodeMubuf0(0x1f, 0, true, false),
EncodeMubuf1(6, 4, outer_loop ? 31u : 30u)});
code[skip] = EncodeSopp(0x08, code.size() - skip - 1u);
code.insert(code.end(), {EncodeSop2(0x15, 54, 54, 106), EncodeSop1(0x04, 126, 58)});
code.push_back(EncodeSopp(0x05, static_cast<int16_t>(loop - code.size() - 1u)));
code[empty] = EncodeSopp(0x08, code.size() - empty - 1u);
if (outer_loop) {
code.push_back(EncodeSop1(0x04, 126, 50));
AppendVMovLiteral(&code, 9, 0x13579bdfu);
code.insert(code.end(), {EncodeMubuf0(0x1c, 12u, true, false), EncodeMubuf1(9, 4, 31),
EncodeVop2(0x1b, 10, InlineU32(4), 30), EncodeVopc(0xd2, InlineU32(0), 10),
EncodeSop2(0x00, 36, 36, InlineU32(1)), EncodeSopc(0x0a, 36, InlineU32(iterations))});
code.push_back(EncodeSopp(0x05, static_cast<int16_t>(outer - code.size() - 1u)));
}
code.push_back(EncodeSop1(0x04, 126, 20));
AppendVMovLiteral(&code, 9, 0x76543210u);
code.insert(code.end(), {EncodeMubuf0(0x1c, 12u, true, false), EncodeMubuf1(9, 4, 30)});
AppendEnd(&code);
test.decoded_counts = {{direct_gt ? "V_CMPX_GT_U32" : "V_CMP_LE_U32", 1u}};
if (outer_loop) test.decoded_counts.insert(test.decoded_counts.end(),
{{"V_CMPX_EQ_U32", 2u}, {"S_CBRANCH_SCC1", 2u}});
test.required_spirv = {"OpLoopMerge", "OpGroupNonUniformShuffle", "Volatile"};
test.forbidden_spirv = {"load_indirect_buffer_", "indirect_buffer_format_geometry"};
const auto compiled = CompileCase(test, vulkan.SubgroupSize());
for (const auto &descriptor : compiled.resources.buffers) {
ShaderBufferResource resource{};
std::copy_n(descriptor.dwords.begin(), 4u, resource.fields);
test.storage_buffer_ranges.emplace_back(resource.Base48(), std::max(4u, u32(resource.GetSize())));
}
TestCase producer;
producer.name = "WaterfallRecordKeyProducer";
producer.initial = test.initial;
producer.has_compute_info = true;
producer.compute_info.wave_size = 32u;
producer.compute_info.threads_num[0] = producer.compute_info.threads_num[1] =
producer.compute_info.threads_num[2] = 1u;
producer.compute_info.lds_size_dwords = 0;
test.expected = test.initial;
// Exclude key == N and the key that wraps to the unsupported descriptor at byte 480.
constexpr std::array keys{0u, 1u, 2u, 0x029f79b5u};
for (u32 lane = 0; lane < wave; ++lane) {
const u32 key = keys[lane % keys.size()];
AppendVMovLiteral(&producer.code, 1, key);
AppendVMovLiteral(&producer.code, 2, selectors + lane * 4u);
AppendBufferStoreDword(&producer.code, 1, 2);
test.expected[selectors / 4u + lane] = key;
if (upper_only && lane < 32u) continue;
if (outer_loop && (lane & 8u) != 0u) continue;
for (u32 iteration = 0; iteration < iterations; ++iteration) {
if (iteration != 0u && (lane & 4u) != 0u) continue;
const u32 out = slot * 3u / 4u + (iteration * wave + lane) * 4u;
if (key < 2u)
std::copy_n(test.initial.begin() + slot * (key + 1u) / 4u + lane * 3u, 3u,
test.expected.begin() + out);
if (outer_loop) test.expected[out + 3u] = 0x13579bdfu;
}
test.expected[slot * 3u / 4u + lane * 4u + 3u] = 0x76543210u;
}
AppendEnd(&producer.code);
const auto producer_shader = CompileCase(producer, vulkan.SubgroupSize());
auto buffer = vulkan.CreateStorageBuffer(test.name, test.initial, test.initial.size());
vulkan.Dispatch(producer, producer_shader, buffer);
vulkan.Dispatch(test, compiled, buffer);
CompareWords(test, "same-lane bound, narrowing loop, key == N, poison, EXEC restoration and canaries",
test.expected, vulkan.ReadBuffer(test.name, buffer, test.expected.size()));
vulkan.DestroyBuffer(&buffer);
}
std::printf("[compute] %-32s ok\n", "GuardedWaterfallBufferRecords");
}
void CheckGpuSelectedFormattedRecordStore(VulkanHarness &vulkan) {
constexpr u32 selectors = 128, table = 256, stride = 592, member = 448;
const u32 alignment = vulkan.RuntimeContext().GetPhysicalDeviceProperties()
.limits.minStorageBufferOffsetAlignment;
const u32 slot = (4096u + alignment - 1u) / alignment * alignment;
for (const bool wide : {false, true})
for (const u32 wave : {32u, 64u}) {
const std::array keys{wide ? 31u : 5u, 0u, 3u, 1u, 4u, 2u, 6u};
const auto selector_index = [wide](u32 group) { return wide ? group ^ 1u : group; };
TestCase test;
test.name = wide ? "GpuSelectedFormattedRgba16Store" : "GpuSelectedFormattedRecordStore";
test.initial.assign((slot * 7u + keys.size() * wave * 4u) / 4u, 0xdeadbeefu);
auto bytes = std::as_writable_bytes(std::span(test.initial));
if (wide) std::fill_n(test.initial.begin() + table / 4u, 32u * 4u, 0u);
for (u32 row = 0; row < 6u; ++row) {
if (wide && row != 0u && row != 3u) continue;
const std::array descriptor{slot * (row + 1u), (wide ? 8u : 1u) << 16u, 20u,
wide ? (u32(Prospero::BufferFormat::k16_16_16_16Float) << 12u) | DstSel(4, 5, 6, 7)
: (u32(Prospero::BufferFormat::k8SInt) << 12u) | DstSel(4, 0, 0, 0)};
std::copy(descriptor.begin(), descriptor.end(),
test.initial.begin() + (table + (wide ? row * 16u : row * stride + member)) / 4u);
for (u32 element = 0; element < 20u; ++element) {
if (wide) {
test.initial[slot * (row + 1u) / 4u + element * 2u] = 0x40003c00u;
test.initial[slot * (row + 1u) / 4u + element * 2u + 1u] = 0x38003400u;
} else bytes[slot * (row + 1u) + element] = std::byte(0x80u + row + element);
}
}
auto records = MakeStructuredStorageBufferData(stride, 6u);
records[0] = table;
std::copy_n(records.begin(), 4, test.user_data.begin() + 32u);
if (wide) { test.user_data[32] = table - 48u; test.user_data[33] = 0u; }
auto selector_buffer = MakeStructuredStorageBufferData(0u, (keys.size() + u32(wide)) * 4u);
selector_buffer[0] = selectors;
std::copy_n(selector_buffer.begin(), 4, test.user_data.begin() + 4u);
auto output = MakeStructuredStorageBufferData(0u, keys.size() * wave * 4u);
output[0] = slot * 7u;
std::copy_n(output.begin(), 4, test.user_data.begin() + 48u);
test.has_user_data = test.has_compute_info = true;
test.bda_mappings = {{0u, 0u}};
test.compute_info.wave_size = wave;
test.compute_info.threads_num[0] = wave;
test.compute_info.threads_num[1] = test.compute_info.threads_num[2] = 1u;
test.compute_info.thread_ids_num = 1u;
test.compute_info.group_id[0] = true;
test.compute_info.workgroup_register = 44u;
test.dispatch_x = keys.size();
auto &code = test.code;
// Keep the finite32-key proof independent of workgroup-prefix materialization:
// this fixture changes GPU keys after compiling its complete candidate domain.
if (wide) code.push_back(EncodeSop2(0x12, 23, 44, InlineU32(1)));
code.push_back(EncodeSop2(0x1e, 23, wide ? 23u : 44u, InlineU32(2)));
if (wide) {
code.insert(code.end(), {EncodeSmem0(0x08, 22, 2), EncodeSmem1(0, 23),
EncodeSop2(0x27, 22, 22, 255), (5u << 16u) | 16u,
EncodeSop2(0x1e, 23, 22, InlineU32(4)),
EncodeSmem0(0x02, 8, 16), EncodeSmem1(48, 23)});
} else {
code.insert(code.end(), {EncodeVop1(0x01, 30, 23), EncodeMubuf0(0x0c),
EncodeMubuf1(4, 1, 30), EncodeVop2(0x16, 6, InlineU32(16), 4)});
AppendVop3(&code, 0x360, 22, Vgpr(6), InlineU32(0));
code.insert(code.end(), {EncodeSop2(0x26, 23, 22, 255), stride,
EncodeSmem0(0x0a, 8, 16), EncodeSmem1(member, 23)});
}
code.insert(code.end(), {EncodeMubuf0(0x00, 0, true, false, true), EncodeMubuf1(4, 2, 0),
EncodeSop2(0x1e, 34, 44, InlineU32(std::countr_zero(wave))),
EncodeVop2(0x25, 30, 34, 0), EncodeVop2(0x1a, 30, InlineU32(2), 30)});
AppendBufferStoreDword(&code, 4, 30);
if (wide) {
// The existing direct RGBA16 fixture proves these rounded FP32-to-FP16 values.
for (const auto [vgpr, bits] : std::array<std::pair<u32, u32>, 4>{{
{10u, 0x3f807e63u}, {11u, 0xc0000000u}, {12u, 0x3f000000u}, {13u, 0x3e800000u}}})
AppendVMovLiteral(&code, vgpr, bits);
} else {
code.push_back(EncodeVop2(0x1a, 10, InlineU32(5), 0));
code.insert(code.end(), {EncodeVop2(0x25, 10, 255, 10), u32(-200)});
}
AppendSMovLiteral(&code, 126, 0x55555555u);
AppendSMovLiteral(&code, 127, wave == 64u ? 0x55555555u : 0u);
// Captured e01c6000/80010f01, adjusted only to v10:13/v0 and s8:11.
code.insert(code.end(), {EncodeMubuf0(wide ? 0x07 : 0x04, 0, true, false, true),
EncodeMubuf1(10, 2, 0)});
AppendEnd(&code);
test.decoded_counts = {{"V_LSHLREV_B32", wide ? 1u : 2u},
{wide ? "BUFFER_STORE_FORMAT_XYZW" : "BUFFER_STORE_FORMAT_X", 1u}};
test.required_spirv = {"get_bda_pointer", "convert_indirect_buffer_format"};
if (!wide) test.required_spirv.push_back("StorageBufferArrayNonUniformIndexing");
test.forbidden_spirv = {"OpAtomicCompareExchange"};
const auto compiled = CompileCase(test, vulkan.SubgroupSize());
if (!wide) {
std::set<u32> nonuniform;
u32 shared_stores = 0;
for (size_t offset = 5; offset < compiled.spirv.size();) {
const auto words = std::span(compiled.spirv).subspan(offset, compiled.spirv[offset] >> 16u);
const auto opcode = static_cast<spv::Op>(words[0] & 0xffffu);
if (opcode == spv::OpDecorate && words[2] == spv::DecorationNonUniform)
nonuniform.insert(words[1]);
if (opcode == spv::OpStore && nonuniform.contains(words[1])) ++shared_stores;
offset += words.size();
}
Require(test.name, "one shared store after the null candidate",
shared_stores == (wave > vulkan.SubgroupSize() ? 2u : 1u),
"homogeneous children duplicated the native store per descriptor");
}
u32 bda_loads = 0, native_stores = 0;
for (const auto *block : compiled.program.blocks)
for (const auto &inst : *block) {
using namespace ShaderRecompiler::IR;
if (inst.GetOpcode() != ValueOpcode::LoadBufferU32 &&
inst.GetOpcode() != (wide ? ValueOpcode::StoreBufferU32x4 : ValueOpcode::StoreBufferU32)) continue;
const auto &memory = compiled.program.memory_info[inst.Flags<MemoryFlags>().index];
if (inst.GetOpcode() == ValueOpcode::LoadBufferU32)
bda_loads += memory.kind == ResourceKind::IndirectBuffer && memory.formatted;
else
native_stores += memory.kind == ResourceKind::Buffer && memory.formatted;
}
Require(test.name, "shared scalar descriptor reads", bda_loads == 1u && native_stores == 1u,
"native store planning replaced the BDA read or bypassed formatted conversion");
for (const auto &value : compiled.resources.buffers) {
ShaderBufferResource descriptor{};
std::copy_n(value.dwords.begin(), 4u, descriptor.fields);
auto range = std::max(4u, u32(descriptor.GetSize()));
if (wide && descriptor.Base48() == slot * 4u) range -= 11u; // Last active record is partial.
test.storage_buffer_ranges.emplace_back(descriptor.Base48(), range);
}
TestCase producer;
producer.name = "FormattedRecordSelectorProducer";
producer.initial = test.initial;
producer.has_compute_info = true;
producer.compute_info.threads_num[0] = producer.compute_info.threads_num[1] =
producer.compute_info.threads_num[2] = 1u;
producer.compute_info.wave_size = 32u;
for (u32 group = 0; group < keys.size(); ++group) {
AppendVMovLiteral(&producer.code, 1, (keys[group] << 16u) | 0xbeefu);
AppendVMovLiteral(&producer.code, 2, selectors + selector_index(group) * 4u);
AppendBufferStoreDword(&producer.code, 1, 2);
}
AppendEnd(&producer.code);
const auto producer_shader = CompileCase(producer, vulkan.SubgroupSize());
test.expected = test.initial;
auto expected_bytes = std::as_writable_bytes(std::span(test.expected));
for (u32 group = 0; group < keys.size(); ++group) {
const auto row = keys[group];
test.expected[selectors / 4u + selector_index(group)] = (row << 16u) | 0xbeefu;
for (u32 lane = 0; lane < wave; ++lane) {
const bool valid = wide ? row == 0u || row == 3u : row < 6u;
test.expected[slot * 7u / 4u + group * wave + lane] =
valid && lane < 20u ? (wide ? 0x3f800000u : u32(int32_t(0x80u + row + lane) - 256)) : 0u;
if (valid && lane < (wide && row == 3u ? 18u : 20u) && (lane & 1u) == 0u) {
if (wide) {
test.expected[slot * (row + 1u) / 4u + lane * 2u] = 0xc0003c04u;
test.expected[slot * (row + 1u) / 4u + lane * 2u + 1u] = 0x34003800u;
} else expected_bytes[slot * (row + 1u) + lane] =
std::byte(u32(std::clamp(int32_t(lane * 32u) - 200, -128, 127)) & 255u);
}
}
}
auto buffer = vulkan.CreateStorageBuffer(test.name, test.initial, test.initial.size(), true);
vulkan.Dispatch(producer, producer_shader, buffer);
vulkan.Dispatch(test, compiled, buffer);
CompareWords(test, "shared BDA reads, formatted conversion, EXEC, OOB and neighboring canaries",
test.expected, vulkan.ReadBuffer(test.name, buffer, test.expected.size()));
vulkan.DestroyBuffer(&buffer);
std::printf("[compute] %s wave=%u ok\n", test.name, wave);
}
}
void CheckFiniteBufferSelection(VulkanHarness &vulkan) {
for (const u32 wave : {32u, 64u}) {
TestCase test;
test.name = wave == 32 ? "FiniteBufferSelection32" : "FiniteBufferSelection64";
const u32 first_lane = wave == 32 ? 1u : 33u;
// Water Lighting's masked V_CNDMASK -> READFIRSTLANE -> S_LOAD_DWORDX4 -> store.
test.code = {EncodeSMovB32(126, InlineU32(wave == 32 ? 6u : 0u)),
EncodeSMovB32(127, InlineU32(wave == 64 ? 6u : 0u)),
EncodeMubuf0(0x0cu, 0, true, false), EncodeMubuf1(20, 2, 0),
EncodeSopp(0x0cu, 0)};
AppendVMovU32(&test.code, 4, 0);
test.code.push_back(EncodeSopc(0x0au, 2, InlineU32(1)));
const auto skip = test.code.size();
test.code.push_back(0);
for (u32 key = 1; key < 5u; ++key) {
AppendVMovU32(&test.code, 22, key);
test.code.push_back(EncodeVopc(0xc2u, InlineU32(key), 20));
test.code.push_back(EncodeVop2(0x01u, 4, Vgpr(4), 22));
}
test.code[skip] = EncodeSopp(0x05u, static_cast<u32>(test.code.size() - skip - 1u));
test.code.insert(test.code.end(), {
EncodeVop1(0x02u, 3, Vgpr(4)), EncodeSop2(0x31u, 12, 3, InlineU32(32)),
EncodeSmem0(0x02u, 4), EncodeSmem1(0, 12), EncodeSopp(0x0cu, 0),
EncodeVop2(0x25u, 1, 0xffu, 0), 0x1000u,
EncodeMubuf0(0x1cu, 0, true, false, true), EncodeMubuf1(1, 1, 0)});
AppendEnd(&test.code);
constexpr u32 table = 256u, input = 512u, output = 1024u, slot = 512u;
test.initial.assign((output + 5u * slot) / 4u, 0xdeadbeefu);
test.has_user_data = true;
test.user_data[0] = table;
test.user_data[2] = 1u;
const std::array<u32, 4> input_descriptor{input, 4u << 16u, wave, 0x5204u};
std::copy(input_descriptor.begin(), input_descriptor.end(), test.user_data.begin() + 8u);
for (u32 key = 0; key < 5u; ++key) {
const std::array<u32, 4> descriptor{output + key * slot, 4u << 16u, wave, 0x5204u};
std::copy(descriptor.begin(), descriptor.end(), test.initial.begin() + table / 4u + 8u + key * 4u);
}
test.has_compute_info = true;
test.compute_info.wave_size = wave;
test.compute_info.threads_num[0] = wave;
test.compute_info.threads_num[1] = test.compute_info.threads_num[2] = 1;
test.compute_info.lds_size_dwords = 0;
test.compute_info.thread_ids_num = 1;
const auto compiled = CompileCase(test, vulkan.SubgroupSize());
Require(test.name, "finite destination count", compiled.resources.buffers.size() == 6u,
"selector lost a destination or retained a dynamic descriptor load");
for (const auto& descriptor : compiled.resources.buffers)
test.storage_buffer_ranges.emplace_back(descriptor.dwords[0], descriptor.dwords[2] * 4u);
for (u32 key = 0; key < 5u; ++key) {
for (u32 lane = 0; lane < wave; ++lane)
test.initial[input / 4u + lane] = lane == first_lane ? key : (key + 1u) % 5u;
test.expected = test.initial;
for (const auto lane : {first_lane, first_lane + 1u})
test.expected[(output + key * slot) / 4u + lane] = 0x1000u + lane;
auto buffer = vulkan.CreateStorageBuffer(test.name, test.initial, test.initial.size());
vulkan.Dispatch(test, compiled, buffer);
const auto actual = vulkan.ReadBuffer(test.name, buffer, test.expected.size());
vulkan.DestroyBuffer(&buffer);
CompareWords(test, "first active lane, masked writes and destination canaries", test.expected, actual);
}
std::printf("[compute] %-32s ok\n", test.name);
}
}
void CheckIndirectBufferStore(VulkanHarness &vulkan) {
using namespace ShaderRecompiler::IR;
TestCase test;
test.name = "IndirectBufferStore";
test.initial.assign(64, 0xdeadbeefu);
test.storage_buffer_range_bytes = 64;
test.storage_buffer_offsets = {12, 108};
CompiledShader compiled;
auto &program = compiled.program;
program.stage = ShaderType::Compute;
program.wave_size = 32;
program.srt_plan_complete = true;
program.resource_tracking_complete = true;
program.shader_info_complete = true;
program.block_storage.push_back(std::make_unique<Block>());
auto *block = program.block_storage.back().get();
program.blocks.push_back(block);
auto &lane = block->AppendNewInst(ValueOpcode::LaneId);
auto &selector = block->AppendNewInst(ValueOpcode::BitwiseAnd32,
{Value(&lane), Value(7u)});
auto &key = block->AppendNewInst(ValueOpcode::IAdd32,
{Value(&selector), Value(8u)});
auto &selection = block->AppendNewInst(ValueOpcode::GetBufferResourceIndex, {Value(&key)});
selection.SetFlags<uint32_t>(0u);
auto &handle = block->AppendNewInst(
ValueOpcode::GetBufferResource,
{Value(&selection), Value(0u), Value(0u), Value(0u)});
handle.SetFlags<uint32_t>(0u);
auto &lane_offset = block->AppendNewInst(ValueOpcode::IMul32,
{Value(&lane), Value(4u)});
auto &overflow = block->AppendNewInst(ValueOpcode::IEqual32,
{Value(&lane), Value(6u)});
auto &offset = block->AppendNewInst(ValueOpcode::SelectU32,
{Value(&overflow), Value(UINT32_MAX), Value(&lane_offset)});
auto &index = block->AppendNewInst(ValueOpcode::SelectU32,
{Value(&overflow), Value(0x40000000u), Value(&lane)});
auto &data = block->AppendNewInst(ValueOpcode::IAdd32,
{Value(&lane), Value(0x1000u)});
auto &exec = block->AppendNewInst(ValueOpcode::INotEqual32,
{Value(&lane), Value(4u)});
auto &store = block->AppendNewInst(ValueOpcode::StoreBufferU32,
{Value(&handle), Value(0u), Value(&offset), Value(0u),
Value(&data), Value(&exec)});
program.memory_info.push_back({.kind = ResourceKind::Buffer, .offen = true});
BufferResource root;
root.packed_stride = 1;
root.written = true;
root.indirect_root = 0;
root.indirect_mapping_offset = 4;
root.indirect_search_iterations = std::bit_width(3u);
root.indirect_resources = {0u, 1u};
auto candidate = root;
candidate.indirect_search_iterations = 0;
candidate.indirect_resources.clear();
program.descriptor_sources.resize(1);
program.descriptor_sources[0].indirect_descriptor =
DescriptorSource::IndirectDescriptor{.table_stride = 16};
ShaderComputeInputInfo compute;
compute.wave_size = 32;
compute.host_subgroup_size = vulkan.SubgroupSize();
compute.threads_num[0] = 32;
compute.threads_num[1] = compute.threads_num[2] = 1;
enum class Layout { Shared, NullChild, AllNull, DifferentStride, ShiftedPushData, StorageData,
Indexed, IndexedNullChild, IndexedSingle };
for (const auto layout : {Layout::Shared, Layout::NullChild, Layout::AllNull,
Layout::DifferentStride, Layout::ShiftedPushData, Layout::StorageData,
Layout::Indexed, Layout::IndexedNullChild, Layout::IndexedSingle}) {
const bool indexed = layout == Layout::Indexed || layout == Layout::IndexedNullChild ||
layout == Layout::IndexedSingle;
const bool null_child = layout == Layout::NullChild || layout == Layout::IndexedNullChild;
const bool single = layout == Layout::AllNull || layout == Layout::IndexedSingle;
program.info.buffers = {root, candidate};
if (indexed) {
for (auto &buffer : program.info.buffers) buffer.packed_stride = 4;
}
program.descriptor_sources[0].indirect_descriptor->indexed = indexed;
program.memory_info[0].idxen = indexed;
program.memory_info[0].offen = !indexed;
store.SetArg(1, indexed ? Value(&index) : Value(0u));
store.SetArg(2, indexed ? Value(0u) : Value(&offset));
if (null_child) program.info.buffers[1].packed_stride = 0;
if (layout == Layout::DifferentStride) program.info.buffers[1].packed_stride = 2;
if (single) {
// Materialization deduplicates all null destinations into one unmapped root.
program.info.buffers.resize(1);
if (layout == Layout::AllNull) program.info.buffers[0].packed_stride = 0;
program.info.buffers[0].indirect_root = BufferResource::NoIndirectBuffer;
program.info.buffers[0].indirect_search_iterations = 0;
program.info.buffers[0].indirect_resources.clear();
}
program.shader_info_complete = false;
CollectShaderInfo(program, {.compute = &compute});
const auto push_start = layout == Layout::ShiftedPushData ? 3u :
layout == Layout::StorageData ? PushData::NoStart : 0u;
compiled.spirv = ShaderRecompiler::Spirv::EmitProgram(program, {.compute = &compute}, push_start);
Require(test.name, "candidate bindings",
program.bindings.descriptor_counts[0] == (single ? 1u : 2u) &&
program.info.buffers[0].descriptor_index == 0u &&
(single || program.info.buffers[1].descriptor_index == 1u),
"indirect store lost a buffer binding");
compiled.packed_user_data.resize(program.bindings.ShaderDataDwords());
compiled.packed_user_data[program.bindings.memory_offset_dword] = 12u | (108u << 8u);
ValidateSpirv(test.name, compiled.spirv);
test.required_spirv.clear();
test.forbidden_spirv = {"PhysicalStorageBuffer"};
if (!single && !null_child && layout != Layout::DifferentStride) {
test.required_spirv = {"StorageBufferArrayNonUniformIndexing"};
test.forbidden_spirv.push_back("OpSwitch");
std::set<u32> nonuniform;
std::vector<u32> accessed_pointers;
u32 prefix_extracts = 0;
for (size_t offset = 5; offset < compiled.spirv.size();) {
const auto words = std::span(compiled.spirv).subspan(offset, compiled.spirv[offset] >> 16u);
const auto opcode = static_cast<spv::Op>(words[0] & 0xffffu);
if (opcode == spv::OpDecorate && words[2] == spv::DecorationNonUniform)
nonuniform.insert(words[1]);
if (opcode == spv::OpArrayLength) accessed_pointers.push_back(words[3]);
if (opcode == spv::OpStore) accessed_pointers.push_back(words[1]);
if (opcode == spv::OpBitFieldUExtract) ++prefix_extracts;
offset += words.size();
}
Require(test.name, "shared store and nonuniform pointers",
prefix_extracts == 1u && accessed_pointers.size() == 2u &&
std::ranges::all_of(accessed_pointers, [&](u32 pointer) {
return nonuniform.contains(pointer);
}),
"indirect store retained static prefixes, was duplicated or lost a NonUniform pointer decoration");
} else if (single) {
test.forbidden_spirv.push_back("OpSwitch");
} else {
test.required_spirv = {"OpSwitch"};
}
CheckSpirvEmission(test, compiled.spirv);
for (const bool swapped : {false, true}) {
if (layout != Layout::Shared && layout != Layout::IndexedNullChild && swapped) continue;
// Repeated keys select two distinct logical buffers within one native allocation.
// Keys below, between and above the live keys must not select either buffer.
compiled.resources.flattened_srt = {0u, 0u, 0u, 0u, 3u,
10u, swapped ? 0u : 1u,
12u, swapped ? 1u : 0u,
14u, swapped ? 0u : 1u};
test.expected = test.initial;
for (u32 lane = 0; lane < 16u; ++lane) {
if (lane == 4u || lane == 6u) continue;
if (layout == Layout::IndexedSingle) {
test.expected[test.storage_buffer_offsets[0] / 4u + lane] = 0x1000u + lane;
continue;
}
if (lane != 2u && lane != 10u && lane != 12u && lane != 14u) continue;
const u32 resource = (lane & 7u) == 4u ? swapped : !swapped;
if (layout == Layout::AllNull || (null_child && resource == 1u)) continue;
test.expected[test.storage_buffer_offsets[resource] / 4u + lane] = 0x1000u + lane;
}
auto output = vulkan.CreateStorageBuffer(test.name, test.initial, test.initial.size());
vulkan.Dispatch(test, compiled, output);
const auto actual = vulkan.ReadBuffer(test.name, output, test.expected.size());
vulkan.DestroyBuffer(&output);
CompareWords(test, "key selection, EXEC, bounds and null descriptors", test.expected, actual);
}
}
std::printf("[compute] %-32s ok\n", test.name);
}
void CheckFiniteFirstLaneImageChoice(VulkanHarness &vulkan) {
using O = ShaderOpcode;
constexpr u32 table = 512u, stride = 440u, poison = 0xdeadbeefu;
for (const u32 wave : {32u, 64u})
for (const u32 count : {0u, 4u}) {
TestCase test;
test.name = "FiniteFirstLaneImageChoice";
test.initial.assign((table + 4u * stride) / 4u, poison);
for (u32 row = 0; row < 4u; ++row) {
auto image = MakeSampledTextureData(Prospero::BufferFormat::k32Float);
image[0] = 10u + row;
image[3] = DstSel(4, 0, 0, 1) |
(static_cast<u32>(Prospero::ImageType::kColor2D) << 28u);
std::copy_n(image.begin(), 8, test.initial.begin() + (table + row * stride) / 4u);
}
auto records = MakeStructuredStorageBufferData(stride, count);
records[0] = table;
std::copy_n(records.begin(), 4, test.user_data.begin() + 56);
const auto output = MakeStructuredStorageBufferData(0, wave * 4u);
std::copy_n(output.begin(), 4, test.user_data.begin() + 48);
test.has_user_data = test.has_compute_info = true;
test.compute_info.wave_size = wave;
test.compute_info.threads_num[0] = wave;
test.compute_info.threads_num[1] = test.compute_info.threads_num[2] = 1;
test.compute_info.thread_ids_num = 1;
auto &code = test.code;
code.push_back(EncodeSop1(0x04, 40, 126));
code.push_back(EncodeVop2(0x1b, 1, InlineU32(3), 0));
for (u32 word = 0; word < 8u; ++word)
code.push_back(EncodeVop1(0x01, 20u + word, InlineU32(0)));
// Masked vector descriptor writes become synchronized Select/Phi chains.
for (u32 row = 0; row < 4u; ++row) {
code.push_back(EncodeVopc(0xd2, InlineU32(row), 1));
const auto skip = code.size();
code.push_back(0u);
code.insert(code.end(), {EncodeSmem0(0x0b, 8, 28), EncodeSmem1(row * stride, 125)});
for (u32 word = 0; word < 8u; ++word)
code.push_back(EncodeVop1(0x01, 20u + word, 8u + word));
code[skip] = EncodeSopp(0x08, code.size() - skip - 1u);
code.push_back(EncodeSop1(0x04, 126, 40));
}
AppendVMovLiteral(&code, 5, poison);
AppendVMovLiteral(&code, 10, 0x3f000000u);
AppendVMovLiteral(&code, 11, 0x3f000000u);
code.push_back(EncodeVop2(0x1b, 2, InlineU32(31), 0));
code.push_back(EncodeVopc(0xd5, InlineU32(0), 2));
const auto loop = code.size();
// Same native waterfall as 7ceb0f3417f926f9, including narrowing EXEC.
for (u32 word = 0; word < 8u; ++word)
code.push_back(EncodeVop1(0x02, 32u + word, Vgpr(20u + word)));
code.push_back(EncodeSop1(0x04, 44, 126));
for (u32 word = 0; word < 8u; ++word)
code.push_back(EncodeVopc(0xd2, 32u + word, 20u + word));
code.insert(code.end(), {EncodeMimg0(0x27, 1), EncodeMimg1(5, 10, 8, 13),
EncodeSop2(0x15, 126, 44, 126)});
code.push_back(EncodeSopp(0x09, loop - code.size() - 1u));
code.push_back(EncodeSop1(0x04, 126, 40));
code.push_back(EncodeVop2(0x1a, 30, InlineU32(2), 0));
AppendBufferStoreDword(&code, 5, 30);
AppendEnd(&code);
test.opcodes = {O::S_MOV_B64, O::V_AND_B32, O::V_MOV_B32,
O::V_CMPX_EQ_U32, O::V_CMPX_NE_U32, O::S_CBRANCH_EXECZ,
O::S_BUFFER_LOAD_DWORDX8, O::V_READFIRSTLANE_B32, O::IMAGE_SAMPLE,
O::S_ANDN2_B64, O::S_CBRANCH_EXECNZ, O::V_LSHLREV_B32,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
const auto compiled = CompileCase(test, vulkan.SubgroupSize());
Require(test.name, "finite correlated image choices", compiled.resources.images.size() <= 5u,
"descriptor words formed an uncorrelated product");
std::vector<VulkanHarness::Image> images;
for (const auto &descriptor : compiled.resources.images)
images.push_back(vulkan.CreateImage2D(test.name, 1, 1, vk::Format::eR32Sfloat,
vk::ImageUsageFlagBits::eSampled,
{std::bit_cast<u32>(float(descriptor.dwords[0]))}, 1,
vk::ImageLayout::eShaderReadOnlyOptimal));
const auto sampler = vulkan.CreateSampler(test.name, 1);
auto buffer = vulkan.CreateStorageBuffer(test.name, test.initial, test.initial.size());
vulkan.Dispatch(test, compiled, buffer, nullptr, nullptr, nullptr, nullptr, sampler, images);
test.expected = test.initial;
for (u32 lane = 0; lane < wave; ++lane)
if ((lane & 31u) != 0u)
test.expected[lane] = count == 0u ? 0u : std::bit_cast<u32>(float(10u + (lane & 3u)));
CompareWords(test, "first active lane, descriptor correlation and inactive canaries", test.expected,
vulkan.ReadBuffer(test.name, buffer, test.expected.size()));
vulkan.DestroyBuffer(&buffer);
vulkan.Device().destroySampler(sampler);
for (auto &image : images) vulkan.DestroyImage(&image);
std::printf("[compute] %s wave=%u count=%u ok\n", test.name, wave, count);
}
}
void CheckFiniteInlineSamplerPhi(VulkanHarness &vulkan) {
TestCase test;
test.name = "FiniteInlineSamplerPhi";
auto &code = test.code;
code.insert(code.end(), {EncodeSop1(0x04, 36, InlineU32(0)),
EncodeSop1(0x04, 38, InlineU32(0)), EncodeSop1(0x04, 24, 126),
EncodeVop1(0x01, 10, 16), EncodeVopc(0xd5, InlineU32(0), 10)});
const auto skip_sampler = code.size();
code.push_back(EncodeSopp(0x08));
// Actual correlated inline sampler words from 905b2e2c: zero or {0,fff000,a500000,0}.
code.insert(code.end(), {EncodeSop2(0x24, 37, InlineU32(12), InlineU32(12)),
EncodeSop1(0x04, 38, 255), 0x0a500000u});
code[skip_sampler] = EncodeSopp(0x08, code.size() - skip_sampler - 1u);
code.push_back(EncodeSop1(0x04, 126, 24));
AppendVMovLiteral(&code, 20, std::bit_cast<u32>(0.5f));
AppendVMovLiteral(&code, 21, std::bit_cast<u32>(0.5f));
AppendVMovLiteral(&code, 1, 0x12345678u);
AppendVMovLiteral(&code, 2, 0x87654321u);
code.insert(code.end(), {EncodeVopc(0xd4, InlineU32(2), 10),
EncodeMimg0(0x27, 1), EncodeMimg1(1, 20, 0, 9),
EncodeMimg0(0x27, 1), EncodeMimg1(2, 20, 2, 9),
EncodeSop1(0x04, 126, 24), EncodeVop1(0x01, 30, 16),
EncodeVop2(0x1a, 31, InlineU32(3), 30),
EncodeMubuf0(0x1c), EncodeMubuf1(1, 12, 31),
EncodeMubuf0(0x1c, 4), EncodeMubuf1(2, 12, 31)});
AppendEnd(&code);
test.initial.assign(7, 0xa5a5a5a5u);
test.expected = {std::bit_cast<u32>(4.0f), 0xfffffff9u,
std::bit_cast<u32>(2.0f), 0xfffffff9u, 0x12345678u, 0x87654321u, 0xa5a5a5a5u};
test.user_data = MakeSampledTextureData(Prospero::BufferFormat::k32_32_32_32Float);
test.user_data[3] |= static_cast<u32>(Prospero::ImageType::kColor2D) << 28u;
const auto signed_image = MakeSampledTextureData(Prospero::BufferFormat::k32SInt);
std::copy_n(signed_image.begin(), 8, test.user_data.begin() + 8);
test.user_data[11] |= static_cast<u32>(Prospero::ImageType::kColor2D) << 28u;
test.user_data[50] = test.initial.size() * sizeof(u32);
test.user_data[51] = 3u << 28u;
test.has_user_data = true;
test.compute_info.threads_num[0] = test.compute_info.threads_num[1] = test.compute_info.threads_num[2] = 1;
test.compute_info.wave_size = 32;
test.compute_info.group_id[0] = true;
test.compute_info.workgroup_register = 16;
test.has_compute_info = true;
test.dispatch_x = 3;
test.opcodes = {ShaderOpcode::S_MOV_B64, ShaderOpcode::S_BFM_B32,
ShaderOpcode::S_CBRANCH_EXECZ, ShaderOpcode::V_MOV_B32, ShaderOpcode::V_CMPX_NE_U32,
ShaderOpcode::V_CMPX_GT_U32, ShaderOpcode::V_LSHLREV_B32,
ShaderOpcode::IMAGE_SAMPLE, ShaderOpcode::BUFFER_STORE_DWORD, ShaderOpcode::S_ENDPGM};
test.required_spirv = {"OpPhi", "NonUniform", "OpImageSampleExplicitLod"};
auto compiled = CompileCase(test, vulkan.SubgroupSize());
std::set<u32> nonuniform_ids;
std::vector<std::pair<u32, u32>> sampled_ids;
for (size_t offset = 5; offset < compiled.spirv.size();) {
const auto words = std::span(compiled.spirv).subspan(offset, compiled.spirv[offset] >> 16u);
if (static_cast<spv::Op>(words[0] & 0xffffu) == spv::OpDecorate &&
words[2] == spv::DecorationNonUniform) nonuniform_ids.insert(words[1]);
if (static_cast<spv::Op>(words[0] & 0xffffu) == spv::OpSampledImage)
sampled_ids.emplace_back(words[2], words[4]);
offset += words.size();
}
Require(test.name, "dynamic sampler decorations", sampled_ids.size() >= 2 &&
std::ranges::all_of(sampled_ids, [&](const auto &ids) {
return nonuniform_ids.contains(ids.first) && nonuniform_ids.contains(ids.second);
}),
"fetched sampler or sampled image lost its NonUniform decoration");
const auto &samplers = compiled.program.info.samplers;
u32 roots = 0;
for (const auto &root : samplers) {
if (root.indirect_resources.empty()) continue;
++roots;
Require(test.name, "correlated sampler candidates", root.indirect_resources.size() == 2,
"finite sampler was folded or lost a correlated branch state");
for (const auto child : root.indirect_resources)
Require(test.name, "sampler class remapping", child < samplers.size() &&
samplers[child].force_point_filtering == root.force_point_filtering &&
samplers[child].integer_border == root.integer_border,
"a finite candidate retained another image class's sampler");
}
Require(test.name, "finite sampler class variants", roots == 2 && samplers.size() == 4 &&
compiled.resources.samplers.size() == 2,
"the two-state sampler did not retain both float and signed image classes");
constexpr std::array<u32, 4> zero_sampler{};
constexpr std::array<u32, 4> linear_sampler{0, 0x00fff000u, 0x0a500000u, 0};
Require(test.name, "inline sampler states",
std::ranges::all_of(compiled.resources.samplers, [&](const auto &value) {
return std::equal(zero_sampler.begin(), zero_sampler.end(), value.dwords.begin()) ||
std::equal(linear_sampler.begin(), linear_sampler.end(), value.dwords.begin());
}) && compiled.resources.samplers[0] != compiled.resources.samplers[1],
"descriptor words were mixed between the two native sampler states");
auto pixels = MakeRgbaImage(4, 4);
for (u32 y = 0; y < 4; ++y)
for (u32 x = 2; x < 4; ++x) SetRgbaPixel(&pixels, 4, x, y, std::bit_cast<u32>(4.0f), 0, 0, 0);
std::array<VulkanHarness::Image, 2> images{
vulkan.CreateImageMips(test.name, 4, 4, vk::Format::eR32G32B32A32Sfloat,
vk::ImageUsageFlagBits::eSampled, {pixels}, 4, vk::ImageLayout::eShaderReadOnlyOptimal,
vk::ImageType::e2D, vk::ImageViewType::e2D, 1),
vulkan.CreateImageMips(test.name, 4, 4, vk::Format::eR32Sint,
vk::ImageUsageFlagBits::eSampled, {std::vector<u32>(16, 0xfffffff9u)}, 1,
vk::ImageLayout::eShaderReadOnlyOptimal, vk::ImageType::e2D, vk::ImageViewType::e2D, 1)};
std::vector<vk::Sampler> native_samplers;
for (const auto &resource : samplers) {
ShaderSamplerResource descriptor{};
std::copy_n(compiled.resources.samplers[resource.snapshot_index].dwords.begin(), 4, descriptor.fields);
const auto filter = !resource.force_point_filtering && descriptor.XyMagFilter() == 1
? vk::Filter::eLinear : vk::Filter::eNearest;
native_samplers.push_back(vulkan.CreateSampler(test.name, 1, filter));
}
auto output = vulkan.CreateStorageBuffer(test.name, test.initial, test.initial.size());
vulkan.Dispatch(test, compiled, output, nullptr, nullptr, nullptr, nullptr, nullptr, images, native_samplers);
CompareWords(test, "point, linear, remapped signed sampler and inactive output", test.expected,
vulkan.ReadBuffer(test.name, output, test.expected.size()));
vulkan.DestroyBuffer(&output);
for (const auto sampler : native_samplers) vulkan.Device().destroySampler(sampler);
for (auto &image : images) vulkan.DestroyImage(&image);
std::printf("[compute] %-32s ok\n", test.name);
}
void CheckCountedRowImageSelector(VulkanHarness &vulkan) {
using O = ShaderOpcode;
using F = Prospero::BufferFormat;
constexpr u32 poison = 0x345d1746u;
const auto alignment = static_cast<u32>(vulkan.RuntimeContext().GetPhysicalDeviceProperties()
.limits.minStorageBufferOffsetAlignment);
for (const bool masked : {false, true})
for (const u32 wave : {32u, 64u}) {
const auto output_words = masked ? 48u * wave : 64u;
const auto slot = (output_words * 4u + alignment - 1u) / alignment * alignment;
const u32 srt = slot, counts = slot * 2u, keys = slot * 3u, material = slot * 4u;
const auto tile_list = slot * 5u;
TestCase test;
test.name = masked ? (wave == 32u ? "MaskedRowImageSelector32" : "MaskedRowImageSelector64")
: (wave == 32u ? "CountedRowImageSelector32" : "CountedRowImageSelector64");
test.initial.resize((masked ? tile_list + 12u : material + 396u * 4u) / 4u, 0u);
std::fill_n(test.initial.begin(), output_words, 0xdeadbeefu);
const std::array<u32, 12> descriptors{counts, 4u << 16u, 3u, 0x14004u,
keys, 4u << 16u, 48u, 0x14004u, material, 396u << 16u, 4u, 0x14004u};
std::copy(descriptors.begin(), descriptors.end(), test.initial.begin() + srt / 4u);
test.initial[counts / 4u] = 3u;
test.initial[counts / 4u + 2u] = 1u;
std::fill_n(test.initial.begin() + keys / 4u, 48u, poison);
test.initial[keys / 4u] = 2u;
test.initial[keys / 4u + 1u] = 0x80000001u;
test.initial[keys / 4u + 2u] = 0x1890cedeu; // Dynamic +184 wraps to material byte32.
test.initial[keys / 4u + 32u] = 3u;
const auto put_image = [&](u32 offset, u32 identity, F format) {
auto at = test.initial.begin() + (material + offset) / 4u;
at[0] = identity;
at[1] = static_cast<u32>(format) << 20u;
at[3] = DstSel(4, 5, 6, 7) |
(static_cast<u32>(Prospero::ImageType::kColor2D) << 28u);
};
for (u32 key = 0; key < 4u; ++key)
put_image(key * 396u + 184u, 0x100u + key, F::k32_32_32_32Float);
put_image(32u, 0x104u, F::k32_32_32_32Float);
put_image(256u, 0x200u, F::k8SScaled);
if (masked) {
// The scalar tile list reverses workgroups; row selection must remain on the GPU.
test.initial[tile_list / 4u] = 2u;
test.initial[tile_list / 4u + 1u] = 1u;
test.user_data[52] = tile_list;
test.user_data[53] = 4u << 16u;
test.user_data[54] = 3u;
test.user_data[55] = 0x14004u;
}
test.expected = test.initial;
for (const auto [index, bits] : std::array<std::pair<u32, u32>, 4>{{
{0u, 0x3f800002u}, {1u, 0x3f800001u}, {2u, 0x3f800004u}, {32u, 0x3f800003u}}}) {
if (masked) {
for (u32 lane = 1u; lane < wave; lane += 2u)
test.expected[index * wave + lane] = bits;
} else test.expected[index] = bits;
}
test.user_data[0] = srt;
test.user_data[50] = output_words * 4u;
test.user_data[51] = 3u << 28u;
test.has_user_data = true;
test.has_compute_info = true;
test.compute_info.wave_size = wave;
test.compute_info.threads_num[0] = masked ? wave : 1u;
test.compute_info.threads_num[1] = test.compute_info.threads_num[2] = 1u;
test.compute_info.thread_ids_num = masked ? 1 : 0;
test.compute_info.group_id[0] = true;
test.compute_info.workgroup_register = 44u;
test.dispatch_x = 3u;
auto &code = test.code;
code.insert(code.end(), {EncodeSmem0(0x03, 8), EncodeSmem1(0, 125),
EncodeSmem0(0x02, 16), EncodeSmem1(32, 125)});
code.push_back(EncodeSop2(0x1e, 22, 44, InlineU32(2)));
if (masked) {
code.push_back(EncodeVop1(0x01, 2, Vgpr(0))); // Preserve the lane for disjoint output addresses.
code.insert(code.end(), {EncodeSmem0(0x08, 23, 26), EncodeSmem1(0, 22)});
code.push_back(EncodeVop1(0x01, 18, 23));
code.push_back(EncodeVop1(0x02, 22, Vgpr(18)));
code.push_back(EncodeSop2(0x1e, 22, 22, InlineU32(2)));
}
code.insert(code.end(), {EncodeSmem0(0x08, 21, 4), EncodeSmem1(0, 22)});
AppendSMovLiteral(&code, 20, 0u);
AppendVMovU32(&code, 0, 0x3f000000u);
AppendVMovU32(&code, 1, 0x3f000000u);
const auto masked_exit_branch = masked ? code.size() + 2u : 0u;
if (masked) {
code.push_back(EncodeVop2(0x1b, 6, InlineU32(1), 2));
code.push_back(EncodeVopc(0xd5, InlineU32(0), 6));
code.push_back(0u); // Skip the loop when the new EXEC is empty.
}
const auto header = code.size();
code.push_back(EncodeSopc(0x0a, 20, 21));
const auto exit_branch = code.size();
code.push_back(0u);
if (masked) {
code.insert(code.end(), {EncodeVop3Word0(0x346, 3),
EncodeVop3Word1(Vgpr(18), InlineU32(4), 20)});
code.push_back(EncodeVop1(0x02, 22, Vgpr(3)));
} else {
code.push_back(EncodeSop2(0x1e, 22, 44, InlineU32(4)));
code.push_back(EncodeSop2(0x02, 22, 22, 20));
}
code.push_back(EncodeSop2(0x1e, 22, 22, InlineU32(2)));
code.insert(code.end(), {EncodeSmem0(0x08, 23, 6), EncodeSmem1(0, 22)});
code.push_back(EncodeSop2(0x26, 24, 23, 255)); code.push_back(396u);
code.push_back(EncodeSop2(0x02, 24, 24, 255)); code.push_back(184u);
code.insert(code.end(), {EncodeSmem0(0x0b, 32, 8), EncodeSmem1(0, 24),
EncodeMimg0(0x27, 1), EncodeMimg1(4, 0, 8, 10)});
if (masked) {
code.push_back(EncodeVop2(0x1a, 5, InlineU32(2), 2));
code.insert(code.end(), {EncodeVop3Word0(0x346, 30),
EncodeVop3Word1(22, InlineU32(std::countr_zero(wave)), Vgpr(5))});
} else code.push_back(EncodeVop1(0x01, 30, 22));
AppendBufferStoreDword(&code, 4, 30);
code.push_back(EncodeSop2(0x02, 20, 20, InlineU32(1)));
code.push_back(EncodeSopp(0x02, header - code.size() - 1u));
code[exit_branch] = EncodeSopp(0x04, code.size() - exit_branch - 1u);
if (masked) code[masked_exit_branch] = EncodeSopp(0x08, code.size() - masked_exit_branch - 1u);
AppendEnd(&code);
test.opcodes = {O::S_LOAD_DWORDX8, O::S_LOAD_DWORDX4, O::S_BUFFER_LOAD_DWORD,
O::S_BUFFER_LOAD_DWORDX8, O::S_MOV_B32, O::S_LSHL_B32, O::S_ADD_I32,
O::S_MUL_I32, O::S_CMP_LT_U32, O::S_CBRANCH_SCC0, O::S_BRANCH,
O::V_MOV_B32, O::IMAGE_SAMPLE, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
if (masked) test.opcodes.insert(test.opcodes.end(), {O::V_READFIRSTLANE_B32,
O::V_LSHL_ADD_U32, O::V_LSHLREV_B32, O::V_AND_B32, O::V_CMPX_NE_U32, O::S_CBRANCH_EXECZ});
const auto compiled = CompileCase(test, vulkan.SubgroupSize());
Require(test.name, "exact active images", compiled.resources.images.size() == 4u &&
std::ranges::all_of(compiled.resources.images, [](const auto &image) {
return image.dwords[0] >= 0x101u && image.dwords[0] <= 0x104u;
}), "unused row entries selected malformed material fields");
for (u32 word = 0; word < 4u; ++word)
Require(test.name, "count descriptor capture",
std::ranges::any_of(compiled.resources.specialization_reads, [&](const auto &read) {
return read.first <= srt + word * 4u &&
read.first + read.second >= srt + (word + 1u) * 4u;
}), "count V# loaded from SRT escaped writable-alias capture");
for (const auto &buffer : compiled.resources.buffers) {
ShaderBufferResource descriptor{};
std::copy_n(buffer.dwords.begin(), 4u, descriptor.fields);
test.storage_buffer_ranges.emplace_back(static_cast<u32>(descriptor.Base48()),
static_cast<u32>(descriptor.GetSize()));
}
std::vector<VulkanHarness::Image> images;
for (const auto &image : compiled.resources.images)
images.push_back(vulkan.CreateImageMips(test.name, 1, 1, vk::Format::eR32G32B32A32Sfloat,
vk::ImageUsageFlagBits::eSampled, {{0x3f800000u + image.dwords[0] - 0x100u, 0u, 0u, 0u}},
4u, vk::ImageLayout::eShaderReadOnlyOptimal, vk::ImageType::e2D, vk::ImageViewType::e2D, 1));
auto sampler = vulkan.CreateSampler(test.name, 1);
auto output = vulkan.CreateStorageBuffer(test.name, test.initial, test.initial.size());
vulkan.Dispatch(test, compiled, output, nullptr, nullptr, nullptr, nullptr, sampler, images);
CompareWords(test, "active row images, wrapped keys and untouched unused slots", test.expected,
vulkan.ReadBuffer(test.name, output, test.expected.size()));
vulkan.DestroyBuffer(&output);
vulkan.Device().destroySampler(sampler);
for (auto &image : images) vulkan.DestroyImage(&image);
std::printf("[compute] %-32s ok\n", test.name);
}
}
void CheckPointerImageTable(VulkanHarness &vulkan) {
using O = ShaderOpcode;
constexpr u32 base = 256u, table_offset = 24364u;
for (const u32 variant : {0u, 1u, 2u, 3u, 4u})
for (const u32 wave : {32u, 64u}) {
const bool packed = variant == 2u, post_tested = variant >= 3u;
const u32 count = post_tested ? (variant == 3u ? 1u : 6u) : variant * 2u;
TestCase test;
test.name = post_tested ? "PostTestedPointerImageTable"
: packed ? "PackedPointerImageTable" : "CountedPointerImageTable";
test.dispatch_x = packed ? 4u : 1u;
test.bda_mappings = {{0u, 0u}};
test.initial.resize((base + table_offset + 32u * 32u) / 4u);
std::fill_n(test.initial.begin(), test.dispatch_x, 0xdeadbeefu);
test.initial[(base + 492u) / 4u] = count;
for (u32 row = 0; row < 32u; ++row) {
auto image = MakeSampledTextureData(Prospero::BufferFormat::k32Float);
image[0] = row + 1u;
image[3] = DstSel(4, 0, 0, 1) |
(static_cast<u32>(Prospero::ImageType::kColor2D) << 28u);
std::copy_n(image.begin(), 8,
test.initial.begin() + (base + table_offset + row * 32u) / 4u);
}
test.initial[(base + 2756u) / 4u] = 1u;
test.initial[(base + 2756u + 48u) / 4u] = 31u;
constexpr std::array fields{31u, 0u, 7u, 1u};
for (u32 row = 0; row < fields.size(); ++row)
test.initial[(base + 5008u + row * 60u) / 4u] = 0xffe00abcu | (fields[row] << 12u);
test.user_data[0] = base;
const auto output = MakeStructuredStorageBufferData(0u, test.dispatch_x * 4u);
std::copy_n(output.begin(), 4, test.user_data.begin() + 48);
test.has_user_data = test.has_compute_info = true;
test.compute_info.wave_size = wave;
test.compute_info.threads_num[0] = test.compute_info.threads_num[1] =
test.compute_info.threads_num[2] = 1u;
test.compute_info.group_id[0] = true;
test.compute_info.workgroup_register = 44u;
auto &code = test.code;
AppendVMovLiteral(&code, 0, 0u);
AppendVMovLiteral(&code, 20, post_tested ? 0u : 0x3f000000u);
AppendVMovLiteral(&code, 21, post_tested ? 0u : 0x3f000000u);
size_t loop = 0, exit = 0, skip = 0;
if (post_tested) {
// Captured 5f3fdf61a7ca4a20: load row before incrementing and testing next < 6.
code.push_back(EncodeSMovB32(20, InlineU32(0)));
loop = code.size();
code.push_back(EncodeSop2(0x1e, 23, 20, InlineU32(5)));
} else if (packed) {
code.insert(code.end(), {EncodeSop2(0x26, 23, 44, InlineU32(60)),
EncodeSop2(0x02, 23, 23, 255), 5008u,
EncodeSmem0(0x00, 24), EncodeSmem1(0, 23),
EncodeSop2(0x20, 24, 24, InlineU32(12)),
EncodeSop2(0x1e, 23, 24, InlineU32(5)),
EncodeSop2(0x0e, 23, 23, 255), 992u});
} else {
code.insert(code.end(), {EncodeSmem0(0x00, 21), EncodeSmem1(492, 125),
EncodeSMovB32(20, InlineU32(0))});
loop = code.size();
code.push_back(EncodeSopc(0x0a, 20, 21));
exit = code.size(); code.push_back(0u);
code.insert(code.end(), {EncodeSop2(0x26, 23, 20, InlineU32(48)),
EncodeSop2(0x02, 23, 23, 255), 2756u,
EncodeSmem0(0x00, 24), EncodeSmem1(0, 23),
EncodeSopc(0x03, 24, InlineU32(0))});
skip = code.size(); code.push_back(0u);
code.push_back(EncodeSop2(0x1e, 23, 24, InlineU32(5)));
}
code.insert(code.end(), {EncodeSop2(0x02, 23, 23, 255), table_offset,
EncodeSmem0(0x03, 8), EncodeSmem1(0, 23),
EncodeMimg0(post_tested ? 0x00 : 0x27, 1), EncodeMimg1(4, 20, 2, 10),
EncodeVop2(0x03, 0, Vgpr(4), 0)});
if (post_tested) {
code.insert(code.end(), {EncodeSop2(0x02, 20, 20, InlineU32(1)),
EncodeSopc(0x04, 20, InlineU32(count))});
code.push_back(EncodeSopp(0x05, loop - code.size() - 1u));
} else if (!packed) {
code[skip] = EncodeSopp(0x04, code.size() - skip - 1u);
code.push_back(EncodeSop2(0x02, 20, 20, InlineU32(1)));
code.push_back(EncodeSopp(0x02, loop - code.size() - 1u));
code[exit] = EncodeSopp(0x04, code.size() - exit - 1u);
}
code.push_back(EncodeSop2(0x1e, 25, 44, InlineU32(2)));
code.push_back(EncodeVop1(0x01, 30, 25));
AppendBufferStoreDword(&code, 0, 30);
AppendEnd(&code);
test.opcodes = {O::S_ADD_I32, O::S_LSHL_B32,
O::S_LOAD_DWORDX8, post_tested ? O::IMAGE_LOAD : O::IMAGE_SAMPLE, O::V_ADD_F32, O::V_MOV_B32,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
if (!post_tested) test.opcodes.insert(test.opcodes.end(), {O::S_LOAD_DWORD, O::S_MUL_I32});
if (post_tested) test.opcodes.insert(test.opcodes.end(), {O::S_MOV_B32, O::S_CMP_LT_I32,
O::S_CBRANCH_SCC1});
else if (packed) test.opcodes.insert(test.opcodes.end(), {O::S_LSHR_B32, O::S_AND_B32});
else test.opcodes.insert(test.opcodes.end(), {O::S_MOV_B32, O::S_CMP_LT_U32,
O::S_CMP_GE_I32, O::S_CBRANCH_SCC0, O::S_BRANCH});
const auto compiled = CompileCase(test, vulkan.SubgroupSize());
Require(test.name, "bounded image candidates",
packed ? compiled.resources.images.size() == 32u
: compiled.resources.images.size() == std::max(count, 1u),
"descriptor planning escaped its selector domain");
std::vector<VulkanHarness::Image> images;
for (const auto &descriptor : compiled.resources.images)
images.push_back(vulkan.CreateImage2D(test.name, 1, 1, vk::Format::eR32Sfloat,
vk::ImageUsageFlagBits::eSampled,
{std::bit_cast<u32>(static_cast<float>(descriptor.dwords[0]))}, 1,
vk::ImageLayout::eShaderReadOnlyOptimal));
auto buffer = vulkan.CreateStorageBuffer(test.name, test.initial, test.initial.size(), true);
auto sampler = post_tested ? vk::Sampler{} : vulkan.CreateSampler(test.name, 1);
vulkan.Dispatch(test, compiled, buffer, nullptr, nullptr, nullptr, nullptr, sampler, images);
std::vector<u32> expected(test.dispatch_x, std::bit_cast<u32>(post_tested ? float(count * (count + 1u) / 2u) : count == 0u ? 0.0f : 34.0f));
if (packed) for (u32 row = 0; row < fields.size(); ++row)
expected[row] = std::bit_cast<u32>(static_cast<float>(fields[row] + 1u));
CompareWords(test, "bounded pointer selectors and packed fields", expected,
vulkan.ReadBuffer(test.name, buffer, expected.size()));
vulkan.DestroyBuffer(&buffer);
if (sampler) vulkan.Device().destroySampler(sampler);
for (auto &image : images) vulkan.DestroyImage(&image);
std::printf("[compute] %s wave=%u count=%u ok\n", test.name, wave, count);
}
}
void CheckNestedPointerImageTable(VulkanHarness &vulkan) {
using O = ShaderOpcode;
constexpr u32 list_address = 160, objects = 4096;
constexpr std::array keys{5u, 0u, 3u, 1u, 4u, 2u};
constexpr std::array list{2u, 1u, 4u, 0u, 5u, 3u};
for (const auto [member, counted] : {std::pair{0u, false}, std::pair{0x98u, false},
std::pair{0x16cu, false}, std::pair{64u, false}, std::pair{64u, true}})
for (const auto [wave, upper_only] : {std::pair{32u, false}, std::pair{64u, false},
std::pair{64u, true}}) {
const bool child_table = member == 64u, compact = member != 0u && !child_table;
if (upper_only && !child_table) continue;
const u32 keys_address = child_table ? 1024u : 128u, table = child_table ? 2048u : 256u;
const u32 rows = child_table ? 4u : keys.size(), children = child_table ? 3u : 1u;
const u32 records_count = counted ? 2u : rows, count_address = 768u;
const u32 stride = child_table ? 136u : compact ? 0x330u : 592u;
const u32 object_stride = child_table ? 1536u : 0x430u;
const u32 table_bytes = (rows - 1u) * stride + member + 16u;
const u32 outputs = child_table ? wave * 2u : keys.size();
TestCase test;
test.name = counted ? (upper_only ? "CountedPointerChildTableUpper64" : "CountedPointerChildTable")
: child_table ? (upper_only ? "IndexedPointerChildTableUpper64" : "IndexedPointerChildTable")
: member == 0x16cu ? "CompactPairedImageSamplerTable"
: compact ? "CompactRecordImageSamplerTable" : "NestedPointerImageSamplerTable";
test.initial.resize((compact ? table + table_bytes : objects + rows * object_stride) / 4u,
(compact || child_table) ? 0xdeadbeefu : 0u);
std::fill_n(test.initial.begin(), outputs, 0xdeadbeefu);
std::fill_n(test.initial.begin() + keys_address / 4u, child_table ? wave * 2u : keys.size(), UINT32_MAX);
if (compact) std::copy(list.begin(), list.end(), test.initial.begin() + list_address / 4u);
for (u32 row = 0; row < records_count; ++row) {
const u32 object = compact ? table + row * stride : objects + row * object_stride;
if (!compact) {
test.initial[(table + row * stride) / 4u] = object;
// The actual vector path sign-extends WORD0 and discards the packed upper bits.
test.initial[(table + row * stride) / 4u + 1u] = child_table ? 0xbeef0000u : 0u;
}
for (u32 child = 0; child < children; ++child) {
auto image = MakeSampledTextureData(Prospero::BufferFormat::k32Float);
image[0] = 10u + row * children + child;
image[1] |= 1u << 30u; // Two texels: zero, then the selected child's value.
image[3] = DstSel(4, 0, 0, 1) |
(static_cast<u32>(Prospero::ImageType::kColor2D) << 28u);
std::copy_n(image.begin(), compact ? 4u : 8u,
test.initial.begin() + (object + (child_table ? child * 368u + member
: compact ? member : 0x3d0u)) / 4u);
}
const std::array sampler{0u, 0x00fff000u, (row & 1u) ? 0x0a500000u : 0u, 0u};
std::copy(sampler.begin(), sampler.end(), test.initial.begin() +
(child_table ? table + row * stride + 80u : object + (compact ? 0x88u : 0x3f8u)) / 4u);
}
if (compact) {
// Member 0x16c admits this tuple's residue; only the paired selector list excludes it.
const std::array unreachable{0x00fff000u, 0x09500000u, 0u, 0x900672b0u};
std::copy(unreachable.begin(), unreachable.end(),
test.initial.begin() + (table + (member == 0x16cu ? 0x17cu : 0xbcu)) / 4u);
}
auto records = MakeStructuredStorageBufferData(compact ? 0u : stride,
compact ? table_bytes : records_count);
records[0] = table;
std::copy_n(records.begin(), 4, test.user_data.begin() + 32);
auto key_buffer = MakeStructuredStorageBufferData(child_table ? 8u : 0u,
child_table ? wave : keys.size() * 4u);
key_buffer[0] = keys_address;
std::copy_n(key_buffer.begin(), 4, test.user_data.begin() + 4);
if (compact) {
auto indices = MakeStructuredStorageBufferData(0, list.size() * 4u);
indices[0] = list_address;
std::copy_n(indices.begin(), 4, test.user_data.begin() + 36);
}
const auto output = MakeStructuredStorageBufferData(0, outputs * 4u);
std::copy_n(output.begin(), 4, test.user_data.begin() + 48);
if (counted) {
test.user_data[52] = count_address;
test.initial[count_address / 4u] = records_count;
}
test.bda_mappings = {{0u, 0u}};
test.has_user_data = test.has_compute_info = true;
test.compute_info.wave_size = wave;
test.compute_info.threads_num[0] = child_table ? wave : 1u;
test.compute_info.threads_num[1] = test.compute_info.threads_num[2] = 1u;
test.compute_info.thread_ids_num = child_table ? 1u : 0u;
test.compute_info.group_id[0] = true;
test.compute_info.workgroup_register = 44;
test.dispatch_x = child_table ? 1u : keys.size();
auto &code = test.code;
size_t loop = 0, empty = 0, skip_child = 0;
if (child_table) {
code.push_back(EncodeVop2(0x1a, 30, InlineU32(2), 0));
if (counted)
code.insert(code.end(), {EncodeSmem0(0x00, 54, 26), EncodeSmem1(0, 125),
EncodeSopp(0x0c, 0)});
if (upper_only) {
AppendSMovLiteral(&code, 126, 0u);
AppendSMovLiteral(&code, 127, UINT32_MAX);
}
code.insert(code.end(), {EncodeSop1(0x04, 60, 126),
EncodeMubuf0(0x0d, 0, true, false), EncodeMubuf1(4, 1, 0), EncodeSopp(0x0c, 0)});
AppendVMovLiteral(&code, 6, UINT32_MAX);
for (u32 row = 0; row < rows; ++row) {
if (counted)
code.insert(code.end(), {EncodeSopc(0x08, 54, InlineU32(row)),
EncodeSop2(0x0b, 62, 126, InlineU32(0))});
code.push_back(EncodeVopc(0xc2, InlineU32(row), 4));
if (counted) code.push_back(EncodeSop2(0x0f, 106, 106, 62));
AppendVop3(&code, 0x101, 6, Vgpr(6), InlineU32(row), 106);
}
code.push_back(EncodeVopc(0x93, InlineU32(0), 6)); // Only finite nonnegative parent rows.
empty = code.size(); code.push_back(0u);
code.insert(code.end(), {EncodeMubuf0(0x0d, 0, true, false), EncodeMubuf1(10, 8, 6),
EncodeSopp(0x0c, 0)});
AppendVop3(&code, 0x149, 11, Vgpr(11), InlineU32(0), InlineU32(16));
loop = code.size();
code.insert(code.end(), {EncodeVop1(0x02, 22, Vgpr(6)),
EncodeVop1(0x02, 20, Vgpr(10)), EncodeVop1(0x02, 21, Vgpr(11)),
EncodeVop1(0x02, 24, Vgpr(5)), EncodeVopc(0xc2, 22, 6), EncodeSop1(0x04, 56, 106)});
for (const auto [sgpr, vgpr] : {std::pair{20u, 10u}, std::pair{21u, 11u}, std::pair{24u, 5u}})
code.insert(code.end(), {EncodeVopc(0xc2, sgpr, vgpr), EncodeSop2(0x0f, 56, 56, 106)});
code.insert(code.end(), {EncodeSop1(0x24, 58, 56), EncodeSopc(0x09, 24, InlineU32(children))});
skip_child = code.size(); code.push_back(0u);
code.insert(code.end(), {EncodeSop2(0x26, 23, 24, 255), 368u,
EncodeSmem0(0x03, 8, 10), EncodeSmem1(member, 23),
EncodeSop2(0x26, 25, 22, 255), stride,
EncodeSmem0(0x0a, 16, 16), EncodeSmem1(80u, 25), EncodeSopp(0x0c, 0)});
} else {
code.push_back(EncodeSop2(0x1e, 23, 44, InlineU32(2)));
code.push_back(EncodeVop1(0x01, 30, 23));
code.insert(code.end(), {EncodeMubuf0(0x0c), EncodeMubuf1(4, 1, 30),
EncodeVop2(0x16, 6, InlineU32(16), 4)});
AppendVop3(&code, 0x360, 22, Vgpr(6), InlineU32(0));
}
if (compact) {
code.insert(code.end(), {EncodeSop2(0x1e, 23, 22, InlineU32(2)),
EncodeSmem0(0x08, 22, 18), EncodeSmem1(0, 23),
EncodeSop2(0x26, 23, 22, 255), stride});
if (member == 0x98u)
code.insert(code.end(), {EncodeSmem0(0x0b, 8, 16), EncodeSmem1(0x88u, 23)});
else code.insert(code.end(), {EncodeSmem0(0x0a, 8, 16), EncodeSmem1(0x88u, 23),
EncodeSmem0(0x0a, 12, 16), EncodeSmem1(member, 23)});
for (u32 reg = 16u; reg < 20u; ++reg) AppendSMovLiteral(&code, reg, 0xdeadbeefu);
} else if (!child_table) code.insert(code.end(), {EncodeSop2(0x26, 23, 22, 255), stride,
EncodeSmem0(0x09, 20, 16), EncodeSmem1(0, 23),
EncodeSmem0(0x03, 8, 10), EncodeSmem1(0x3d0, 125),
EncodeSmem0(0x02, 16, 10), EncodeSmem1(0x3f8, 125)});
AppendVMovLiteral(&code, 20, std::bit_cast<u32>(0.5f));
AppendVMovLiteral(&code, 21, std::bit_cast<u32>(0.5f));
code.insert(code.end(), {EncodeMimg0(0x27, 1, 0, false, 1, compact),
EncodeMimg1(12, 20, compact ? 3u : 2u, compact ? 2u : 4u)});
AppendBufferStoreDword(&code, 12, 30);
if (child_table) {
code[skip_child] = EncodeSopp(0x05, code.size() - skip_child - 1u);
code.push_back(EncodeSop2(0x15, 126, 58, 56));
code.push_back(EncodeSopp(0x09, static_cast<int16_t>(loop - code.size() - 1u)));
code[empty] = EncodeSopp(0x08, code.size() - empty - 1u);
code.push_back(EncodeSop1(0x04, 126, 60));
AppendVMovLiteral(&code, 12, 0x76543210u);
code.insert(code.end(), {EncodeMubuf0(0x1c, wave * 4u), EncodeMubuf1(12, 12, 30)});
}
AppendEnd(&code);
if (child_table) {
test.decoded_counts = {{"BUFFER_LOAD_DWORDX2", 2u}, {"V_READFIRSTLANE_B32", 4u},
{"V_BFE_I32", 1u}, {"V_CMPX_LE_I32", 1u}, {"S_CBRANCH_EXECNZ", 1u}};
test.required_spirv = {"OpLoopMerge", "OpGroupNonUniformShuffle", "OpBitFieldSExtract"};
if (counted) test.decoded_counts.insert(test.decoded_counts.end(),
{{"S_LOAD_DWORD ", 1u}, {"S_CMP_GT_U32", rows}, {"S_CSELECT_B64", rows}});
} else test.opcodes = {O::S_LSHL_B32, O::V_MOV_B32, O::BUFFER_LOAD_DWORD,
O::V_LSHRREV_B32, O::V_READLANE_B32, O::S_MUL_I32,
O::IMAGE_SAMPLE, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
if (compact) {
test.opcodes.insert(test.opcodes.end(), {O::S_BUFFER_LOAD_DWORD,
member == 0x98u ? O::S_BUFFER_LOAD_DWORDX8 : O::S_BUFFER_LOAD_DWORDX4, O::S_MOV_B32});
test.decoded_counts = {{"r128=1", 1}};
} else if (!child_table) test.opcodes.insert(test.opcodes.end(),
{O::S_BUFFER_LOAD_DWORDX2, O::S_LOAD_DWORDX8, O::S_LOAD_DWORDX4});
const auto compiled = CompileCase(test, vulkan.SubgroupSize());
if (child_table) {
std::vector<u32> candidates;
for (const auto &image : compiled.resources.images)
if (image.dwords[0] != 0u) candidates.push_back(image.dwords[0]);
std::ranges::sort(candidates);
std::vector<u32> expected_candidates(records_count * children);
std::iota(expected_candidates.begin(), expected_candidates.end(), 10u);
Require(test.name, "complete child domain", candidates == expected_candidates,
"parent/child planning omitted a record or admitted poisoned padding");
Require(test.name, "GPU selectors remain unread by materialization",
std::ranges::none_of(compiled.resources.specialization_reads, [&](const auto &span) {
return span.first < keys_address + wave * 8u && keys_address < span.first + span.second;
}), "host planning captured GPU-only selector bytes");
if (counted)
Require(test.name, "runtime parent count is captured",
std::ranges::any_of(compiled.resources.specialization_reads, [&](const auto &span) {
return span.first <= count_address && count_address - span.first < span.second &&
sizeof(u32) <= span.second - (count_address - span.first);
}), "the proven scalar count was absent from the captured alias ranges");
}
if (compact) {
std::vector<u32> candidates;
for (const auto &image : compiled.resources.images) candidates.push_back(image.dwords[0]);
std::ranges::sort(candidates);
Require(test.name, "paired R128 descriptor boundary",
candidates == std::vector<u32>{10u, 11u, 12u, 13u, 14u, 15u} &&
std::ranges::all_of(compiled.resources.images, [](const auto &image) {
return image.dword_count == 8u &&
std::ranges::all_of(std::span(image.dwords).subspan(4u),
[](u32 word) { return word == 0u; });
}), "paired selector admitted poison or omitted the trailing 16-byte image");
}
for (const auto &value : compiled.resources.buffers) {
ShaderBufferResource descriptor{};
std::copy_n(value.dwords.begin(), 4, descriptor.fields);
test.storage_buffer_ranges.emplace_back(descriptor.Base48(),
std::max(4u, u32(descriptor.GetSize())));
}
std::vector<VulkanHarness::Image> images;
for (const auto &descriptor : compiled.resources.images)
images.push_back(vulkan.CreateImage2D(test.name, 2, 1, vk::Format::eR32Sfloat,
vk::ImageUsageFlagBits::eSampled,
{0u, std::bit_cast<u32>(float(descriptor.dwords[0]))}, 1,
vk::ImageLayout::eShaderReadOnlyOptimal));
std::vector<vk::Sampler> samplers;
for (const auto &resource : compiled.program.info.samplers) {
ShaderSamplerResource descriptor{};
std::copy_n(compiled.resources.samplers[resource.snapshot_index].dwords.begin(),
4, descriptor.fields);
samplers.push_back(vulkan.CreateSampler(test.name, 1,
descriptor.XyMagFilter() == 1 ? vk::Filter::eLinear : vk::Filter::eNearest));
}
TestCase producer;
producer.name = "NestedPointerSelectorProducer";
producer.initial = test.initial;
producer.has_compute_info = true;
producer.compute_info.threads_num[0] = producer.compute_info.threads_num[1] =
producer.compute_info.threads_num[2] = 1;
producer.compute_info.wave_size = 32;
test.expected = test.initial;
for (u32 group = 0; group < (child_table ? wave : keys.size()); ++group) {
const u32 outer = child_table ? (group % 8u == 7u ? UINT32_MAX : group % rows) : keys[group];
const u32 child = child_table ? (group / 8u) % 4u : 0u;
const u32 address = keys_address + group * (child_table ? 8u : 4u);
AppendVMovLiteral(&producer.code, 1, child_table ? outer : (outer << 16u) | 0xbeefu);
AppendVMovLiteral(&producer.code, 2, address);
AppendBufferStoreDword(&producer.code, 1, 2);
test.expected[address / 4u] = child_table ? outer : (outer << 16u) | 0xbeefu;
if (child_table) {
AppendVMovLiteral(&producer.code, 1, child);
AppendVMovLiteral(&producer.code, 2, address + 4u);
AppendBufferStoreDword(&producer.code, 1, 2);
test.expected[address / 4u + 1u] = child;
if (upper_only && group < 32u) continue;
test.expected[wave + group] = 0x76543210u;
if (outer >= records_count || child >= children) continue;
}
const u32 row = compact ? list[outer] : outer;
test.expected[group] = std::bit_cast<u32>(float(10u + row * children + child) *
((row & 1u) ? 0.5f : 1.0f));
}
AppendEnd(&producer.code);
const auto producer_shader = CompileCase(producer, vulkan.SubgroupSize());
auto buffer = vulkan.CreateStorageBuffer(test.name, test.initial, test.initial.size(), true);
vulkan.Dispatch(producer, producer_shader, buffer);
vulkan.Dispatch(test, compiled, buffer, nullptr, nullptr, nullptr, nullptr,
nullptr, images, samplers);
CompareWords(test, "GPU-selected parent/child, paired sampler, EXEC restoration and canaries",
test.expected, vulkan.ReadBuffer(test.name, buffer, test.expected.size()));
vulkan.DestroyBuffer(&buffer);
for (const auto sampler : samplers) vulkan.Device().destroySampler(sampler);
for (auto &image : images) vulkan.DestroyImage(&image);
std::printf("[compute] %s wave=%u ok\n", test.name, wave);
}
}
void CheckGuardedRecordImageTable(VulkanHarness &vulkan) {
using O = ShaderOpcode;
constexpr u32 table = 256u;
for (const u32 variant : {0u, 1u, 2u, 3u})
for (const u32 wave : {32u, 64u})
for (const u32 count : {0u, 2u, 3u}) {
const bool masked = variant == 1u, bitset = variant == 2u, witnessed = variant == 3u;
const u32 stride = bitset ? 32u : masked ? 784u : 272u;
const u32 offset = bitset ? 3344u : masked ? 672u : witnessed ? 32856u : 3696u;
const u32 count_offset = witnessed ? 35948u : masked ? 0u : 4u;
const u32 rows = bitset ? 12u : count, members = bitset ? 1u : 2u;
const u32 lanes = masked || witnessed ? 2u : 1u;
TestCase test;
test.name = witnessed ? (wave == 32u ? "WitnessedRecordTable32" : "WitnessedRecordTable64")
: bitset ? (wave == 32u ? "BitsetRecordTable32" : "BitsetRecordTable64")
: masked ? (wave == 32u ? "ExecGuardedRecordTable32" : "ExecGuardedRecordTable64")
: (wave == 32u ? "GuardedRecordImageTable32" : "GuardedRecordImageTable64");
test.initial.resize((table + std::max(offset + stride * rows, count_offset + 4u)) / 4u, 0xdeadbeefu);
test.initial[(table + count_offset) / 4u] = bitset ? (1u << count) - 1u : count;
for (u32 row = 0; row < rows; ++row)
for (u32 member = 0; member < members; ++member) {
auto image = MakeSampledTextureData(Prospero::BufferFormat::k32Float);
image[0] = (row + 1u) * (member == 0u ? 1u : 10u);
image[3] = DstSel(4, 0, 0, 1) |
(static_cast<u32>(Prospero::ImageType::kColor2D) << 28u);
std::copy_n(image.begin(), 8,
test.initial.begin() + (table + offset + row * stride + member * 32u) / 4u);
}
test.user_data[0] = table;
auto destination = MakeSampledTextureData(Prospero::BufferFormat::k32Float);
destination[0] = 0x10000u;
destination[1] |= (4u * lanes - 1u) << 30u;
destination[2] |= (4u * lanes - 1u) >> 2u;
destination[3] = DstSel(4, 0, 0, 1) |
(static_cast<u32>(Prospero::ImageType::kColor2D) << 28u);
std::copy_n(destination.begin(), 8, test.user_data.begin() + 32);
test.has_user_data = test.has_compute_info = true;
test.compute_info.wave_size = wave;
test.compute_info.threads_num[0] = lanes;
test.compute_info.threads_num[1] = test.compute_info.threads_num[2] = 1u;
test.compute_info.thread_ids_num = 1u;
test.compute_info.group_id[0] = true;
test.compute_info.workgroup_register = 44u;
test.dispatch_x = 4u;
auto &code = test.code;
code.insert(code.end(), {EncodeSop2(0x20, 20, InlineU32(5), 44),
EncodeSmem0(0x00, 21), EncodeSmem1(count_offset, 125)});
if (bitset) code.insert(code.end(), {
EncodeSop2(0x0e, 20, 20, 21),
EncodeSop2(0x27, 20, 20, 255), 12u << 16u});
code.push_back(EncodeSop1(0x04, 30, 126)); // Preserve EXEC for the final write.
code.push_back(EncodeVop1(0x01, 13, Vgpr(0))); // Odd lane is already done.
AppendVMovLiteral(&code, 10, 0xdeadbeefu);
AppendVMovLiteral(&code, 0, 0u);
AppendVMovLiteral(&code, 20, 0x3f000000u);
AppendVMovLiteral(&code, 21, 0x3f000000u);
if (witnessed) code.push_back(EncodeVopc(0xd2, InlineU32(0), 13));
const auto loop = code.size();
code.push_back(EncodeSop1(0x13, 22, 20)); // Find the next set bit in the GPU worklist.
size_t empty_exit = 0;
if (!witnessed) {
code.push_back(EncodeSopc(0x07, 22, 193));
empty_exit = code.size(); code.push_back(0u);
}
size_t range_exit = 0;
if (!bitset) {
if (masked) code.insert(code.end(), {EncodeSop2(0x0b, 28, 126, InlineU32(0)),
EncodeSop1(0x04, 126, 28)});
code.push_back(EncodeSopc(0x09, 22, 21));
if (witnessed) code.insert(code.end(), {
EncodeSop2(0x0b, 4, 126, InlineU32(0)),
EncodeSopc(0x06, 193, 22),
EncodeSop2(0x0b, 106, 126, InlineU32(0)),
EncodeSop2(0x11, 106, 4, 106)});
if (masked) code.insert(code.end(), {
EncodeVopc(0xc2, 249, 22), // Mask the FF1 sentinel into s2:s3.
EncodeVopcSdwa(193, 2, 1, 6, 6, 0, 0, 0, 0, 0, 0, 1, 1),
EncodeVop2(0x1c, 10, 253, 13), // SCC encoded as 0/1, OR per-lane done.
EncodeVopc(0xc5, InlineU32(0), 10),
EncodeSop2(0x11, 106, 106, 2),
EncodeSop2(0x15, 126, 28, 106)});
range_exit = code.size(); code.push_back(0u);
}
code.insert(code.end(), {EncodeSop2(0x26, 23, 22, 255), stride,
EncodeSop2(0x02, 23, 23, 255), offset});
size_t skip_first = 0;
if (!bitset) {
code.push_back(EncodeSopc(0x0d, 22, InlineU32(0)));
skip_first = code.size(); code.push_back(0u);
}
code.insert(code.end(), {EncodeSmem0(0x03, 8), EncodeSmem1(0, 23),
EncodeMimg0(0x27, 1), EncodeMimg1(4, 20, 2, 10),
EncodeVop2(0x03, 0, Vgpr(4), 0)});
if (bitset) {
// An optional second sample reuses the dominating descriptor load.
code.push_back(EncodeSopc(0x0d, 22, InlineU32(0)));
const auto skip_second = code.size(); code.push_back(0u);
code.insert(code.end(), {EncodeMimg0(0x27, 1), EncodeMimg1(4, 20, 2, 10),
EncodeVop2(0x03, 0, Vgpr(4), 0)});
code[skip_second] = EncodeSopp(0x05, code.size() - skip_second - 1u);
} else {
code[skip_first] = EncodeSopp(0x05, code.size() - skip_first - 1u);
// Both sides of the optional first sample reach the second descriptor.
code.insert(code.end(), {EncodeSop2(0x02, 23, 23, InlineU32(32)),
EncodeSmem0(0x03, 8), EncodeSmem1(0, 23),
EncodeMimg0(0x27, 1), EncodeMimg1(4, 20, 2, 10),
EncodeVop2(0x03, 0, Vgpr(4), 0)});
}
code.insert(code.end(), {EncodeSop2(0x1e, 24, InlineU32(1), 22),
EncodeSop2(0x14, 20, 20, 24)});
code.push_back(EncodeSopp(0x02, loop - code.size() - 1u));
if (!witnessed) code[empty_exit] = EncodeSopp(0x04, code.size() - empty_exit - 1u);
if (!bitset) code[range_exit] =
EncodeSopp(witnessed ? 0x07 : masked ? 0x08 : 0x05, code.size() - range_exit - 1u);
code.push_back(EncodeSop1(0x04, 126, 30));
code.push_back(EncodeSop2(0x26, 25, 44, InlineU32(lanes)));
code.push_back(EncodeVop2(0x25, 30, 25, 13));
AppendVMovLiteral(&code, 31, 0u);
code.insert(code.end(), {EncodeMimg0(0x08, 1), EncodeMimg1(0, 30, 8, 0)});
AppendEnd(&code);
test.opcodes = {O::S_LSHR_B32, O::S_LOAD_DWORD, O::V_MOV_B32, O::S_FF1_I32_B32,
O::S_MUL_I32, O::S_ADD_I32, O::S_LOAD_DWORDX8, O::IMAGE_SAMPLE,
O::V_ADD_F32, O::S_LSHL_B32, O::S_ANDN2_B32, O::S_BRANCH,
O::IMAGE_STORE, O::S_ENDPGM};
if (witnessed) test.opcodes.insert(test.opcodes.end(), {O::V_CMPX_EQ_U32,
O::S_CSELECT_B64, O::S_CMP_EQ_U32, O::S_OR_B64, O::S_CBRANCH_VCCNZ});
else test.opcodes.insert(test.opcodes.end(), {O::S_CMP_LG_U32, O::S_CBRANCH_SCC0});
if (bitset) test.opcodes.insert(test.opcodes.end(), {O::S_AND_B32, O::S_BFE_U32,
O::S_BITCMP1_B32, O::S_CBRANCH_SCC1});
else test.opcodes.insert(test.opcodes.end(), {O::S_CMP_GE_U32, O::S_CBRANCH_SCC1,
O::S_BITCMP1_B32});
if (masked) test.opcodes.insert(test.opcodes.end(), {O::S_CSELECT_B64,
O::V_CMP_EQ_U32, O::V_OR_B32, O::V_CMP_NE_U32, O::S_OR_B64,
O::S_ANDN2_B64, O::S_CBRANCH_EXECZ});
test.opcodes.insert(test.opcodes.end(), {O::S_MOV_B64, O::V_ADD_NC_U32});
auto compiled = CompileCase(test, vulkan.SubgroupSize());
const auto sampled_count = std::ranges::count_if(compiled.program.info.images,
[](const auto &image) {
return image.resource_class == ShaderRecompiler::IR::ImageResourceClass::Sampled;
});
Require(test.name, "bounded descriptor rows",
rows == 0u ? sampled_count <= members : sampled_count == rows * members,
"the runtime count lost a row or probed record padding");
std::vector<VulkanHarness::Image> images(compiled.program.info.images.size());
u32 storage_index = 0;
for (u32 resource = 0; resource < images.size(); ++resource) {
if (compiled.program.info.images[resource].resource_class ==
ShaderRecompiler::IR::ImageResourceClass::Storage) {
storage_index = resource;
images[resource] = vulkan.CreateImage2D(test.name, 4u * lanes, 1, vk::Format::eR32Sfloat,
vk::ImageUsageFlagBits::eStorage, std::vector<u32>(4u * lanes, 0xdeadbeefu), 1,
vk::ImageLayout::eGeneral);
continue;
}
const auto value = static_cast<float>(compiled.resources.images[resource].dwords[0]);
images[resource] = vulkan.CreateImage2D(test.name, 1, 1, vk::Format::eR32Sfloat,
vk::ImageUsageFlagBits::eSampled, {std::bit_cast<u32>(value)}, 1,
vk::ImageLayout::eShaderReadOnlyOptimal);
}
auto buffer = vulkan.CreateStorageBuffer(test.name, test.initial, test.initial.size());
auto sampler = vulkan.CreateSampler(test.name, 1);
vulkan.Dispatch(test, compiled, buffer, nullptr, nullptr, nullptr, nullptr, sampler, images);
std::vector<u32> expected(4u * lanes, 0u);
if (count != 0u) {
expected[0] = std::bit_cast<u32>(bitset ? (count == 3 ? 8.0f : 2.0f)
: (count == 3 ? 44.0f : 11.0f));
expected[lanes] = std::bit_cast<u32>(bitset ? 2.0f : 20.0f);
expected[2u * lanes] = std::bit_cast<u32>(bitset ? 2.0f : 11.0f);
}
CompareWords(test, "GPU worklist, guarded count and image writes", expected,
vulkan.ReadImage(test.name, &images[storage_index]));
vulkan.DestroyBuffer(&buffer);
vulkan.Device().destroySampler(sampler);
for (auto &image : images) if (image.image != nullptr) vulkan.DestroyImage(&image);
std::printf("[compute] %s count=%u ok\n", test.name, count);
}
}
void CheckIndirectImageKeySwitch(VulkanHarness &vulkan) {
constexpr const char *name = "IndirectImageKeySwitch";
constexpr uint32_t mapping_capacity = 1793u;
using namespace ShaderRecompiler::IR;
Program program{};
program.stage = ShaderType::Compute;
program.wave_size = 32;
program.srt_plan_complete = true;
program.resource_tracking_complete = true;
program.shader_info_complete = true;
program.block_storage.push_back(std::make_unique<Block>());
auto *block = program.block_storage.back().get();
program.blocks.push_back(block);
auto &key = block->AppendNewInst(ValueOpcode::LaneId);
auto &image =
block->AppendNewInst(ValueOpcode::GetImageResource,
{Value(&key), Value(0u), Value(0u), Value(0u),
Value(0u), Value(0u), Value(0u), Value(0u)});
image.SetFlags<uint32_t>(0u);
auto &sampler =
block->AppendNewInst(ValueOpcode::GetSamplerResource,
{Value(0u), Value(0u), Value(0u), Value(0u)});
sampler.SetFlags<uint32_t>(0u);
auto &address = block->AppendNewInst(
ValueOpcode::MakeImageAddress,
{Value(0u), Value(0u), Value(0u), Value(0u), Value(0u), Value(0u),
Value(0u), Value(0u), Value(0u), Value(0u), Value(0u), Value(0u),
Value(0u)});
MemoryInfo memory{};
memory.kind = ResourceKind::Image;
memory.resource = 0;
memory.sampler = 0;
memory.dmask = 0xf;
memory.image_sample_flags = ShaderRecompiler::Decoder::ImageSampleFlagLod;
memory.image_dimension = ShaderRecompiler::Decoder::ImageDimension::Dim2D;
memory.image_address_components = 3;
program.memory_info.push_back(memory);
const MemoryFlags memory_flags{0u, 0x10f0u};
uint64_t memory_flag_bits = 0;
std::memcpy(&memory_flag_bits, &memory_flags, sizeof(memory_flags));
auto &sample = block->AppendNewInst(
ValueOpcode::ImageSampleRaw,
{Value(&image), Value(&sampler), Value(&address), Value(true)}, memory_flag_bits);
auto &sample_x = block->AppendNewInst(ValueOpcode::CompositeExtractU32x4,
{Value(&sample), Value(0u)});
block->AppendNewInst(ValueOpcode::ReferenceU32, {Value(&sample_x)});
program.descriptor_sources.resize(2);
program.descriptor_sources[0].dword_count = 8;
program.descriptor_sources[0].indirect_descriptor =
DescriptorSource::IndirectDescriptor{
.selector = DescriptorSource::IndirectDescriptor::SelectorRead{
.source = 0, .stride = 224, .offset = 12},
.table_source = 0, .table_stride = 32};
program.descriptor_sources[1].dword_count = 4;
ImageResource root{};
root.source = 0;
root.first_use_pc = 0x10f0u;
root.resource_class = ImageResourceClass::Sampled;
root.numeric_class = Prospero::TextureNumericClass::Float;
root.dimension = ShaderRecompiler::Decoder::ImageDimension::Dim2D;
root.read = true;
root.indirect_root = 0;
root.indirect_mapping_offset = 0;
root.indirect_search_iterations = std::bit_width(mapping_capacity);
root.indirect_resources = {0u, 1u};
auto candidate = root;
candidate.indirect_search_iterations = 0;
candidate.indirect_resources.clear();
program.info.images = {root, candidate};
program.info.samplers.push_back({1u, 0x10f0u});
program.info.sampled_pairs.push_back({0u, 0u, 0x10f0u});
ShaderComputeInputInfo compute{};
program.shader_info_complete = false;
CollectShaderInfo(program, {.compute = &compute});
auto spirv = ShaderRecompiler::Spirv::EmitProgram(program,
{.compute = &compute});
ValidateSpirv(name, spirv);
spvtools::SpirvTools tools(SPV_ENV_VULKAN_1_2);
std::string text;
Require(name, "SPIR-V disassembly", tools.Disassemble(spirv, &text),
"failed to disassemble indirect image shader");
Require(name, "homogeneous image array",
text.find("OpSwitch") == std::string::npos &&
CountText(text, "OpImageSampleExplicitLod") == 1 &&
text.find("SampledImageArrayNonUniformIndexing") != std::string::npos,
"homogeneous image keys did not select one descriptor array sample");
u32 sampled_image = 0;
std::vector<u32> nonuniform;
for (size_t offset = 5; offset < spirv.size();) {
const auto words = std::span<const u32>(spirv).subspan(offset, spirv[offset] >> 16u);
const auto opcode = static_cast<spv::Op>(words[0] & 0xffffu);
if (opcode == spv::OpDecorate && words[2] == spv::DecorationNonUniform)
nonuniform.push_back(words[1]);
if (opcode == spv::OpImageSampleExplicitLod) sampled_image = words[3];
offset += words.size();
}
Require(name, "nonuniform sampled image operand",
sampled_image != 0u && std::ranges::find(nonuniform, sampled_image) != nonuniform.end(),
"the sample operand lacks its nonuniform decoration");
program.info.images[0].indirect_search_iterations = 0;
program.shader_info_complete = false;
CollectShaderInfo(program, {.compute = &compute});
spirv = ShaderRecompiler::Spirv::EmitProgram(program, {.compute = &compute});
ValidateSpirv("DenseIndirectImageTable", spirv);
Require(name, "dense image map disassembly", tools.Disassemble(spirv, &text),
"failed to disassemble dense image table shader");
Require(name, "bounded dense image selection",
CountText(text, "OpULessThan") == 1 &&
CountText(text, "OpImageSampleExplicitLod") == 1 &&
text.find("SampledImageArrayNonUniformIndexing") != std::string::npos,
"dense image selection lost its bounds check or native nonuniform array");
// Materialized children follow all native roots, including unrelated images.
root.indirect_resources = {0u};
program.info.images = {root, candidate};
program.info.images[1].indirect_root = ImageResource::NoIndirectImage;
for (u32 ordinal = 1; ordinal < 189u; ++ordinal) {
root.indirect_resources.push_back(static_cast<u32>(program.info.images.size()));
program.info.images.push_back(candidate);
}
program.info.images[0] = root;
program.shader_info_complete = false;
CollectShaderInfo(program, {.compute = &compute});
spirv = ShaderRecompiler::Spirv::EmitProgram(program, {.compute = &compute});
ValidateSpirv(name, spirv);
Require(name, "large image array disassembly", tools.Disassemble(spirv, &text),
"failed to disassemble the materialized image array");
Require(name, "large homogeneous image array",
text.find("OpSwitch") == std::string::npos &&
CountText(text, "OpImageSampleExplicitLod") == 1,
"materialized images expanded into per-candidate samples");
program.info.images[root.indirect_resources[103]].dimension =
ShaderRecompiler::Decoder::ImageDimension::Dim2DArray;
program.info.images[root.indirect_resources[103]].cube = true;
program.shader_info_complete = false;
CollectShaderInfo(program, {.compute = &compute});
spirv = ShaderRecompiler::Spirv::EmitProgram(program, {.compute = &compute});
ValidateSpirv(name, spirv);
Require(name, "mixed contiguous image runs", tools.Disassemble(spirv, &text) &&
CountText(text, "OpImageSampleExplicitLod") == 3,
"one cube split the native arrays into per-image samples");
program.memory_info[0].image_dimension =
ShaderRecompiler::Decoder::ImageDimension::Dim2DArray;
program.memory_info[0].image_address_components = 4;
for (u32 component = 0; component < 4u; component++) {
constexpr std::array values{1.375f, 1.625f, 10.0f, 2.0f};
address.SetArg(component, Value(std::bit_cast<u32>(values[component])));
}
root.indirect_resources = {0u, 1u, 2u};
program.info.images = {root, candidate, candidate};
for (const bool cube_first : {true, false}) {
for (u32 resource = 0; resource < 3u; resource++) {
auto &image_resource = program.info.images[resource];
image_resource.cube = resource == (cube_first ? 0u : 1u);
image_resource.dimension =
resource == (cube_first ? 1u : 0u)
? ShaderRecompiler::Decoder::ImageDimension::Dim2D
: ShaderRecompiler::Decoder::ImageDimension::Dim2DArray;
}
program.shader_info_complete = false;
CollectShaderInfo(program, {.compute = &compute});
spirv = ShaderRecompiler::Spirv::EmitProgram(program, {.compute = &compute});
ValidateSpirv(name, spirv);
Require(name, "mixed SPIR-V disassembly", tools.Disassemble(spirv, &text),
"failed to disassemble mixed indirect image shader");
Require(name, "mixed key switch",
CountText(text, "OpImageSampleExplicitLod") == 3 &&
CountText(text, "OpFSub") == 2,
"mixed candidates lost a sample or shared cube coordinate conversion");
std::vector<std::span<const u32>> definitions(spirv[3]);
u32 samples = 0;
for (size_t offset = 5; offset < spirv.size();) {
const auto words = std::span<const u32>(spirv).subspan(offset, spirv[offset] >> 16u);
const auto opcode = static_cast<spv::Op>(words[0] & 0xffffu);
if (opcode == spv::OpTypeVector) {
definitions[words[1]] = words;
} else if (opcode == spv::OpCompositeConstruct || opcode == spv::OpBitcast ||
opcode == spv::OpConstant) {
definitions[words[2]] = words;
} else if (opcode == spv::OpImageSampleExplicitLod) {
const auto coord = definitions[words[4]];
const auto lod = definitions[words[6]];
const auto components = samples == (cube_first ? 1u : 0u) ? 2u : 3u;
Require(name, "mixed coordinate and LOD layout",
!coord.empty() && definitions[coord[1]][3] == components &&
words[5] == spv::ImageOperandsLodMask && lod.size() == 4u &&
definitions[lod[3]].size() == 4u &&
definitions[lod[3]][3] == std::bit_cast<u32>(2.0f),
"candidate coordinates changed the instruction's LOD operand");
samples++;
}
offset += words.size();
}
Require(name, "mixed sample count", samples == 3u,
"mixed candidate switch did not retain every image");
}
// Keep an unrelated native root between the null image and appended children,
// then select both sides of a cube boundary and the unmapped suffix on the GPU.
root.indirect_resources = {0u, 2u, 3u, 4u, 5u, 6u};
root.indirect_search_iterations = 0;
program.info.images.assign(7, candidate);
program.info.images[0] = root;
program.info.images[1].indirect_root = ImageResource::NoIndirectImage;
program.info.images[4].dimension =
ShaderRecompiler::Decoder::ImageDimension::Dim2DArray;
program.info.images[4].cube = true;
for (u32 component = 0; component < 4u; ++component) {
constexpr std::array values{1.5f, 1.5f, 0.0f, 0.0f};
address.SetArg(component, Value(std::bit_cast<u32>(values[component])));
}
auto &byte_offset = *block->PrependNewInst(
std::next(block->begin()), ValueOpcode::IMul32, {Value(&key), Value(4u)});
image.SetArg(0, Value(&byte_offset));
auto &output = block->AppendNewInst(ValueOpcode::GetBufferResource,
{Value(0u), Value(0u), Value(0u), Value(0u)});
output.SetFlags<u32>(0u);
program.memory_info.push_back({.kind = ResourceKind::Buffer, .offen = true});
auto &store = block->AppendNewInst(
ValueOpcode::StoreBufferU32,
{Value(&output), Value(0u), Value(&byte_offset), Value(0u), Value(&sample_x), Value(true)});
store.SetFlags(MemoryFlags{1u, 0u});
program.info.buffers.push_back({.packed_stride = 1, .written = true});
program.shader_info_complete = false;
CollectShaderInfo(program, {.compute = &compute});
TestCase test;
test.name = "IndirectImageContiguousRuns";
CompiledShader compiled;
compiled.program = std::move(program);
constexpr std::array ordinals{0u, 1u, 2u, 3u, 4u, 5u, 2u, 0u};
compiled.resources.flattened_srt.push_back(ordinals.size());
compiled.resources.flattened_srt.insert(compiled.resources.flattened_srt.end(),
ordinals.begin(), ordinals.end());
std::vector<VulkanHarness::Image> textures;
for (u32 resource = 0; resource < 7u; ++resource) {
const auto layers = resource == 4u ? 6u : 1u;
std::vector<u32> pixels(layers * 16u, std::bit_cast<u32>(float(resource)));
std::vector<u32> mip(layers * 4u,
std::bit_cast<u32>(resource == 0u ? 0.0f : float(resource + 10u)));
textures.push_back(vulkan.CreateImageMips(
test.name, 2, 2, vk::Format::eR32G32B32A32Sfloat,
vk::ImageUsageFlagBits::eSampled, {pixels, mip}, 4u,
vk::ImageLayout::eShaderReadOnlyOptimal, vk::ImageType::e2D,
resource == 4u ? vk::ImageViewType::e2DArray : vk::ImageViewType::e2D, layers));
}
const auto native_sampler = vulkan.CreateSampler(test.name);
enum class Access { Sample, Gather, GatherHomogeneous, Fetch, Fetch1D };
auto *access = &sample;
for (const auto mode : {Access::Sample, Access::Gather,
Access::GatherHomogeneous, Access::Fetch, Access::Fetch1D}) {
const bool fetch = mode == Access::Fetch || mode == Access::Fetch1D;
const bool gather = mode == Access::Gather || mode == Access::GatherHomogeneous;
const bool homogeneous = mode == Access::GatherHomogeneous;
if (mode == Access::Gather) {
test.name = "IndirectImageGatherContiguousRuns";
const auto position = std::ranges::find_if(*block, [&](const Inst &inst) {
return &inst == access;
});
auto &read = *block->PrependNewInst(
position, ValueOpcode::ImageGatherRaw,
{Value(&image), Value(&sampler), Value(&address), Value(0u), Value(0u),
Value(0u), Value(0u), Value(true)}, memory_flag_bits);
access->ReplaceUsesWith(Value(&read), false);
access = &read;
compiled.program.info.images[4].cube = false;
auto &gather_memory = compiled.program.memory_info[0];
gather_memory.dmask = 1;
gather_memory.image_sample_flags = ShaderRecompiler::Decoder::ImageSampleFlagLevelZero |
ShaderRecompiler::Decoder::ImageSampleFlagOffset;
// GATHER4_LZ_O places its packed signed offset before the coordinates.
address.SetArg(0, Value(0x3fu));
address.SetArg(1, Value(std::bit_cast<u32>(0.75f)));
address.SetArg(2, Value(std::bit_cast<u32>(0.25f)));
address.SetArg(3, Value(0u));
}
auto &selected_resources = compiled.program.info.images[0].indirect_resources;
selected_resources = homogeneous ? std::vector<u32>{0u, 2u, 3u, 5u, 6u}
: root.indirect_resources;
if (homogeneous) test.name = "IndirectImageGatherHomogeneous";
compiled.resources.flattened_srt[6] = homogeneous ? 4u : ordinals[5];
if (fetch) {
test.name = "IndirectImageFetchContiguousRuns";
const auto position = std::ranges::find_if(*block, [&](const Inst &inst) {
return &inst == access;
});
auto &read = *block->PrependNewInst(
position, ValueOpcode::ImageRead,
{Value(&image), Value(&address), Value(true)}, memory_flag_bits);
access->ReplaceUsesWith(Value(&read), false);
access = &read;
sampler.Invalidate();
compiled.program.info.samplers.clear();
compiled.program.info.sampled_pairs.clear();
compiled.program.info.images[4].cube = false;
compiled.program.memory_info[0].dmask = 0xf;
compiled.program.memory_info[0].image_sample_flags = 0;
compiled.program.memory_info[0].image_has_mip = true;
for (u32 component = 0; component < 4u; ++component)
address.SetArg(component, Value(0u));
// Native DIM=2DArray places LOD after the layer, even for a 2D candidate.
address.SetArg(3, Value(1u));
if (mode == Access::Fetch1D) {
test.name = "IndirectImageFetch1D2D";
compiled.program.memory_info[0].image_dimension = ShaderRecompiler::Decoder::ImageDimension::Dim2D;
compiled.program.memory_info[0].image_address_components = 3;
address.SetArg(2, Value(1u));
address.SetArg(3, Value(0u));
for (const u32 resource : {3u, 4u}) {
const bool one_d = resource == 3u;
compiled.program.info.images[resource].dimension = one_d
? ShaderRecompiler::Decoder::ImageDimension::Dim1D
: ShaderRecompiler::Decoder::ImageDimension::Dim2D;
vulkan.DestroyImage(&textures[resource]);
textures[resource] = vulkan.CreateImageMips(test.name, 2, 1, vk::Format::eR32G32B32A32Sfloat,
vk::ImageUsageFlagBits::eSampled,
{std::vector<u32>(8u, std::bit_cast<u32>(float(resource))),
std::vector<u32>(4u, std::bit_cast<u32>(float(resource + 10u)))},
4u, vk::ImageLayout::eShaderReadOnlyOptimal,
one_d ? vk::ImageType::e1D : vk::ImageType::e2D,
one_d ? vk::ImageViewType::e1D : vk::ImageViewType::e2D, 1u);
}
}
}
compiled.program.shader_info_complete = false;
CollectShaderInfo(compiled.program, {.compute = &compute});
for (const u32 wave_size : {32u, 64u}) {
compute.wave_size = wave_size;
compute.host_subgroup_size = vulkan.SubgroupSize();
compute.threads_num[0] = wave_size;
compute.threads_num[1] = compute.threads_num[2] = 1u;
compiled.program.wave_size = wave_size;
compiled.spirv = ShaderRecompiler::Spirv::EmitProgram(compiled.program, {.compute = &compute});
compiled.packed_user_data.resize(compiled.program.bindings.ShaderDataDwords());
ValidateSpirv(test.name, compiled.spirv);
const auto halves = wave_size > compute.host_subgroup_size ? 2u : 1u;
const auto image_opcode = fetch ? "OpImageFetch" : gather ? "OpImageGather"
: "OpImageSampleExplicitLod";
Require(test.name, "native array image runs", tools.Disassemble(compiled.spirv, &text) &&
CountText(text, image_opcode) == (homogeneous ? 1u : 3u) * halves,
"mixed runs expanded into per-image accesses");
if (homogeneous) {
Require(test.name, "homogeneous gather array", text.find("OpSwitch") == std::string::npos,
"homogeneous gather emitted a descriptor switch");
}
if (fetch || gather) {
std::vector<u32> decorated;
std::vector<u32> image_operands;
for (size_t offset = 5; offset < compiled.spirv.size();) {
const auto words = std::span<const u32>(compiled.spirv).subspan(
offset, compiled.spirv[offset] >> 16u);
const auto opcode = static_cast<spv::Op>(words[0] & 0xffffu);
if (opcode == spv::OpDecorate && words[2] == spv::DecorationNonUniform)
decorated.push_back(words[1]);
if (opcode == (fetch ? spv::OpImageFetch : spv::OpImageGather))
image_operands.push_back(words[3]);
offset += words.size();
}
Require(test.name, "nonuniform image operands",
std::ranges::count_if(image_operands, [&](u32 operand) {
return std::ranges::find(decorated, operand) != decorated.end();
}) == (homogeneous ? 1u : 2u) * halves,
"a dynamically selected image operand lacks its nonuniform decoration");
}
test.initial.assign(wave_size, 0xdeadbeefu);
test.expected.assign(wave_size, 0u);
for (u32 lane = 0; lane < ordinals.size(); ++lane) {
const auto resource = selected_resources[compiled.resources.flattened_srt[1u + lane]];
test.expected[lane] = std::bit_cast<u32>(float(resource + (fetch && resource != 0u ? 10u : 0u)));
}
auto buffer = vulkan.CreateStorageBuffer(test.name, test.initial, test.initial.size());
vulkan.Dispatch(test, compiled, buffer, nullptr, nullptr, nullptr, nullptr,
fetch ? vk::Sampler{} : native_sampler, textures);
const auto actual = vulkan.ReadBuffer(test.name, buffer, test.expected.size());
vulkan.DestroyBuffer(&buffer);
CompareWords(test, "root, native slots, array runs and mapping bounds", test.expected, actual);
if (wave_size == 32u && !homogeneous) {
// A single integer candidate must keep raw register bits across the
// same descriptor runs used by samples, gathers, and image fetches.
const auto coordinate_conversions = CountText(text, "OpConvertUToF");
auto &typed = compiled.program.info.images[3];
typed.numeric_class = Prospero::TextureNumericClass::Uint;
if (!fetch) {
auto integer_sampler = compiled.program.info.samplers[0];
integer_sampler.force_point_filtering = true;
integer_sampler.integer_border = true;
compiled.program.info.samplers.push_back(integer_sampler);
}
compiled.program.shader_info_complete = false;
CollectShaderInfo(compiled.program, {.compute = &compute});
const auto mixed_spirv = ShaderRecompiler::Spirv::EmitProgram(
compiled.program, {.compute = &compute});
ValidateSpirv("IndirectImageMixedNumeric", mixed_spirv);
u32 unsigned_type = 0u, raw_vector = 0u;
bool raw_join = false;
for (size_t offset = 5; offset < mixed_spirv.size();) {
const auto words = std::span<const u32>(mixed_spirv).subspan(
offset, mixed_spirv[offset] >> 16u);
const auto opcode = static_cast<spv::Op>(words[0] & 0xffffu);
if (opcode == spv::OpTypeInt && words[2] == 32u && words[3] == 0u)
unsigned_type = words[1];
if (opcode == spv::OpTypeVector && words[2] == unsigned_type && words[3] == 4u)
raw_vector = words[1];
if (opcode == spv::OpPhi && words[1] == raw_vector) raw_join = true;
offset += words.size();
}
Require(test.name, "mixed numeric image result", raw_join &&
tools.Disassemble(mixed_spirv, &text) &&
CountText(text, image_opcode) == (mode == Access::Fetch1D ? 3u : 4u) * halves &&
CountText(text, "OpConvertUToF") == coordinate_conversions,
"mixed image classes lost raw result bits or expanded per image");
typed.numeric_class = Prospero::TextureNumericClass::Float;
if (!fetch) compiled.program.info.samplers.pop_back();
compiled.program.shader_info_complete = false;
CollectShaderInfo(compiled.program, {.compute = &compute});
}
}
std::printf("[compute] %-32s ok\n", test.name);
}
vulkan.Device().destroySampler(native_sampler);
for (auto &texture : textures) vulkan.DestroyImage(&texture);
textures.clear();
// Reuse the GPU key map, including the separated root slot and unmapped suffix,
// for storage writes. Every lane has a distinct texel address.
store.Invalidate();
for (auto &inst : *block)
if (inst.GetOpcode() == ValueOpcode::ReferenceU32) inst.Invalidate();
compiled.program.info.buffers.clear();
compiled.program.memory_info.resize(1);
auto &store_memory = compiled.program.memory_info[0];
store_memory.image_dimension = ShaderRecompiler::Decoder::ImageDimension::Dim2D;
store_memory.image_address_components = 2;
store_memory.image_has_mip = false;
address.SetArg(0, Value(&key));
address.SetArg(2, Value(0u));
address.SetArg(3, Value(0u));
for (auto &resource : compiled.program.info.images) {
resource.resource_class = ImageResourceClass::Storage;
resource.dimension = ShaderRecompiler::Decoder::ImageDimension::Dim2D;
resource.cube = false;
resource.read = false;
resource.written = true;
}
auto &written_x = block->AppendNewInst(ValueOpcode::IAdd32,
{Value(&key), Value(0x3f800000u)});
auto &texel = block->AppendNewInst(ValueOpcode::CompositeConstructU32x4,
{Value(&written_x), Value(0x40000000u), Value(0x40400000u), Value(0x40800000u)});
auto &write_exec = block->AppendNewInst(ValueOpcode::INotEqual32,
{Value(&key), Value(2u)});
block->AppendNewInst(ValueOpcode::ImageWrite,
{Value(&image), Value(&address), Value(&texel), Value(&write_exec)}, memory_flag_bits);
EliminateDeadCode(compiled.program.blocks);
for (const bool mixed : {false, true}) {
test.name = mixed ? "IndirectImageStoreMixed" : "IndirectImageStoreHomogeneous";
compiled.program.info.images[3].dimension = mixed
? ShaderRecompiler::Decoder::ImageDimension::Dim1D
: ShaderRecompiler::Decoder::ImageDimension::Dim2D;
compiled.program.info.images[5].shader_swizzle = mixed ? DstSel(5, 4, 6, 7)
: ShaderImageIdentitySwizzle;
compiled.program.info.images[6].shader_swizzle = mixed ? 0u : ShaderImageIdentitySwizzle;
compiled.program.shader_info_complete = false;
CollectShaderInfo(compiled.program, {.compute = &compute});
for (const u32 wave_size : {32u, 64u}) {
compute.wave_size = wave_size;
compute.threads_num[0] = wave_size;
compiled.program.wave_size = wave_size;
compiled.spirv = ShaderRecompiler::Spirv::EmitProgram(compiled.program, {.compute = &compute});
compiled.packed_user_data.resize(compiled.program.bindings.ShaderDataDwords());
ValidateSpirv(test.name, compiled.spirv);
const auto halves = wave_size > compute.host_subgroup_size ? 2u : 1u;
Require(test.name, "native array writes per dimension and swizzle run",
tools.Disassemble(compiled.spirv, &text) &&
CountText(text, "OpImageWrite") == (mixed ? 5u : 1u) * halves &&
(mixed || text.find("OpSwitch") == std::string::npos) &&
text.find("StorageImageArrayNonUniformIndexing") != std::string::npos,
"storage candidates lost their typed native array runs");
std::vector<std::vector<u32>> expected_images;
for (u32 resource = 0; resource < compiled.program.info.images.size(); ++resource) {
expected_images.emplace_back((wave_size + 2u) * 4u, 0x41000000u + resource);
const bool one_d = compiled.program.info.images[resource].dimension ==
ShaderRecompiler::Decoder::ImageDimension::Dim1D;
textures.push_back(vulkan.CreateImageMips(test.name, wave_size + 2u, 1,
vk::Format::eR32G32B32A32Sfloat, vk::ImageUsageFlagBits::eStorage,
{expected_images.back()}, 4u, vk::ImageLayout::eGeneral,
one_d ? vk::ImageType::e1D : vk::ImageType::e2D,
one_d ? vk::ImageViewType::e1D : vk::ImageViewType::e2D, 1u));
}
test.initial = {0xdeadbeefu};
for (u32 lane = 0; lane < wave_size; ++lane) {
const auto resource = root.indirect_resources[lane < ordinals.size() ? ordinals[lane] : 0u];
if (lane == 2u) continue;
std::array written{0x3f800000u + lane, 0x40000000u, 0x40400000u, 0x40800000u};
if (mixed && resource == 5u) std::swap(written[0], written[1]);
if (mixed && resource == 6u) written.fill(0u);
std::copy(written.begin(), written.end(), expected_images[resource].begin() + lane * 4u);
}
auto buffer = vulkan.CreateStorageBuffer(test.name, test.initial, test.initial.size());
vulkan.Dispatch(test, compiled, buffer, nullptr, nullptr, nullptr, nullptr, nullptr, textures);
for (u32 resource = 0; resource < textures.size(); ++resource) {
CompareWords(test, "selected writes, inactive lane and neighboring texels",
expected_images[resource], vulkan.ReadImage(test.name, &textures[resource]));
vulkan.DestroyImage(&textures[resource]);
}
textures.clear();
vulkan.DestroyBuffer(&buffer);
std::printf("[compute] %s wave=%u ok\n", test.name, wave_size);
}
}
}
TestCase ImageStoreMipSelectsPpsa01340Descriptor() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 20, 0);
AppendVMovU32(&code, 21, 0);
AppendVMovU32(&code, 22, 2);
AppendVMovU32(&code, 23, 0);
AppendVMovLiteral(&code, 0, 0x3f800000u);
AppendVMovLiteral(&code, 1, 0x40000000u);
AppendVMovLiteral(&code, 2, 0x40400000u);
AppendVMovLiteral(&code, 3, 0x40800000u);
code.push_back(EncodeMimg0(0x09, 0xf));
code.push_back(EncodeMimg1(0, 20));
AppendEnd(&code);
TestCase test;
test.name = "ImageStoreMipSelectsPpsa01340Descriptor";
test.code = std::move(code);
test.opcodes = {O::V_MOV_B32, O::IMAGE_STORE_MIP, O::S_ENDPGM};
const std::array<u32, 8> descriptor{0x0294dc00u, 0xc4700000u, 0x00b3c13fu,
0x91b31facu, 0x00000000u, 0x00700030u,
0xb07b0000u, 0x0002ac3cu};
std::copy(descriptor.begin(), descriptor.end(), test.user_data.begin());
test.has_user_data = true;
test.compile_only = true;
test.expected_mip_descriptors = 3;
test.required_spirv = {"OpSwitch", "OpImageWrite"};
return test;
}
template <u32 mode> TestCase ImageStoreSingleMipBounds() {
using O = ShaderOpcode;
constexpr std::array names{"ImageStoreSingleMipValid", "ImageStoreSingleMipBeyondEnd",
"ImageStoreSingleMipNegative"};
constexpr std::array lods{0u, 1u, UINT32_MAX};
TestCase test;
test.name = names[mode];
AppendVMovU32(&test.code, 20, 0);
AppendVMovU32(&test.code, 21, 0);
AppendVMovU32(&test.code, 22, lods[mode]);
for (u32 component = 0; component < 4; component++) {
AppendVMovLiteral(&test.code, component, 0x40000000u);
}
test.code.push_back(EncodeMimg0(0x09, 0xf));
test.code.push_back(EncodeMimg1(0, 20));
AppendEnd(&test.code);
test.opcodes = {O::V_MOV_B32, O::IMAGE_STORE_MIP, O::S_ENDPGM};
test.user_data = MakeStorageTextureData(Prospero::BufferFormat::k32_32_32_32Float);
test.has_user_data = true;
test.storage_image_rgba = std::vector<u32>(4 * 4 * 4, 0x3f800000u);
test.expected_storage_image_rgba = test.storage_image_rgba;
if constexpr (mode == 0) {
std::fill_n(test.expected_storage_image_rgba.begin(), 4, 0x40000000u);
}
test.expected_mip_descriptors = 1;
if constexpr (mode == 0) {
test.required_spirv = {"OpImageWrite"};
}
return test;
}
TestCase ImageStoreRgbOneUsesInverseSwizzle() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 20, 1);
AppendVMovU32(&code, 21, 2);
AppendVMovU32(&code, 22, 0);
AppendVMovLiteral(&code, 0, 0x3f800000u);
AppendVMovLiteral(&code, 1, 0x3f008081u);
AppendVMovLiteral(&code, 2, 0x3e808081u);
AppendVMovLiteral(&code, 3, 0x3f40c0c1u);
code.push_back(EncodeMimg0(0x08, 0xf));
code.push_back(EncodeMimg1(0, 20));
AppendEnd(&code);
TestCase test;
test.name = "ImageStoreRgbOneUsesInverseSwizzle";
test.code = code;
test.opcodes = {O::V_MOV_B32, O::IMAGE_STORE, O::S_ENDPGM};
test.has_user_data = true;
test.user_data[0] = 0x1000u;
test.user_data[1] =
static_cast<uint32_t>(Prospero::BufferFormat::k8_8_8_8UNorm) << 20u;
test.image_descriptor_swizzle = DstSel(4, 5, 6, 1);
test.storage_image_format = vk::Format::eR8G8B8A8Unorm;
test.storage_image_dwords_per_pixel = 1;
test.storage_image_rgba = std::vector<u32>(16, 0);
test.expected_storage_image_rgba = std::vector<u32>(16, 0);
test.expected_storage_image_rgba[2 * 4 + 1] = 0x004080ffu;
return test;
}
TestCase ImageStoreDuplicateSelectorUsesInverseSwizzle() {
auto test = ImageStoreRgbOneUsesInverseSwizzle();
test.name = "ImageStoreDuplicateSelectorUsesInverseSwizzle";
test.image_descriptor_swizzle = DstSel(4, 4, 6, 7);
test.expected_storage_image_rgba[2 * 4 + 1] = 0xc04000ffu;
return test;
}
TestCase ImageStoreBgraUsesInverseSwizzle() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 20, 1);
AppendVMovU32(&code, 21, 2);
AppendVMovU32(&code, 22, 0);
AppendVMovLiteral(&code, 0, 0x3f800000u);
AppendVMovLiteral(&code, 1, 0x3f008081u); // 128/255
AppendVMovLiteral(&code, 2, 0x3e808081u); // 64/255
AppendVMovLiteral(&code, 3, 0x3f3fbfc0u); // 191/255
code.push_back(EncodeMimg0(0x08, 0xf));
code.push_back(EncodeMimg1(0, 20));
AppendEnd(&code);
TestCase test;
test.name = "ImageStoreBgraUsesInverseSwizzle";
test.code = code;
test.opcodes = {O::V_MOV_B32, O::IMAGE_STORE, O::S_ENDPGM};
test.has_user_data = true;
test.user_data[0] = 0x1000u;
test.user_data[1] =
static_cast<uint32_t>(Prospero::BufferFormat::k8_8_8_8UNorm) << 20u;
test.image_descriptor_swizzle = DstSel(6, 5, 4, 7);
test.storage_image_format = vk::Format::eR8G8B8A8Unorm;
test.storage_image_dwords_per_pixel = 1;
test.storage_image_rgba = std::vector<u32>(16, 0);
test.expected_storage_image_rgba = std::vector<u32>(16, 0);
test.expected_storage_image_rgba[2 * 4 + 1] = 0xbfff8040u;
return test;
}
TestCase ImageStoreYzwxUsesInverseSwizzle() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 20, 1);
AppendVMovU32(&code, 21, 2);
AppendVMovU32(&code, 22, 0);
AppendVMovLiteral(&code, 0, 0x3f800000u);
AppendVMovLiteral(&code, 1, 0x3f000000u);
AppendVMovLiteral(&code, 2, 0x3e800000u);
AppendVMovLiteral(&code, 3, 0x3f400000u);
code.push_back(EncodeMimg0(0x08, 0xf));
code.push_back(EncodeMimg1(0, 20));
AppendEnd(&code);
TestCase test;
test.name = "ImageStoreYzwxUsesInverseSwizzle";
test.code = code;
test.opcodes = {O::V_MOV_B32, O::IMAGE_STORE, O::S_ENDPGM};
test.user_data =
MakeStorageTextureData(Prospero::BufferFormat::k32_32_32_32Float);
test.has_user_data = true;
test.image_descriptor_swizzle = DstSel(5, 6, 7, 4);
test.storage_image_rgba = MakeRgbaImage(4, 4);
test.expected_storage_image_rgba = MakeRgbaImage(4, 4);
SetRgbaPixel(&test.expected_storage_image_rgba, 4, 1, 2, 0x3f400000u,
0x3f800000u, 0x3f000000u, 0x3e800000u);
return test;
}
TestCase ImageStoreR32FloatUsesFormatlessStorageImage() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 20, 2);
AppendVMovU32(&code, 21, 1);
AppendVMovU32(&code, 22, 0);
AppendVMovLiteral(&code, 0, 0x3f800000u);
code.push_back(EncodeMimg0(0x08, 0x1));
code.push_back(EncodeMimg1(0, 20));
AppendEnd(&code);
std::vector<u32> expected_image(16, 0);
expected_image[1 * 4 + 2] = 0x3f800000u;
TestCase test;
test.name = "ImageStoreR32FloatUsesFormatlessStorageImage";
test.code = code;
test.opcodes = {O::V_MOV_B32, O::IMAGE_STORE, O::S_ENDPGM};
test.storage_image_format = vk::Format::eR32Sfloat;
test.storage_image_dwords_per_pixel = 1;
test.storage_image_rgba = std::vector<u32>(16, 0);
test.expected_storage_image_rgba = expected_image;
test.required_spirv = {"OpCapability StorageImageWriteWithoutFormat"};
test.forbidden_spirv = {"OpCapability StorageImageReadWithoutFormat",
"Rgba32f", "OpTypeSampler", "OpTypeSampledImage"};
return test;
}
void CheckSigned16ImageStore(VulkanHarness &vulkan) {
constexpr std::array<int32_t, 8> values{INT32_MIN, -32769, -32768, -1, 0, 32767, 32768, INT32_MAX};
const u32 alignment = vulkan.RuntimeContext().GetPhysicalDeviceProperties()
.limits.minStorageBufferOffsetAlignment;
for (const u32 wave : {32u, 64u}) {
for (const bool d16 : {false, true}) {
for (const u32 component : {0u, 2u}) {
TestCase test;
test.name = "Signed16ImageAndBufferStore";
const u32 width = wave - 2u;
const u32 output = (wave * 4u + alignment - 1u) / alignment * alignment;
test.user_data = MakeStorageTextureData(Prospero::BufferFormat::k16SInt);
test.user_data[1] |= ((width - 1u) & 3u) << 30u;
test.user_data[2] = ((width - 1u) >> 2u) | (1u << 14u);
test.has_user_data = test.has_compute_info = true;
test.image_descriptor_swizzle = component == 0u ? DstSel(4, 0, 0, 1) : DstSel(0, 0, 4, 1);
const std::array<u32, 8> descriptors{0x1000u, 4u << 16u, wave, 0x5204u,
0x2000u, 2u << 16u, wave, (u32(Prospero::BufferFormat::k16SInt) << 12u) | 4u};
std::copy(descriptors.begin(), descriptors.end(), test.user_data.begin() + 8u);
test.storage_buffer_ranges = {{0u, wave * 4u}, {output, wave * 2u}};
test.compute_info.wave_size = wave;
test.compute_info.threads_num[0] = wave;
test.compute_info.threads_num[1] = test.compute_info.threads_num[2] = 1u;
test.compute_info.thread_ids_num = 1u;
test.compute_info.lds_size_dwords = 0u;
test.initial.assign(output / 4u + wave / 2u, 0xdeadbeefu);
for (u32 lane = 0; lane < wave; ++lane)
test.initial[lane] = d16 ? 0x7a5b0000u | (u32(values[lane % values.size()]) & 0xffffu)
: u32(values[lane % values.size()]);
test.expected = test.initial;
std::vector<u32> pixels(width, 0x13571357u), expected = pixels;
for (u32 lane = 0; lane < wave; ++lane) {
const auto input = values[lane % values.size()];
const auto value = d16 ? int32_t(std::bit_cast<int16_t>(uint16_t(input))) : input;
const u32 encoded = uint16_t(std::clamp(value, -32768, 32767));
const auto set_half = [&](std::vector<u32> &words, u32 index) {
const auto shift = (index & 1u) * 16u;
words[index / 2u] = (words[index / 2u] & ~(0xffffu << shift)) | (encoded << shift);
};
set_half(test.expected, output / 2u + lane);
if (lane < width) {
set_half(expected, lane);
if (lane & 1u) set_half(expected, width + lane);
}
}
auto &code = test.code;
code = {EncodeMubuf0(0x0c, 0, true, false), EncodeMubuf1(4, 2, 0),
EncodeSopp(0x0c, 0), EncodeMubuf0(d16 ? 0x84 : 0x04, 0, true, false),
EncodeMubuf1(4, 3, 0), EncodeVop1(0x01, 20, Vgpr(0))};
AppendVMovU32(&code, 21, 0u);
code.insert(code.end(), {EncodeMimg0(0x08, 1u << component),
EncodeMimg1(4, 20, 0, 0, false, d16)});
AppendVMovU32(&code, 21, 1u);
AppendSMovLiteral(&code, 126, 0xaaaaaaaau);
AppendSMovLiteral(&code, 127, wave == 64u ? 0xaaaaaaaau : 0u);
code.insert(code.end(), {EncodeMimg0(0x08, 1u << component),
EncodeMimg1(4, 20, 0, 0, false, d16)});
AppendEnd(&code);
test.opcodes = {ShaderOpcode::IMAGE_STORE};
const auto compiled = CompileCase(test, vulkan.SubgroupSize());
auto buffer = vulkan.CreateStorageBuffer(test.name, test.initial, test.initial.size());
auto image = vulkan.CreateImage2D(test.name, width, 2u, vk::Format::eR16Sint,
vk::ImageUsageFlagBits::eStorage, pixels, 1u, vk::ImageLayout::eGeneral, vk::Format::eR16Uint);
vulkan.Dispatch(test, compiled, buffer, nullptr, nullptr, nullptr, &image);
CompareWords(test, "signed image saturation, sparse channels, masked lanes and OOB", expected,
vulkan.ReadImage(test.name, &image));
CompareWords(test, "shared buffer conversion and adjacent halfwords", test.expected,
vulkan.ReadBuffer(test.name, buffer, test.expected.size()));
vulkan.DestroyImage(&image);
vulkan.DestroyBuffer(&buffer);
}
}
}
std::printf("[compute] %-32s ok\n", "Signed16ImageAndBufferStore");
}
TestCase ImageStoreR32SintUsesRawUintView() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 20, 2);
AppendVMovU32(&code, 21, 1);
AppendVMovU32(&code, 22, 0);
AppendVMovLiteral(&code, 0, 0x80000001u);
code.push_back(EncodeMimg0(0x08, 0x1));
code.push_back(EncodeMimg1(0, 20));
AppendEnd(&code);
std::vector<u32> expected_image(16, 0);
expected_image[1 * 4 + 2] = 0x80000001u;
TestCase test;
test.name = "ImageStoreR32SintUsesRawUintView";
test.code = code;
test.opcodes = {O::V_MOV_B32, O::IMAGE_STORE, O::S_ENDPGM};
test.user_data = MakeStorageTextureData(Prospero::BufferFormat::k32SInt);
test.has_user_data = true;
test.storage_image_r32ui = std::vector<u32>(16, 0);
test.expected_storage_image_r32ui = expected_image;
test.required_spirv = {"OpCapability StorageImageWriteWithoutFormat",
StorageUint2DImageBindingName(false)};
test.forbidden_spirv = {"R32ui"};
return test;
}
TestCase ImageStoreR32UintUsesFormatlessStorageImage() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 20, 2);
AppendVMovU32(&code, 21, 1);
AppendVMovU32(&code, 22, 0);
AppendVMovLiteral(&code, 0, 0x12345678u);
code.push_back(EncodeMimg0(0x08, 0x1));
code.push_back(EncodeMimg1(0, 20));
AppendEnd(&code);
std::vector<u32> expected_image(16, 0);
expected_image[1 * 4 + 2] = 0x12345678u;
TestCase test;
test.name = "ImageStoreR32UintUsesFormatlessStorageImage";
test.code = code;
test.opcodes = {O::V_MOV_B32, O::IMAGE_STORE, O::S_ENDPGM};
test.user_data = MakeStorageTextureData(Prospero::BufferFormat::k32UInt);
test.has_user_data = true;
test.storage_image_rgba = MakeRgbaImage(4, 4);
test.storage_image_r32ui = std::vector<u32>(16, 0);
test.expected_storage_image_r32ui = expected_image;
test.required_spirv = {"OpCapability StorageImageWriteWithoutFormat",
StorageUint2DImageBindingName(false)};
test.forbidden_spirv = {"R32ui"};
return test;
}
TestCase ImageStorePackedUintSaturatesChannels() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 20, 2);
AppendVMovU32(&code, 21, 1);
AppendVMovU32(&code, 22, 0);
AppendVMovU32(&code, 0, 0x800u);
AppendVMovLiteral(&code, 1, UINT32_MAX);
AppendVMovU32(&code, 2, 0x400u);
AppendVMovU32(&code, 3, 0x55u);
code.push_back(EncodeMimg0(0x08, 0xf));
code.push_back(EncodeMimg1(0, 20));
AppendEnd(&code);
std::vector<u32> expected_image(16, 0);
expected_image[1 * 4 + 2] = UINT32_MAX;
TestCase test;
test.name = "ImageStorePackedUintSaturatesChannels";
test.code = std::move(code);
test.opcodes = {O::V_MOV_B32, O::IMAGE_STORE, O::S_ENDPGM};
test.user_data =
MakeStorageTextureData(Prospero::BufferFormat::k11_11_10UInt);
test.has_user_data = true;
test.image_descriptor_swizzle = DstSel(4, 5, 6, 0);
test.storage_image_r32ui = std::vector<u32>(16, 0);
test.expected_storage_image_r32ui = std::move(expected_image);
test.required_spirv = {"OpCapability StorageImageWriteWithoutFormat",
StorageUint2DImageBindingName(false), "UMin"};
test.forbidden_spirv = {"R32ui"};
return test;
}
TestCase ImageStorePackedUintHonorsSparseDmask() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 20, 2);
AppendVMovU32(&code, 21, 1);
AppendVMovU32(&code, 22, 0);
AppendVMovU32(&code, 0, 0x321u);
AppendVMovU32(&code, 1, 0x2abu);
code.push_back(EncodeMimg0(0x08, 0x5));
code.push_back(EncodeMimg1(0, 20));
AppendEnd(&code);
std::vector<u32> expected_image(16, 0);
expected_image[1 * 4 + 2] = 0xaac00321u;
TestCase test;
test.name = "ImageStorePackedUintHonorsSparseDmask";
test.code = std::move(code);
test.opcodes = {O::V_MOV_B32, O::IMAGE_STORE, O::S_ENDPGM};
test.user_data =
MakeStorageTextureData(Prospero::BufferFormat::k11_11_10UInt);
test.has_user_data = true;
test.image_descriptor_swizzle = DstSel(4, 5, 6, 0);
test.storage_image_r32ui = std::vector<u32>(16, 0);
test.expected_storage_image_r32ui = std::move(expected_image);
test.required_spirv = {"OpCapability StorageImageWriteWithoutFormat",
StorageUint2DImageBindingName(false)};
test.forbidden_spirv = {"R32ui"};
return test;
}
template <bool sparse> TestCase ImageStorePackedUnorm() {
using O = ShaderOpcode;
// Float inputs and exact packed UNorm components, including ties to even.
constexpr float values[][3] = {
{-1.0f, 0.5f, 2.0f}, {0.0f, 1.0f, 0.5f},
{std::bit_cast<float>(0x7fc00000u), std::bit_cast<float>(0x7f800000u),
std::bit_cast<float>(0xff800000u)},
{0.25f, 0.75f, 0.25f}, {2.5f / 2047.0f, 3.5f / 2047.0f, 2.5f / 1023.0f}};
constexpr u32 converted[][3] = {
{0, 1024, 1023}, {0, 2047, 512}, {0, 2047, 0}, {512, 1535, 256}, {2, 4, 2}};
TestCase test;
test.name = sparse ? "ImageStorePackedUnormSparseDmask" : "ImageStorePackedUnormBoundaries";
test.user_data = MakeStorageTextureData(Prospero::BufferFormat::k11_11_10UNorm);
test.has_user_data = true;
test.image_descriptor_swizzle = DstSel(4, 5, 6, 0);
test.storage_image_r32ui.assign(16, 0xabcdef01u);
test.expected_storage_image_r32ui = test.storage_image_r32ui;
auto &code = test.code;
code.push_back(EncodeSop1(0x04, 12, 126));
for (u32 i = 0; i < std::size(values); ++i) {
AppendVMovU32(&code, 20, i & 3u);
AppendVMovU32(&code, 21, i >> 2u);
AppendVMovLiteral(&code, 0, std::bit_cast<u32>(values[i][0]));
if constexpr (sparse) {
AppendVMovLiteral(&code, 1, std::bit_cast<u32>(values[i][2]));
} else {
AppendVMovLiteral(&code, 1, std::bit_cast<u32>(values[i][1]));
AppendVMovLiteral(&code, 2, std::bit_cast<u32>(values[i][2]));
}
code.push_back(EncodeMimg0(0x08, sparse ? 0x5 : 0x7));
code.push_back(EncodeMimg1(0, 20));
test.expected_storage_image_r32ui[i] = converted[i][0] |
(sparse ? 0u : converted[i][1] << 11u) | (converted[i][2] << 22u);
}
AppendVMovU32(&code, 20, 3);
AppendVMovU32(&code, 21, 3);
code.push_back(EncodeSop1(0x04, 126, InlineU32(0)));
code.push_back(EncodeMimg0(0x08, 0x7));
code.push_back(EncodeMimg1(0, 20));
code.push_back(EncodeSop1(0x04, 126, 12));
AppendEnd(&code);
test.opcodes = {O::V_MOV_B32, O::S_MOV_B64, O::IMAGE_STORE, O::S_ENDPGM};
test.required_spirv = {"OpImageWrite", "RoundEven", "OpConvertFToU",
StorageUint2DImageBindingName(false)};
test.forbidden_spirv = {"OpImageRead", "OpAtomic", "OpTypeSampler", "R32ui"};
return test;
}
TestCase ComputeTgSizeSgprUsesWaveMetadata() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendStoreSgpr(&code, 2, 0);
AppendEnd(&code);
TestCase test;
test.name = "ComputeTgSizeSgprUsesWaveMetadata";
test.code = code;
test.opcodes = {O::V_MOV_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.compute_info.threads_num[0] = 4;
test.compute_info.threads_num[1] = 4;
test.compute_info.threads_num[2] = 16;
test.compute_info.group_id[0] = true;
test.compute_info.group_id[1] = true;
test.compute_info.wave_size = 32;
test.compute_info.thread_ids_num = 3;
test.compute_info.workgroup_register = 0;
test.compute_info.tg_size_en = true;
test.has_compute_info = true;
test.compile_only = true;
test.required_spirv = {"OpUDiv", "OpShiftLeftLogical", "2147483648"};
return test;
}
TestCase ImageStoreAndAtomicShareTypedBinding() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 20, 2);
AppendVMovU32(&code, 21, 1);
AppendVMovU32(&code, 22, 0);
AppendVMovU32(&code, 0, 5);
code.push_back(EncodeMimg0(0x08, 0x1));
code.push_back(EncodeMimg1(0, 20));
AppendVMovU32(&code, 0, 3);
code.push_back(EncodeMimg0(0x11, 0x1, 0, true));
code.push_back(EncodeMimg1(0, 20));
AppendStoreVgpr(&code, 0, 0);
AppendEnd(&code);
std::vector<u32> expected_image(16, 0);
expected_image[1 * 4 + 2] = 8;
TestCase test;
test.name = "ImageStoreAndAtomicShareTypedBinding";
test.code = std::move(code);
test.expected = {5};
test.opcodes = {O::V_MOV_B32, O::IMAGE_STORE, O::IMAGE_ATOMIC_ADD,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.user_data = MakeStorageTextureData(Prospero::BufferFormat::k32UInt);
test.has_user_data = true;
test.storage_image_r32ui = std::vector<u32>(16, 0);
test.expected_storage_image_r32ui = std::move(expected_image);
test.required_spirv = {"OpImageWrite", "OpImageTexelPointer", "R32ui",
StorageUint2DImageBindingName(true)};
test.forbidden_spirv = {"StorageImageReadWithoutFormat",
"StorageImageWriteWithoutFormat",
StorageUint2DImageBindingName(false)};
return test;
}
// RDNA 2 MIMG opcode 0x10: replacement at VDATA, comparison at VDATA+1.
TestCase ImageAtomicCompareSwapGlcAndExec() {
TestCase test;
test.name = "ImageAtomicCompareSwapGlcAndExec";
auto &code = test.code;
code.push_back(EncodeSop1(0x04, 12, 126)); // Preserve EXEC.
AppendVMovU32(&code, 20, 2);
AppendVMovU32(&code, 21, 1);
const auto exchange = [&](u32 replacement, u32 comparison, bool glc, u32 output) {
AppendVMovLiteral(&code, 0, replacement);
AppendVMovLiteral(&code, 1, comparison);
code.push_back(EncodeMimg0(0x10, 3, 0, glc));
code.push_back(EncodeMimg1(0, 20));
AppendStoreVgpr(&code, 0, output);
AppendStoreVgpr(&code, 1, output + 1);
};
exchange(0x12345678u, 0xdeadbeefu, true, 0); // Match; return original bits.
exchange(0x87654321u, 0xdeadbeefu, true, 2); // Mismatch; preserve texel.
exchange(0xabcdef01u, 0x12345678u, false, 4); // GLC=0 preserves VDATA.
AppendVMovLiteral(&code, 0, 0x11111111u);
AppendVMovLiteral(&code, 1, 0xabcdef01u);
code.push_back(EncodeSop1(0x04, 126, InlineU32(0))); // Disable EXEC.
code.push_back(EncodeMimg0(0x10, 3, 0, true));
code.push_back(EncodeMimg1(0, 20));
code.push_back(EncodeSop1(0x04, 126, 12));
AppendStoreVgpr(&code, 0, 6);
AppendStoreVgpr(&code, 1, 7);
AppendEnd(&code);
test.expected = {0xdeadbeefu, 0xdeadbeefu, 0x12345678u, 0xdeadbeefu,
0xabcdef01u, 0x12345678u, 0x11111111u, 0xabcdef01u};
test.opcodes = {ShaderOpcode::V_MOV_B32, ShaderOpcode::S_MOV_B64,
ShaderOpcode::IMAGE_ATOMIC_CMPSWAP, ShaderOpcode::BUFFER_STORE_DWORD,
ShaderOpcode::S_ENDPGM};
test.user_data = MakeStorageTextureData(Prospero::BufferFormat::k32UInt);
test.has_user_data = true;
test.storage_image_r32ui = std::vector<u32>(16, 0);
test.storage_image_r32ui[6] = 0xdeadbeefu;
test.expected_storage_image_r32ui = test.storage_image_r32ui;
test.expected_storage_image_r32ui[6] = 0xabcdef01u;
// Device scope, AcquireRelease | ImageMemory (0x808).
test.required_spirv = {"OpAtomicCompareExchange", "OpImageTexelPointer",
"OpMemoryBarrier %uint_1 %uint_2056"};
return test;
}
TestCase ImageAtomicSwapReturnsPreviousTexel() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 20, 2);
AppendVMovU32(&code, 21, 1);
AppendVMovU32(&code, 22, 0);
AppendVMovU32(&code, 0, 5);
code.push_back(EncodeMimg0(0x08, 0x1));
code.push_back(EncodeMimg1(0, 20));
AppendVMovU32(&code, 0, 3);
code.push_back(EncodeMimg0(0x0f, 0x1, 0, true));
code.push_back(EncodeMimg1(0, 20));
AppendStoreVgpr(&code, 0, 0);
AppendEnd(&code);
std::vector<u32> expected_image(16, 0);
expected_image[1 * 4 + 2] = 3;
TestCase test;
test.name = "ImageAtomicSwapReturnsPreviousTexel";
test.code = std::move(code);
test.expected = {5};
test.opcodes = {O::V_MOV_B32, O::IMAGE_STORE, O::IMAGE_ATOMIC_SWAP,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.user_data = MakeStorageTextureData(Prospero::BufferFormat::k32UInt);
test.has_user_data = true;
test.storage_image_r32ui = std::vector<u32>(16, 0);
test.expected_storage_image_r32ui = std::move(expected_image);
test.required_spirv = {"OpAtomicExchange", "OpImageTexelPointer"};
return test;
}
TestCase ImageStoreAndAtomicUseSeparateBindings() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 20, 2);
AppendVMovU32(&code, 21, 1);
AppendVMovU32(&code, 22, 0);
AppendVMovU32(&code, 0, 5);
code.push_back(EncodeMimg0(0x08, 0x1));
code.push_back(EncodeMimg1(0, 20, 0));
AppendVMovU32(&code, 0, 3);
code.push_back(EncodeMimg0(0x11, 0x1, 0, true));
code.push_back(EncodeMimg1(0, 20, 2));
AppendStoreVgpr(&code, 0, 0);
AppendEnd(&code);
auto user_data = MakeStorageTextureData(Prospero::BufferFormat::k32UInt);
std::copy_n(user_data.begin(), 8, user_data.begin() + 8);
std::vector<u32> expected_image(16, 0);
expected_image[1 * 4 + 2] = 8;
TestCase test;
test.name = "ImageStoreAndAtomicUseSeparateBindings";
test.code = std::move(code);
test.expected = {5};
test.opcodes = {O::V_MOV_B32, O::IMAGE_STORE, O::IMAGE_ATOMIC_ADD,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.user_data = user_data;
test.has_user_data = true;
test.storage_image_r32ui = std::vector<u32>(16, 0);
test.expected_storage_image_r32ui = std::move(expected_image);
test.required_spirv = {"OpCapability StorageImageWriteWithoutFormat",
"OpImageWrite",
"OpImageTexelPointer",
"R32ui",
StorageUint2DImageBindingName(false),
StorageUint2DImageBindingName(true)};
return test;
}
template <bool signed_format = false> TestCase ImageAtomicVariants() {
using O = ShaderOpcode;
const u32 initial[] = {10, 10, 0xf0f0u, 0xf000u, 0xf00fu};
const u32 values[] = {5, 5, 0x0ff0u, 0x0f00u, 0x00ffu};
const u32 ops[] = {0x11, 0x15, 0x18, 0x19, 0x1a};
std::vector<u32> code;
for (u32 i = 0; i < static_cast<u32>(std::size(values)); i++) {
AppendVMovU32(&code, 20, i & 3u);
AppendVMovU32(&code, 21, i >> 2u);
AppendVMovU32(&code, 22, 0);
AppendVMovLiteral(&code, 0, values[i]);
code.push_back(EncodeMimg0(ops[i], 0x1, 0, true));
code.push_back(EncodeMimg1(0, 20));
AppendStoreVgpr(&code, 0, i);
}
AppendEnd(&code);
TestCase test;
test.name = signed_format ? "ImageAtomicSintVariants" : "ImageAtomicVariants";
test.user_data = MakeStorageTextureData(signed_format ? Prospero::BufferFormat::k32SInt
: Prospero::BufferFormat::k32UInt);
test.has_user_data = true;
test.code = code;
test.expected = {10, 10, 0xf0f0u, 0xf000u, 0xf00fu};
test.opcodes = {O::V_MOV_B32, O::IMAGE_ATOMIC_ADD,
O::IMAGE_ATOMIC_UMIN, O::IMAGE_ATOMIC_AND,
O::IMAGE_ATOMIC_OR, O::IMAGE_ATOMIC_XOR,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.required_spirv = {"OpImageTexelPointer", "R32ui",
StorageUint2DImageBindingName(true)};
test.forbidden_spirv = {"OpTypeSampler", "OpTypeSampledImage",
"StorageImageReadWithoutFormat",
"StorageImageWriteWithoutFormat",
StorageUint2DImageBindingName(false)};
test.storage_image_rgba = MakeRgbaImage(4, 4);
test.storage_image_r32ui = std::vector<u32>(16, 0);
for (u32 i = 0; i < static_cast<u32>(std::size(initial)); i++) {
test.storage_image_r32ui[i] = initial[i];
}
test.expected_storage_image_r32ui = std::vector<u32>(16, 0);
test.expected_storage_image_r32ui[0] = 15;
test.expected_storage_image_r32ui[1] = 5;
test.expected_storage_image_r32ui[2] = 0x00f0u;
test.expected_storage_image_r32ui[3] = 0xff00u;
test.expected_storage_image_r32ui[4] = 0xf0f0u;
return test;
}
// Input and old texel cover high-DWORD ordering, carry/wrap, the unsigned sign
// boundary, and equal-high-DWORD ordering through the RG32_UINT -> R64_UINT view.
constexpr uint64_t ImageAtomicInputs[][2] = {
{0x0000000100000001ull, 0x00000000ffffffffull},
{0x00000000ffffffffull, 0x0000000100000001ull},
{0x8000000000000000ull, 0x7fffffffffffffffull},
{0x12345678ffffffffull, 0x1234567800000001ull},
{0xffffffffffffffffull, 0x8000000000000000ull},
};
struct ImageAtomicIntegerCase {
u32 encoding;
ShaderOpcode opcode;
const char *name32;
const char *name64;
const char *spirv;
uint64_t expected64[std::size(ImageAtomicInputs)];
u32 expected32[std::size(ImageAtomicInputs)];
};
constexpr ImageAtomicIntegerCase ImageAtomicIntegerCases[] = {
{0x0fu, ShaderOpcode::IMAGE_ATOMIC_SWAP,
"ImageAtomicSwap32GlcAndExec", "ImageAtomicSwap64GlcAndExec", "OpAtomicExchange",
{0x0000000100000001ull, 0x00000000ffffffffull, 0x8000000000000000ull,
0x12345678ffffffffull, 0xffffffffffffffffull},
{0x00000001u, 0xffffffffu, 0x00000000u, 0xffffffffu, 0xffffffffu}},
{0x11u, ShaderOpcode::IMAGE_ATOMIC_ADD,
"ImageAtomicAdd32GlcAndExec", "ImageAtomicAdd64GlcAndExec", "OpAtomicIAdd",
{0x0000000200000000ull, 0x0000000200000000ull, 0xffffffffffffffffull,
0x2468acf100000000ull, 0x7fffffffffffffffull},
{0x00000000u, 0x00000000u, 0xffffffffu, 0x00000000u, 0xffffffffu}},
{0x15u, ShaderOpcode::IMAGE_ATOMIC_UMIN,
"ImageAtomicUMin32GlcAndExec", "ImageAtomicUMin64GlcAndExec", "OpAtomicUMin",
{0x00000000ffffffffull, 0x00000000ffffffffull, 0x7fffffffffffffffull,
0x1234567800000001ull, 0x8000000000000000ull},
{0x00000001u, 0x00000001u, 0x00000000u, 0x00000001u, 0x00000000u}},
{0x17u, ShaderOpcode::IMAGE_ATOMIC_UMAX,
"ImageAtomicUMax32GlcAndExec", "ImageAtomicUMax64GlcAndExec", "OpAtomicUMax",
{0x0000000100000001ull, 0x0000000100000001ull, 0x8000000000000000ull,
0x12345678ffffffffull, 0xffffffffffffffffull},
{0xffffffffu, 0xffffffffu, 0xffffffffu, 0xffffffffu, 0xffffffffu}},
{0x18u, ShaderOpcode::IMAGE_ATOMIC_AND,
"ImageAtomicAnd32GlcAndExec", "ImageAtomicAnd64GlcAndExec", "OpAtomicAnd",
{0x0000000000000001ull, 0x0000000000000001ull, 0x0000000000000000ull,
0x1234567800000001ull, 0x8000000000000000ull},
{0x00000001u, 0x00000001u, 0x00000000u, 0x00000001u, 0x00000000u}},
{0x19u, ShaderOpcode::IMAGE_ATOMIC_OR,
"ImageAtomicOr32GlcAndExec", "ImageAtomicOr64GlcAndExec", "OpAtomicOr",
{0x00000001ffffffffull, 0x00000001ffffffffull, 0xffffffffffffffffull,
0x12345678ffffffffull, 0xffffffffffffffffull},
{0xffffffffu, 0xffffffffu, 0xffffffffu, 0xffffffffu, 0xffffffffu}},
{0x1au, ShaderOpcode::IMAGE_ATOMIC_XOR,
"ImageAtomicXor32GlcAndExec", "ImageAtomicXor64GlcAndExec", "OpAtomicXor",
{0x00000001fffffffeull, 0x00000001fffffffeull, 0xffffffffffffffffull,
0x00000000fffffffeull, 0x7fffffffffffffffull},
{0xfffffffeu, 0xfffffffeu, 0xffffffffu, 0xfffffffeu, 0xffffffffu}},
};
TestCase ImageAtomicIntegerGlcAndExec(const ImageAtomicIntegerCase &atomic, bool wide) {
const u32 texel_dwords = wide ? 2 : 1;
TestCase test;
test.name = wide ? atomic.name64 : atomic.name32;
test.user_data = MakeStorageTextureData(wide ? Prospero::BufferFormat::k32_32UInt
: Prospero::BufferFormat::k32UInt);
test.has_user_data = true;
test.storage_image_r32ui.assign(16 * texel_dwords, 0);
test.expected_storage_image_r32ui = test.storage_image_r32ui;
auto &code = test.code;
code.push_back(EncodeSop1(0x04, 12, 126)); // Preserve EXEC.
for (u32 i = 0; i < std::size(ImageAtomicInputs); ++i) {
const auto input = ImageAtomicInputs[i][0];
const auto old = ImageAtomicInputs[i][1];
const bool glc = i != 0 && i != 4;
AppendVMovU32(&code, 0, i & 3u);
AppendVMovU32(&code, 1, i >> 2u);
AppendVMovLiteral(&code, 2, static_cast<u32>(input));
AppendVMovLiteral(&code, 3, static_cast<u32>(input >> 32u));
code.push_back(EncodeMimg0(atomic.encoding, wide ? 3u : 1u, 0, glc));
code.push_back(EncodeMimg1(2, 0));
AppendStoreVgpr(&code, 2, i * 2);
AppendStoreVgpr(&code, 3, i * 2 + 1);
test.expected.push_back(static_cast<u32>(glc ? old : input));
test.expected.push_back(static_cast<u32>((wide && glc ? old : input) >> 32u));
const uint64_t result = wide ? atomic.expected64[i] : atomic.expected32[i];
for (u32 word = 0; word < texel_dwords; ++word) {
test.storage_image_r32ui[i * texel_dwords + word] =
static_cast<u32>(old >> (word * 32u));
test.expected_storage_image_r32ui[i * texel_dwords + word] =
static_cast<u32>(result >> (word * 32u));
}
}
// A leaked update must change the inactive texel for every operation.
const bool clear_bits = atomic.opcode == ShaderOpcode::IMAGE_ATOMIC_AND ||
atomic.opcode == ShaderOpcode::IMAGE_ATOMIC_UMIN;
const u32 inactive = clear_bits ? 0xffffffffu : 0u;
for (u32 word = 0; word < texel_dwords; ++word) {
test.storage_image_r32ui[5u * texel_dwords + word] = inactive;
test.expected_storage_image_r32ui[5u * texel_dwords + word] = inactive;
}
AppendVMovU32(&code, 0, 1);
AppendVMovU32(&code, 1, 1);
AppendVMovLiteral(&code, 2, 0xabcdef01u);
AppendVMovLiteral(&code, 3, 0x87654321u);
code.push_back(EncodeSop1(0x04, 126, InlineU32(0)));
code.push_back(EncodeMimg0(atomic.encoding, wide ? 3u : 1u, 0, true));
code.push_back(EncodeMimg1(2, 0));
code.push_back(EncodeSop1(0x04, 126, 12));
AppendStoreVgpr(&code, 2, std::size(ImageAtomicInputs) * 2);
AppendStoreVgpr(&code, 3, std::size(ImageAtomicInputs) * 2 + 1);
test.expected.insert(test.expected.end(), {0xabcdef01u, 0x87654321u});
AppendEnd(&code);
test.opcodes = {ShaderOpcode::V_MOV_B32, ShaderOpcode::S_MOV_B64,
atomic.opcode, ShaderOpcode::BUFFER_STORE_DWORD, ShaderOpcode::S_ENDPGM};
test.required_spirv = {atomic.spirv, "OpImageTexelPointer", wide ? "R64ui" : "R32ui"};
if (wide) {
test.required_spirv.push_back("Int64ImageEXT");
test.required_spirv.push_back("SPV_EXT_shader_image_int64");
}
return test;
}
template <bool max_value, bool signed_format = false> TestCase ImageAtomicSignedMinMax() {
using O = ShaderOpcode;
// Input, old value, signed minimum, signed maximum.
constexpr u32 values[][4] = {
{0xfffffffbu, 7, 0xfffffffbu, 7},
{7, 0xfffffffbu, 0xfffffffbu, 7},
{0x80000000u, 0x7fffffffu, 0x80000000u, 0x7fffffffu},
{0x7fffffffu, 0x80000000u, 0x80000000u, 0x7fffffffu},
{0xfffffff6u, 0xfffffffbu, 0xfffffff6u, 0xfffffffbu},
{0xfffffffbu, 0xfffffff6u, 0xfffffff6u, 0xfffffffbu},
{2, 8, 2, 8}, {8, 2, 2, 8},
};
TestCase test;
test.name = signed_format
? (max_value ? "ImageAtomicSMaxSintGlcAndExec" : "ImageAtomicSMinSintGlcAndExec")
: (max_value ? "ImageAtomicSMaxGlcAndExec" : "ImageAtomicSMinGlcAndExec");
test.user_data = MakeStorageTextureData(signed_format ? Prospero::BufferFormat::k32SInt
: Prospero::BufferFormat::k32UInt);
test.has_user_data = true;
test.storage_image_r32ui.assign(16, 0);
test.expected_storage_image_r32ui = test.storage_image_r32ui;
auto &code = test.code;
code.push_back(EncodeSop1(0x04, 12, 126));
for (u32 i = 0; i < std::size(values); ++i) {
AppendVMovU32(&code, 20, i & 3u);
AppendVMovU32(&code, 21, i >> 2u);
AppendVMovLiteral(&code, 2, values[i][0]);
const bool glc = (i & 1u) == 0;
code.push_back(EncodeMimg0(max_value ? 0x16 : 0x14, 1, 0, glc));
code.push_back(EncodeMimg1(2, 20));
AppendStoreVgpr(&code, 2, i);
test.expected.push_back(values[i][glc ? 1 : 0]);
test.storage_image_r32ui[i] = values[i][1];
test.expected_storage_image_r32ui[i] = values[i][max_value ? 3 : 2];
}
AppendVMovLiteral(&code, 2, max_value ? 0x7fffffffu : 0x80000000u);
code.push_back(EncodeSop1(0x04, 126, InlineU32(0)));
code.push_back(EncodeMimg0(max_value ? 0x16 : 0x14, 1, 0, true));
code.push_back(EncodeMimg1(2, 20));
code.push_back(EncodeSop1(0x04, 126, 12));
AppendStoreVgpr(&code, 2, std::size(values));
test.expected.push_back(max_value ? 0x7fffffffu : 0x80000000u);
AppendEnd(&code);
test.opcodes = {O::V_MOV_B32, O::S_MOV_B64,
max_value ? O::IMAGE_ATOMIC_SMAX : O::IMAGE_ATOMIC_SMIN,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.required_spirv = {max_value ? "OpAtomicSMax" : "OpAtomicSMin", "R32ui"};
return test;
}
template <bool max_value> TestCase ImageAtomicFloatGlcAndExec() {
using O = ShaderOpcode;
std::vector<u32> code;
constexpr u32 atomic_word = max_value ? 0xf07c0108u : 0xf0780108u;
code.push_back(EncodeSop1(0x04, 12, 126)); // Preserve active workgroup lanes.
AppendVMovU32(&code, 8, 2);
AppendVMovU32(&code, 9, 1);
AppendVMovLiteral(&code, 5, 0x40000000u); // 2.0
// FMAX uses the captured GLC=0 instruction word.
code.insert(code.end(), {atomic_word, 0x00010508u});
AppendStoreVgpr(&code, 5, 0);
AppendVMovLiteral(&code, 5, max_value ? 0x40800000u : 0x3f800000u);
code.insert(code.end(), {atomic_word | 0x2000u, 0x00010508u}); // GLC=1.
AppendStoreVgpr(&code, 5, 1);
AppendVMovLiteral(&code, 5,
max_value ? 0x41000000u : 0x3f000000u); // Inactive lane.
code.push_back(EncodeSop1(0x04, 126, InlineU32(0)));
code.insert(code.end(), {atomic_word | 0x2000u, 0x00010508u});
code.push_back(EncodeSop1(0x04, 126, 12));
AppendStoreVgpr(&code, 5, 2);
AppendEnd(&code);
TestCase test;
test.name = max_value ? "ImageAtomicFMaxCapturedGlcAndExec"
: "ImageAtomicFMinGlcAndExec";
test.code = std::move(code);
test.expected = {0x40000000u, 0x40000000u,
max_value ? 0x41000000u : 0x3f000000u};
test.opcodes = {O::V_MOV_B32, O::S_MOV_B64,
max_value ? O::IMAGE_ATOMIC_FMAX : O::IMAGE_ATOMIC_FMIN,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
const std::array<u32, 8> descriptor = {
0x504a7c00u, 0xc1600000u, 0x0086c0efu, 0x91b80924u,
0x00000000u, 0x00700080u, 0x00000000u, 0x00000000u};
std::copy_n(descriptor.begin(), 8, test.user_data.begin() + 4);
test.has_user_data = true;
test.storage_image_r32ui = std::vector<u32>(16, 0);
test.storage_image_r32ui[6] = max_value ? 0x3f800000u : 0x40800000u;
test.expected_storage_image_r32ui = test.storage_image_r32ui;
test.expected_storage_image_r32ui[6] = max_value ? 0x40800000u : 0x3f800000u;
test.required_spirv = {"OpImageTexelPointer", "OpAtomicCompareExchange", "R32ui"};
return test;
}
template <bool max_value> TestCase ImageAtomicFloatSpecialValues() {
using O = ShaderOpcode;
// source, previous texel, expected minimum, expected maximum.
const u32 cases[][4] = {
{0x40000000u, 0x3f800000u, 0x3f800000u, 0x40000000u},
{0xc0000000u, 0xbf800000u, 0xc0000000u, 0xbf800000u},
{0x7f800000u, 0x3f800000u, 0x3f800000u, 0x7f800000u},
{0xff800000u, 0x3f800000u, 0xff800000u, 0x3f800000u},
{0x3f800000u, 0x7fc12345u, 0x7fc12345u, 0x7fc12345u},
{0x7fcabcdeu, 0x3f800000u, 0x3f800000u, 0x3f800000u},
{0x3f800000u, 0x7fa54321u, 0x7fa54321u, 0x7fa54321u},
{0x7faabcdeu, 0x3f800000u, 0x3f800000u, 0x3f800000u},
{0x00000000u, 0x80000000u, 0x80000000u, 0x80000000u},
{0x80000000u, 0x00000000u, 0x00000000u, 0x00000000u},
{0x80000000u, 0x80000001u, 0x80000001u, 0x80000000u},
{0x80000001u, 0x80000000u, 0x80000001u, 0x80000000u},
{0x00000001u, 0x00000000u, 0x00000000u, 0x00000001u},
{0x00000000u, 0x00000001u, 0x00000000u, 0x00000001u},
};
TestCase test;
test.name = max_value ? "ImageAtomicFMaxSpecialValues"
: "ImageAtomicFMinSpecialValues";
test.user_data = MakeStorageTextureData(Prospero::BufferFormat::k32UInt);
test.has_user_data = true;
test.storage_image_r32ui = std::vector<u32>(16, 0);
test.expected_storage_image_r32ui = test.storage_image_r32ui;
for (u32 i = 0; i < std::size(cases); i++) {
AppendVMovU32(&test.code, 20, i & 3u);
AppendVMovU32(&test.code, 21, i >> 2u);
AppendVMovLiteral(&test.code, 0, cases[i][0]);
test.code.push_back(EncodeMimg0(max_value ? 0x1f : 0x1e, 1, 0, true));
test.code.push_back(EncodeMimg1(0, 20));
AppendStoreVgpr(&test.code, 0, i);
test.expected.push_back(cases[i][1]);
test.storage_image_r32ui[i] = cases[i][1];
test.expected_storage_image_r32ui[i] = cases[i][max_value ? 3 : 2];
}
AppendEnd(&test.code);
test.opcodes = {O::V_MOV_B32,
max_value ? O::IMAGE_ATOMIC_FMAX : O::IMAGE_ATOMIC_FMIN,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
return test;
}
template <bool max_value> TestCase ImageAtomicFloatContendedWorkgroup() {
using O = ShaderOpcode;
TestCase test;
test.name = max_value ? "ImageAtomicFMaxContendedWorkgroup"
: "ImageAtomicFMinContendedWorkgroup";
test.code.push_back(EncodeVop1(0x06, 1, Vgpr(0))); // Float thread_id.x.
AppendVMovU32(&test.code, 20, 0);
AppendVMovU32(&test.code, 21, 0);
test.code.push_back(EncodeMimg0(max_value ? 0x1f : 0x1e, 1));
test.code.push_back(EncodeMimg1(1, 20));
AppendEnd(&test.code);
test.opcodes = {O::V_CVT_F32_U32, O::V_MOV_B32,
max_value ? O::IMAGE_ATOMIC_FMAX : O::IMAGE_ATOMIC_FMIN,
O::S_ENDPGM};
test.user_data = MakeStorageTextureData(Prospero::BufferFormat::k32UInt);
test.has_user_data = true;
test.storage_image_r32ui = std::vector<u32>(16, 0);
test.storage_image_r32ui[0] =
max_value ? 0xc2c80000u : 0x42c80000u; // -/+100.0
test.expected_storage_image_r32ui = test.storage_image_r32ui;
test.expected_storage_image_r32ui[0] =
max_value ? 0x427c0000u : 0u; // 63.0 / 0.0
test.compute_info.threads_num[0] = 64;
test.compute_info.threads_num[1] = 1;
test.compute_info.threads_num[2] = 1;
test.compute_info.thread_ids_num = 1;
test.has_compute_info = true;
return test;
}
TestCase ImageAtomicGlc0DoesNotReturnOldValue() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovU32(&code, 20, 0);
AppendVMovU32(&code, 21, 0);
AppendVMovU32(&code, 22, 0);
AppendVMovU32(&code, 0, 5);
code.push_back(EncodeMimg0(0x11, 0x1));
code.push_back(EncodeMimg1(0, 20));
AppendStoreVgpr(&code, 0, 0);
AppendEnd(&code);
TestCase test;
test.name = "ImageAtomicGlc0DoesNotReturnOldValue";
test.code = code;
test.expected = {5};
test.opcodes = {O::V_MOV_B32, O::IMAGE_ATOMIC_ADD, O::BUFFER_STORE_DWORD,
O::S_ENDPGM};
test.storage_image_rgba = MakeRgbaImage(4, 4);
test.storage_image_r32ui = std::vector<u32>(16, 0);
test.storage_image_r32ui[0] = 10;
test.expected_storage_image_r32ui = std::vector<u32>(16, 0);
test.expected_storage_image_r32ui[0] = 15;
return test;
}
void AppendWave64Minimum(std::vector<u32> &code) {
code.insert(code.end(), {EncodeSop1(0x04, 24, 126), // Save contributing lanes.
EncodeSop1(0x28, 106, 126)});
AppendVop3(&code, 0x101, 13, 193, Vgpr(13), 24); // Inactive lanes contribute UINT_MAX.
for (u32 shift : {1u, 2u, 4u, 8u}) {
code.push_back(EncodeVop2(0x13, 13, 250, 13));
code.push_back(EncodeVop2Dpp(13, 0x110u + shift));
}
// Exact RDNA2 wave64 reduction from PS 964747887d898821, PCs 0x224..0x244.
code.insert(code.end(), {0xd778100cu, 0x0305830du,
EncodeVop2(0x13, 13, Vgpr(13), 12), EncodeSop1(0x04, 126, 106),
0xd7600006u, 0x00013f0du, 0xd7600007u, 0x00017f0du,
EncodeSop2(0x07, 8, 6, 7), EncodeVop1(0x01, 0, 8)});
}
GraphicsCase GraphicsSparseWaveMinimum() {
GraphicsCase test;
test.name = "GraphicsSparseWaveMinimum";
auto &code = test.fragment_code;
code = {EncodeVintrp(0x02, 13, 0, 0, 2)};
AppendWave64Minimum(code);
code.insert(code.end(), {EncodeExp0(0x00, 0xf), EncodeExp1(0, 0, 0, 0)});
AppendEnd(&code);
test.pixel_interpolator_settings = {0x00000400u};
test.expected_pixel = {0x3e800000u, 0x3e800000u, 0x3e800000u, 0x3e800000u};
test.opcodes = {ShaderOpcode::V_MIN_U32, ShaderOpcode::V_PERMLANEX16_B32,
ShaderOpcode::V_READLANE_B32, ShaderOpcode::S_MIN_U32};
return test;
}
TestCase ComputeWave64Minimum(bool masked) {
TestCase test;
test.name = masked ? "ComputeWave64MinimumMasked" : "ComputeWave64Minimum";
auto &code = test.code;
code = {EncodeVop2(0x1a, 20, InlineU32(2), 0)};
AppendVMovU32(&code, 13, 100);
code.push_back(EncodeVop2(0x26, 13, Vgpr(13), 0)); // 100 - lane: minimum is in the upper half.
if (masked) {
AppendSMovLiteral(&code, 126, 0xfffffffeu);
AppendSMovLiteral(&code, 127, 0x7fffffffu);
}
AppendWave64Minimum(code);
AppendBufferStoreDword(&code, 0, 20);
AppendEnd(&code);
test.initial.assign(64, 0xa5a5a5a5u);
test.expected.assign(64, masked ? 38u : 37u);
if (masked) test.expected.front() = test.expected.back() = 0xa5a5a5a5u;
test.compute_info.threads_num[0] = 64;
test.compute_info.threads_num[1] = test.compute_info.threads_num[2] = 1;
test.compute_info.thread_ids_num = 1;
test.compute_info.wave_size = 64;
test.has_compute_info = true;
test.opcodes = {ShaderOpcode::V_MIN_U32, ShaderOpcode::V_PERMLANEX16_B32,
ShaderOpcode::V_READLANE_B32, ShaderOpcode::S_MIN_U32};
return test;
}
GraphicsCase GraphicsInterpolationExport() {
using O = ShaderOpcode;
std::vector<u32> code;
code.push_back(EncodeVintrp(0x00, 0, 0, 0, 0));
code.push_back(EncodeVintrp(0x01, 0, 0, 0, 0));
code.push_back(EncodeVintrp(0x02, 1, 0, 1, 2));
AppendVMovLiteral(&code, 2, 0x3f400000u);
AppendVMovLiteral(&code, 3, 0x3f800000u);
code.push_back(EncodeExp0(0x00, 0xf));
code.push_back(EncodeExp1(0, 1, 2, 3));
AppendEnd(&code);
return {"GraphicsInterpolationExport",
code,
{0x3e800000u, 0x3f000000u, 0x3f400000u, 0x3f800000u},
{O::V_INTERP_P1_F32, O::V_INTERP_P2_F32, O::V_INTERP_MOV_F32,
O::V_MOV_B32, O::EXP, O::S_ENDPGM}};
}
GraphicsCase GraphicsPositionWExport() {
GraphicsCase test;
test.name = "GraphicsPositionWExport";
test.pixel_position_w = true;
test.vertex_clip_w = 4.0f;
// POS_W occupies v2; the rasterizer supplies FragCoord.w=1/4 for this triangle.
test.fragment_code = {EncodeExp0(0x00, 0xf), EncodeExp1(2, 2, 2, 2)};
AppendEnd(&test.fragment_code);
test.expected_pixel = {0x40800000u, 0x40800000u, 0x40800000u, 0x40800000u};
test.opcodes = {ShaderOpcode::EXP, ShaderOpcode::S_ENDPGM};
return test;
}
GraphicsCase GraphicsPackedHalfCentroid() {
GraphicsCase test;
test.name = "GraphicsPackedHalfCentroid";
test.pixel_perspective_centroid_vgpr = 0;
test.pixel_custom_interpolation_mask = 1;
test.pixel_interpolator_settings = {0x420u};
// Captured ab810715011baef3 interpolation, with attr3.x remapped to attr0.x.
test.fragment_code = {
0xc8120002u, 0xc80e0000u, // raw vertex0/vertex1 packed halves
0xcc204007u, 0x9c1206f2u, // low(vertex1) - low(vertex0)
0xcc207005u, 0x9c1206f2u, // high(vertex1) - high(vertex0)
0xc80a0001u, // raw vertex2
0xcc204006u, 0x9c1204f2u,
0xcc207003u, 0x9c1204f2u,
0xcc20400eu, 0x04120f00u, // low = I * delta10 + vertex0
0xcc20600fu, 0x04120b00u, // high = I * delta10 + vertex0
0x3e1c0306u, 0x3e1e0303u, // += J * delta20
EncodeExp0(0x00, 0xf), EncodeExp1(14, 15, 14, 15)};
AppendEnd(&test.fragment_code);
// At the pixel center the vertex weights are (5/8, 1/8, 1/4).
// Packed values are (0,1), (1,2), (3,3): expected result (0.875, 1.625).
test.vertices = {
0xbf800000u, 0xbf800000u, 0x3c000000u, 0, 0, 0,
0x40e00000u, 0xbf800000u, 0x40003c00u, 0, 0, 0,
0xbf800000u, 0x40400000u, 0x42004200u, 0, 0, 0};
test.expected_pixel = {0x3f600000u, 0x3fd00000u, 0x3f600000u, 0x3fd00000u};
test.opcodes = {ShaderOpcode::V_INTERP_MOV_F32, ShaderOpcode::V_FMA_F32, ShaderOpcode::V_MAC_F32,
ShaderOpcode::EXP, ShaderOpcode::S_ENDPGM};
return test;
}
GraphicsCase GraphicsPerspectiveSample(bool multisample) {
GraphicsCase test;
test.name = multisample ? "GraphicsPerspectiveSample4x"
: "GraphicsPerspectiveSample1x";
test.pixel_perspective_sample_vgpr = 0;
test.samples = multisample ? vk::SampleCountFlagBits::e4
: vk::SampleCountFlagBits::e1;
test.fragment_code = {
EncodeVop2(0x08, 2, Vgpr(0), 0), // I squared
EncodeVop2(0x08, 3, Vgpr(1), 1), // J squared
EncodeExp0(0x00, 0xf), EncodeExp1(2, 3, 0, 1)};
AppendEnd(&test.fragment_code);
// The triangle gives I=x/2, J=y/2. Squaring before the 4x resolve
// distinguishes sample interpolation (21/256) from pixel-center (1/16).
const auto square = std::bit_cast<u32>(multisample ? 21.0f / 256.0f : 1.0f / 16.0f);
test.expected_pixel = {square, square, 0x3e800000u, 0x3e800000u};
test.opcodes = {ShaderOpcode::V_MUL_F32, ShaderOpcode::EXP, ShaderOpcode::S_ENDPGM};
return test;
}
GraphicsCase GraphicsPackedHalfInputAlias(bool second_weights) {
auto test = GraphicsPackedHalfCentroid();
test.name = second_weights ? "GraphicsPackedHalfAliasSecondWeights"
: "GraphicsPackedHalfInputAlias";
test.pixel_custom_interpolation_mask = 2u;
test.pixel_interpolator_settings = {0u, 0x420u};
// The head shader reads one exported vec4 through two logical inputs:
// ordinary interpolation and raw per-vertex packed-half interpolation.
// Move the captured raw attr0.x loads to logical attr1.x, still mapped to 0.
for (const auto word : {0u, 1u, 6u}) {
test.fragment_code[word] |= 1u << 10u;
}
test.fragment_code.resize(test.fragment_code.size() - 3u);
test.fragment_code.push_back(EncodeVintrp(0x00, 16, 0, 1, 0));
test.fragment_code.push_back(EncodeVintrp(0x01, 16, 0, 1, 1));
AppendVMovLiteral(&test.fragment_code, 17, 0x3f800000u);
test.fragment_code.push_back(EncodeExp0(0x00, 0xf));
test.fragment_code.push_back(EncodeExp1(14, 15, 16, 17));
AppendEnd(&test.fragment_code);
// The smooth component (2,4,8) must vary independently of packed UVs.
test.vertices[3] = 0x40000000u;
test.vertices[9] = 0x40800000u;
test.vertices[15] = 0x41000000u;
if (second_weights) {
// Change the right vertex from x=7 to x=3: weights become (1/2,1/4,1/4).
test.vertices[6] = 0x40400000u;
test.expected_pixel = {0x3f800000u, 0x3fe00000u, 0x40800000u, 0x3f800000u};
} else {
test.expected_pixel = {0x3f600000u, 0x3fd00000u, 0x40700000u, 0x3f800000u};
}
test.opcodes.insert(test.opcodes.end(),
{ShaderOpcode::V_INTERP_P1_F32, ShaderOpcode::V_INTERP_P2_F32,
ShaderOpcode::V_MOV_B32});
return test;
}
GraphicsCase GraphicsSmoothRawInputAlias() {
auto test = GraphicsPackedHalfInputAlias(false);
test.name = "GraphicsSmoothRawInputAlias";
test.pixel_perspective_centroid_vgpr = UINT32_MAX;
// No explicit barycentric builtin: alias promotion must discover that the
// ordinary input needs weights while the raw alias returns vertex zero.
test.fragment_code = {EncodeVintrp(0x00, 0, 0, 1, 0),
EncodeVintrp(0x01, 0, 0, 1, 1),
EncodeVintrp(0x02, 1, 1, 1, 2),
EncodeExp0(0x00, 0xf), EncodeExp1(0, 1, 0, 1)};
AppendEnd(&test.fragment_code);
test.expected_pixel = {0x40700000u, 0x40000000u, 0x40700000u, 0x40000000u};
test.opcodes = {ShaderOpcode::V_INTERP_P1_F32, ShaderOpcode::V_INTERP_P2_F32,
ShaderOpcode::V_INTERP_MOV_F32, ShaderOpcode::EXP,
ShaderOpcode::S_ENDPGM};
return test;
}
GraphicsCase GraphicsSmoothFlatInputAlias(bool flat_first, bool provoking_last = false) {
auto test = GraphicsSmoothRawInputAlias();
test.name = provoking_last
? (flat_first ? "GraphicsFlatSmoothAliasLast" : "GraphicsSmoothFlatAliasLast")
: (flat_first ? "GraphicsFlatSmoothInputAlias" : "GraphicsSmoothFlatInputAlias");
test.pixel_custom_interpolation_mask = 0;
test.provoking_vtx_last = provoking_last;
test.pixel_interpolator_settings = flat_first ? std::vector<u32>{0x400u, 0u}
: std::vector<u32>{0u, 0x400u};
const u32 smooth = flat_first ? 1u : 0u;
const u32 flat = smooth ^ 1u;
// One export carries smooth values (2,4,8) and packed light IDs (5,7,11).
// Flat P0 preserves the ID bits; flat P1/P2 uses the same provoking vertex.
test.vertices[4] = 0x3f800005u;
test.vertices[10] = 0x3f800007u;
test.vertices[16] = 0x3f80000bu;
test.fragment_code = {EncodeVintrp(0x00, 0, smooth, 1, 0),
EncodeVintrp(0x01, 0, smooth, 1, 1),
EncodeVintrp(0x02, 1, flat, 2, 2),
EncodeVop2(0x1b, 1, InlineU32(63), 1),
EncodeVop1(0x06, 1, Vgpr(1)),
EncodeVintrp(0x00, 2, flat, 1, 0),
EncodeVintrp(0x01, 2, flat, 1, 1),
EncodeExp0(0x00, 0xf), EncodeExp1(0, 1, 2, 0)};
AppendEnd(&test.fragment_code);
test.expected_pixel = {0x40700000u, provoking_last ? 0x41300000u : 0x40a00000u,
provoking_last ? 0x41000000u : 0x40000000u, 0x40700000u};
test.opcodes.insert(test.opcodes.end(),
{ShaderOpcode::V_AND_B32, ShaderOpcode::V_CVT_F32_U32});
return test;
}
GraphicsCase GraphicsAncillaryLayer(bool front_face) {
GraphicsCase test;
test.name = front_face ? "GraphicsAncillaryAfterFrontFace" : "GraphicsAncillaryLayer";
test.pixel_ancillary = true;
test.pixel_front_face = front_face;
test.layers = 8;
const auto ancillary = Vgpr(front_face ? 3 : 2);
test.fragment_code.push_back(EncodeSop1(0x04, 8, 126));
test.fragment_code.push_back(EncodeSop1(0x0a, 126, 126));
AppendVop3(&test.fragment_code, 0x148, 4, ancillary, InlineU32(16), InlineU32(11));
AppendVop3(&test.fragment_code, 0x149, 5, ancillary, InlineU32(16), InlineU32(3));
test.fragment_code.push_back(EncodeSop1(0x04, 126, 8));
AppendVMovU32(&test.fragment_code, 6, test.layers);
test.fragment_code.push_back(EncodeVopc(0xc1, Vgpr(4), 6)); // Dynamic layer < count.
test.fragment_code.push_back(EncodeSop1(0x04, 126, 106));
test.fragment_code.push_back(EncodeVop1(0x06, 0, Vgpr(4))); // float(unsigned layer)
test.fragment_code.push_back(EncodeVop1(0x05, 1, Vgpr(5))); // float(signed low 3 bits)
AppendVMovLiteral(&test.fragment_code, 2, 0);
AppendVMovLiteral(&test.fragment_code, 3, 0x3f800000u);
test.fragment_code.push_back(EncodeExp0(0x00, 0xf));
test.fragment_code.push_back(EncodeExp1(0, 1, 2, 3));
AppendEnd(&test.fragment_code);
test.opcodes = {ShaderOpcode::V_BFE_U32, ShaderOpcode::V_BFE_I32,
ShaderOpcode::V_CVT_F32_U32, ShaderOpcode::V_CVT_F32_I32,
ShaderOpcode::V_MOV_B32, ShaderOpcode::S_MOV_B64,
ShaderOpcode::S_WQM_B64, ShaderOpcode::V_CMP_LT_U32,
ShaderOpcode::EXP, ShaderOpcode::S_ENDPGM};
for (u32 layer = 0; layer < test.layers; ++layer) {
const auto signed_bits = layer < 4 ? static_cast<int>(layer) : static_cast<int>(layer) - 8;
test.expected_pixel.insert(test.expected_pixel.end(),
{std::bit_cast<u32>(static_cast<float>(layer)),
std::bit_cast<u32>(static_cast<float>(signed_bits)), 0, 0x3f800000u});
}
return test;
}
GraphicsCase GraphicsAncillarySampleId() {
GraphicsCase test;
test.name = "GraphicsAncillarySampleId";
test.pixel_ancillary = true;
test.samples = vk::SampleCountFlagBits::e4;
AppendVop3(&test.fragment_code, 0x148, 4, Vgpr(2), InlineU32(8), InlineU32(4));
AppendVop3(&test.fragment_code, 0x149, 5, Vgpr(2), InlineU32(8), InlineU32(2));
AppendVop3(&test.fragment_code, 0x148, 6, Vgpr(2), InlineU32(9), InlineU32(1));
test.fragment_code.push_back(EncodeVop1(0x06, 0, Vgpr(4)));
test.fragment_code.push_back(EncodeVop1(0x05, 1, Vgpr(5)));
test.fragment_code.push_back(EncodeVop1(0x06, 2, Vgpr(6)));
AppendVMovLiteral(&test.fragment_code, 3, 0x3f800000u);
test.fragment_code.push_back(EncodeExp0(0x00, 0xf));
test.fragment_code.push_back(EncodeExp1(0, 1, 2, 3));
AppendEnd(&test.fragment_code);
test.opcodes = {ShaderOpcode::V_BFE_U32, ShaderOpcode::V_BFE_I32,
ShaderOpcode::V_CVT_F32_U32, ShaderOpcode::V_CVT_F32_I32,
ShaderOpcode::V_MOV_B32, ShaderOpcode::EXP, ShaderOpcode::S_ENDPGM};
// Average samples 0..3, their signed low two bits, and their second bit.
test.expected_pixel = {0x3fc00000u, 0xbf000000u, 0x3f000000u, 0x3f800000u};
return test;
}
// SPI_BARYC_CNTL[24] selects all-bits mode for uint/bool S_FRONT_FACE;
// float S_FRONT_FACE and __getSpecialVgprFloat(__v_front_face) leave it clear.
GraphicsCase GraphicsFrontFaceBits(bool reversed, bool all_bits) {
GraphicsCase test;
test.name = all_bits ? (reversed ? "GraphicsFrontFaceUintFront" : "GraphicsFrontFaceUintBack")
: (reversed ? "GraphicsFrontFaceFloatFront" : "GraphicsFrontFaceFloatBack");
test.pixel_front_face = true;
test.pixel_front_face_all_bits = all_bits;
// Preserve every raw bit as two finite, exactly representable integer-to-float
// results. Directly exporting the integer front value would export a NaN.
AppendVop3(&test.fragment_code, 0x148, 4, Vgpr(2), InlineU32(0), InlineU32(16));
AppendVop3(&test.fragment_code, 0x148, 5, Vgpr(2), InlineU32(16), InlineU32(16));
test.fragment_code.push_back(EncodeVop1(0x06, 0, Vgpr(4)));
test.fragment_code.push_back(EncodeVop1(0x06, 1, Vgpr(5)));
// Integer inputs test raw zero; float inputs test the sign.
test.fragment_code.push_back(EncodeVopc(all_bits ? 0xc2 : 0x04, InlineU32(0), 2));
AppendVMovLiteral(&test.fragment_code, 3, 0x3f800000u);
test.fragment_code.push_back(EncodeVop2(0x01, 2, InlineU32(0), 3));
test.fragment_code.push_back(EncodeExp0(0x00, 0xf));
test.fragment_code.push_back(EncodeExp1(0, 1, 2, 3));
AppendEnd(&test.fragment_code);
test.opcodes = {ShaderOpcode::V_BFE_U32, ShaderOpcode::V_CVT_F32_U32,
all_bits ? ShaderOpcode::V_CMP_EQ_U32 : ShaderOpcode::V_CMP_GT_F32,
ShaderOpcode::V_CNDMASK_B32, ShaderOpcode::V_MOV_B32,
ShaderOpcode::EXP, ShaderOpcode::S_ENDPGM};
test.vertices = {
0xbf800000u, 0xbf800000u, 0x3e800000u, 0x3f000000u, 0x3f400000u, 0x3f800000u,
0x40400000u, 0xbf800000u, 0x3e800000u, 0x3f000000u, 0x3f400000u, 0x3f800000u,
0xbf800000u, 0x40400000u, 0x3e800000u, 0x3f000000u, 0x3f400000u, 0x3f800000u};
if (reversed) {
std::swap_ranges(test.vertices.begin() + 6, test.vertices.begin() + 12,
test.vertices.begin() + 12);
}
// With the harness's positive-height viewport, the default triangle faces back.
const u32 bits = all_bits ? (reversed ? UINT32_MAX : 0u)
: (reversed ? 0x3f800000u : 0xbf800000u);
test.expected_pixel = {
std::bit_cast<u32>(static_cast<float>(bits & 0xffffu)),
std::bit_cast<u32>(static_cast<float>(bits >> 16u)),
reversed ? 0u : 0x3f800000u, 0x3f800000u};
return test;
}
void CheckPixelFrontFaceModes() {
constexpr const char *name = "PixelFrontFaceModes";
static const std::array<u32, 1> code = {EncodeSopp(0x01)};
HW::PixelShaderInfo regs{};
regs.ps_regs.data_addr = reinterpret_cast<uint64_t>(code.data());
ShaderMapUserData(regs.ps_regs.data_addr,
{.type = Prospero::ShaderBinaryType::kPs,
.code_size_bytes = static_cast<u32>(sizeof(code))});
HW::ShaderRegisters sh{};
sh.ps_input_ena = sh.ps_input_addr = 0x1002u;
const std::array<Prospero::ColorComponentMapping, 8> export_mapping{};
std::vector<u32> float_key;
for (bool all_bits : {false, true}) {
for (u32 other_bits : {0u, 0x120u}) {
sh.baryc_cntl = (all_bits ? 0x01000000u : 0u) | other_bits;
ShaderPixelInputInfo pixel{};
(void)PrepareProgram(regs, sh, export_mapping, pixel);
Require(name, "register format decoding",
pixel.ps_front_face && pixel.ps_front_face_all_bits == all_bits &&
pixel.ps_system_input_base == 2u,
"SPI_BARYC_CNTL[24] did not select the front-face format");
std::vector<u32> key;
BuildStageStaticKey(pixel, key);
if (!all_bits && other_bits == 0u) {
float_key = key;
}
Require(name, "format specialization",
(key != float_key) == all_bits,
"front-face formats must have distinct shader keys and ignore unrelated bits");
}
}
std::printf("[host] %-32s ok\n", name);
}
GraphicsCase GraphicsFlatInterpolatorExport() {
using O = ShaderOpcode;
std::vector<u32> code;
code.push_back(EncodeVintrp(0x02, 0, 0, 0, 2));
AppendVMovLiteral(&code, 1, 0x00000000u);
AppendVMovLiteral(&code, 2, 0x00000000u);
AppendVMovLiteral(&code, 3, 0x3f800000u);
code.push_back(EncodeExp0(0x00, 0xf));
code.push_back(EncodeExp1(0, 1, 2, 3));
AppendEnd(&code);
GraphicsCase test;
test.name = "GraphicsFlatInterpolatorExport";
test.fragment_code = code;
test.expected_pixel = {0x3e800000u, 0x00000000u, 0x00000000u, 0x3f800000u};
test.opcodes = {O::V_INTERP_MOV_F32, O::V_MOV_B32, O::EXP, O::S_ENDPGM};
test.pixel_interpolator_settings = {0x00000400u};
test.vertices = {
0xbf800000u, 0xbf800000u, 0x3e800000u, 0x00000000u, 0x00000000u,
0x3f800000u, 0x40400000u, 0xbf800000u, 0x3f400000u, 0x00000000u,
0x00000000u, 0x3f800000u, 0xbf800000u, 0x40400000u, 0x3e800000u,
0x00000000u, 0x00000000u, 0x3f800000u,
};
return test;
}
GraphicsCase GraphicsDsAddtidScratchExport() {
using O = ShaderOpcode;
std::vector<u32> code;
code.push_back(EncodeSMovB32(124, InlineU32(0)));
AppendVMovLiteral(&code, 0, 0x3f000000u);
code.push_back(EncodeDs0(0xb0, 0));
code.push_back(EncodeDs1(0, 0, 0));
code.push_back(EncodeDs0(0xb1, 0));
code.push_back(EncodeDs1(4, 0, 0));
code.push_back(EncodeExp0(0x00, 0xf));
code.push_back(EncodeExp1(4, 4, 4, 4));
AppendEnd(&code);
return {"GraphicsDsAddtidScratchExport",
code,
{0x3f000000u, 0x3f000000u, 0x3f000000u, 0x3f000000u},
{O::S_MOV_B32, O::V_MOV_B32, O::DS_WRITE_ADDTID_B32,
O::DS_READ_ADDTID_B32, O::EXP, O::S_ENDPGM}};
}
GraphicsCase GraphicsDirectSgprPushConstantExport() {
using O = ShaderOpcode;
const std::vector<u32> values = {0x3e800000u, 0x3f000000u, 0x3f400000u,
0x3f800000u};
std::vector<u32> code;
code.push_back(EncodeVop1(0x01, 0, 0));
code.push_back(EncodeVop1(0x01, 1, 1));
code.push_back(EncodeVop1(0x01, 2, 2));
code.push_back(EncodeVop1(0x01, 3, 3));
code.push_back(EncodeExp0(0x00, 0xf));
code.push_back(EncodeExp1(0, 1, 2, 3));
AppendEnd(&code);
GraphicsCase test;
test.name = "GraphicsDirectSgprPushConstantExport";
test.fragment_code = code;
test.expected_pixel = values;
test.opcodes = {O::V_MOV_B32, O::EXP, O::S_ENDPGM};
for (size_t i = 0; i < values.size(); i++) {
test.user_data[i] = values[i];
}
test.has_user_data = true;
test.push_constants = values;
return test;
}
GraphicsCase GraphicsInlineSrtScalarPromotionExport() {
using O = ShaderOpcode;
const std::vector<u32> values = {0x00000000u, 0x00000000u, 0x00000000u,
0x3f800000u};
std::vector<u32> code;
AppendVop3(&code, 0x12fu, 0, 0, 1);
AppendVop3(&code, 0x12fu, 1, 2, 3);
code.push_back(EncodeExp0(0x00, 0xf, true, true, true));
code.push_back(EncodeExp1(0, 1, 0, 0));
AppendEnd(&code);
GraphicsCase test;
test.name = "GraphicsInlineSrtScalarPromotionExport";
test.fragment_code = code;
test.expected_pixel = values;
test.opcodes = {O::V_CVT_PKRTZ_F16_F32, O::EXP, O::S_ENDPGM};
for (size_t i = 0; i < values.size(); i++) {
test.user_data[i] = values[i];
}
test.has_user_data = true;
test.push_constants = values;
return test;
}
GraphicsCase GraphicsNullVmExportDiscardsInactiveExec() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovLiteral(&code, 0, 0x3f000000u);
AppendVMovLiteral(&code, 1, 0x3f400000u);
AppendVMovLiteral(&code, 2, 0x3f800000u);
AppendVMovLiteral(&code, 3, 0x40000000u);
code.push_back(EncodeExp0(0x00, 0xf, false));
code.push_back(EncodeExp1(0, 1, 2, 3));
code.push_back(EncodeSop1(0x04, 126, InlineU32(0)));
code.push_back(EncodeExp0(0x09, 0x0, true, false, true));
code.push_back(EncodeExp1(0, 0, 0, 0));
AppendEnd(&code);
return {"GraphicsNullVmExportDiscardsInactiveExec",
code,
{0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u},
{O::V_MOV_B32, O::EXP, O::S_MOV_B64, O::S_ENDPGM}};
}
GraphicsCase GraphicsMrt0OffVmExportDiscardsInactiveExec() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovLiteral(&code, 0, 0x3f000000u);
AppendVMovLiteral(&code, 1, 0x3f400000u);
AppendVMovLiteral(&code, 2, 0x3f800000u);
AppendVMovLiteral(&code, 3, 0x40000000u);
code.push_back(EncodeExp0(0x00, 0xf, false));
code.push_back(EncodeExp1(0, 1, 2, 3));
code.push_back(EncodeSop1(0x04, 126, InlineU32(0)));
code.push_back(EncodeExp0(0x00, 0x0, true, true, true));
code.push_back(EncodeExp1(0, 0, 0, 0));
AppendEnd(&code);
return {"GraphicsMrt0OffVmExportDiscardsInactiveExec",
code,
{0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u},
{O::V_MOV_B32, O::EXP, O::S_MOV_B64, O::S_ENDPGM}};
}
GraphicsCase GraphicsFinalVmExportSupersedesEarlierVmMask() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovLiteral(&code, 0, 0x3f000000u);
AppendVMovLiteral(&code, 1, 0x3f400000u);
AppendVMovLiteral(&code, 2, 0x3f800000u);
AppendVMovLiteral(&code, 3, 0x40000000u);
code.push_back(EncodeSop1(0x04, 126, InlineU32(0)));
code.push_back(EncodeExp0(0x00, 0xf, false, false, true));
code.push_back(EncodeExp1(0, 1, 2, 3));
code.push_back(EncodeSMovB32(126, 193u));
code.push_back(EncodeSMovB32(127, 193u));
AppendVMovLiteral(&code, 4, 0x40400000u);
AppendVMovLiteral(&code, 5, 0x40800000u);
AppendVMovLiteral(&code, 6, 0x40a00000u);
AppendVMovLiteral(&code, 7, 0x40c00000u);
code.push_back(EncodeExp0(0x00, 0xf, true, false, true));
code.push_back(EncodeExp1(4, 5, 6, 7));
AppendEnd(&code);
return {"GraphicsFinalVmExportSupersedesEarlierVmMask",
code,
{0x40400000u, 0x40800000u, 0x40a00000u, 0x40c00000u},
{O::V_MOV_B32, O::S_MOV_B64, O::EXP, O::S_MOV_B32, O::S_ENDPGM}};
}
GraphicsCase GraphicsBranchPathFinalVmExportDiscardsInactiveExec() {
using O = ShaderOpcode;
std::vector<u32> code;
AppendVMovLiteral(&code, 0, 0x3f000000u);
AppendVMovLiteral(&code, 1, 0x3f400000u);
AppendVMovLiteral(&code, 2, 0x3f800000u);
AppendVMovLiteral(&code, 3, 0x40000000u);
code.push_back(EncodeExp0(0x00, 0xf, false));
code.push_back(EncodeExp1(0, 1, 2, 3));
code.push_back(EncodeSop1(0x04, 126, InlineU32(0)));
code.push_back(EncodeExp0(0x09, 0x0, true, false, true));
code.push_back(EncodeExp1(0, 0, 0, 0));
code.push_back(EncodeSopc(0x06, InlineU32(0), InlineU32(1)));
const auto branch_index = code.size();
code.push_back(0);
AppendEnd(&code);
const auto later_export_index = code.size();
code[branch_index] = EncodeSopp(
0x05, static_cast<u32>(later_export_index - branch_index - 1u));
code.push_back(EncodeSMovB32(126, 193u));
code.push_back(EncodeSMovB32(127, 193u));
AppendVMovLiteral(&code, 4, 0x40400000u);
AppendVMovLiteral(&code, 5, 0x40800000u);
AppendVMovLiteral(&code, 6, 0x40a00000u);
AppendVMovLiteral(&code, 7, 0x40c00000u);
code.push_back(EncodeExp0(0x00, 0xf, true, false, true));
code.push_back(EncodeExp1(4, 5, 6, 7));
AppendEnd(&code);
return {"GraphicsBranchPathFinalVmExportDiscardsInactiveExec",
code,
{0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u},
{O::V_MOV_B32, O::EXP, O::S_MOV_B64, O::S_CMP_EQ_U32,
O::S_CBRANCH_SCC1, O::S_MOV_B32, O::S_ENDPGM}};
}
GraphicsCase GraphicsObjectPrimitiveId(bool discard) {
auto test = discard ? GraphicsNullVmExportDiscardsInactiveExec()
: GraphicsDirectSgprPushConstantExport();
test.name = discard ? "GraphicsObjectPrimitiveIdDiscard"
: "GraphicsObjectPrimitiveId";
test.width = 2;
test.layers = 2;
test.object_ids = {0x0001ffffu, 0xffffffffu};
// Each triangle covers one pixel; the second instance renders the same pair
// into layer 1, making primitive progression and instance reset observable.
for (const auto &position : std::array<std::array<float, 2>, 6>{{
{-1.0f, -1.0f}, {0.0f, -1.0f}, {-0.5f, 1.0f},
{0.0f, -1.0f}, {1.0f, -1.0f}, {0.5f, 1.0f}}}) {
test.vertices.insert(test.vertices.end(),
{std::bit_cast<u32>(position[0]),
std::bit_cast<u32>(position[1]), 0, 0, 0, 0x3f800000u});
}
const auto color = test.expected_pixel;
test.expected_pixel.clear();
for (u32 object_id : test.object_ids) {
for (u32 pixel = 0; pixel < test.width * test.layers; ++pixel) {
test.expected_pixel.insert(test.expected_pixel.end(), color.begin(), color.end());
}
// RT7 is sparse and scalar uint, not a remapped or float PS export.
for (u32 layer = 0; layer < test.layers; ++layer) {
test.expected_pixel.push_back(discard ? 0xfeedfaceu : object_id);
test.expected_pixel.push_back(discard ? 0xfeedfaceu : object_id + 1u);
}
}
return test;
}
TestCase MultipleWorkitemsGlobalId() {
using O = ShaderOpcode;
std::vector<u32> code = {
EncodeVop1(0x01, 1, 0),
EncodeVop2(0x1a, 1, InlineU32(2), 1),
EncodeVop2(0x25, 1, Vgpr(0), 1),
EncodeVop2(0x25, 2, InlineU32(64), 1),
EncodeVop2(0x1a, 3, InlineU32(2), 1),
};
AppendBufferStoreDword(&code, 2, 3);
AppendEnd(&code);
TestCase test;
test.name = "MultipleWorkitemsGlobalId";
test.code = code;
test.expected = {64, 65, 66, 67, 68, 69, 70, 71};
test.opcodes = {O::V_MOV_B32, O::V_LSHLREV_B32, O::V_ADD_NC_U32,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.compute_info.threads_num[0] = 4;
test.compute_info.threads_num[1] = 1;
test.compute_info.threads_num[2] = 1;
test.compute_info.group_id[0] = true;
test.compute_info.thread_ids_num = 1;
test.compute_info.workgroup_register = 0;
test.has_compute_info = true;
test.dispatch_x = 2;
return test;
}
TestCase DispatcherIrreducibleControlFlow() {
using O = ShaderOpcode;
std::vector<u32> code;
code.push_back(EncodeSopp(0x05, 5)); // entry -> B, fallthrough A
AppendVMovU32(&code, 0, 7);
AppendStoreVgpr(&code, 0, 0);
code.push_back(EncodeSopp(0x04, 1)); // A exits while SCC is initially zero
code.push_back(EncodeSopp(0x02, 0xfffau)); // B -> A
AppendEnd(&code);
TestCase test;
test.name = "DispatcherIrreducibleControlFlow";
test.code = code;
test.expected = {7};
test.opcodes = {O::S_CBRANCH_SCC1, O::V_MOV_B32, O::BUFFER_STORE_DWORD,
O::S_CBRANCH_SCC0, O::S_BRANCH, O::S_ENDPGM};
return test;
}
#include "ShaderRayTracingGpuTests.inc"
std::vector<TestCase> MakeCases() {
std::vector<TestCase> cases;
cases.reserve(128);
auto AddCase = [&cases](TestCase (*factory)()) {
cases.push_back(factory());
};
cases.push_back(BufferWorkgroupPublication(32));
cases.push_back(BufferWorkgroupPublication(64));
cases.push_back(BufferWorkgroupPublication(32, true));
cases.push_back(BufferWorkgroupPublication(32, false, true));
cases.push_back(BufferWorkgroupPublication(64, false, true));
AddCase(IntegerAddSubMul);
AddCase(BitwiseOps);
AddCase(Shifts);
AddCase(ScalarZeroShiftWithRuntimeCount);
AddCase(ExactPushConstantExtent);
AddCase(ScalarShiftCountsMaskLowBits);
AddCase(Rdna2ScalarOpcodes);
AddCase(ScalarTrapDisabledFallsThroughExactRaw);
AddCase(ScalarWaitcntDepctrCapturedVmVsrc);
AddCase(ScalarExtendedArithmetic);
AddCase(ScalarArithmeticSccCarryBorrowOverflow);
AddCase(ScalarMinMaxSccComparisonEdges);
AddCase(ScalarAbsI32UpdatesScc);
AddCase(ScalarAbsdiffBitset64AndSetprio);
AddCase(ScalarShiftLeftAddSccCarryEdges);
AddCase(ScalarCompareOps);
AddCase(ScalarShiftAddAndMaskOps);
AddCase(ScalarNotB64UpdatesScc);
AddCase(ScalarQuadmaskB64);
AddCase(ScalarFlbitI32B64Gpu);
AddCase(ScalarFlbitI32B32Gpu);
AddCase(ScalarSaveExecOps);
AddCase(ScalarOrn2SaveexecUsesSourceOrNotExec);
cases.push_back(ScalarOrn2SaveexecB32(32, 32));
cases.push_back(ScalarOrn2SaveexecB32(64, 64));
cases.push_back(ScalarOrn2SaveexecB32(32, 4));
cases.push_back(ScalarSubvectorLoops(32));
cases.push_back(ScalarSubvectorLoops(64));
AddCase(ScalarGetpcWritesNextInstructionPc);
AddCase(ScalarBitfieldPack);
AddCase(ScalarZeroBitOps);
AddCase(ScalarBitcmpB64DynamicOperands);
AddCase(ScalarBitcmpB64IntegerConstants);
AddCase(ScalarBrevB32PreservesScc);
for (const u32 bits : {8u, 16u}) {
cases.push_back(ScalarSextMasks(bits, 32));
cases.push_back(ScalarSextMasks(bits, 64));
}
AddCase(ScalarSextConstantsAndAliases);
cases.push_back(ScalarBrevB64OperandsAndMasks(32));
cases.push_back(ScalarBrevB64OperandsAndMasks(64));
AddCase(ScalarBfeI32CapturedRawSignExtends);
AddCase(BitfieldExtractWidthPastEndEdges);
AddCase(Scalar64BitOps);
AddCase(ScalarDynamic64BitOps);
cases.push_back(ScalarAshrI64Edges(false));
cases.push_back(ScalarAshrI64Edges(true));
AddCase(ScalarAshrI64OperandsAndAliases);
cases.push_back(ScalarAshrI64Masks(32));
cases.push_back(ScalarAshrI64Masks(64));
AddCase(VectorDynamicU64ShiftEdges);
AddCase(ScalarConditionalMoveB64);
AddCase(ScalarConditionalMoveB64PreservesMasks);
AddCase(ScalarAndn2B64SccUsesMaskShadow);
AddCase(ScalarMaskHighWriteInvalidatesProvenance);
cases.push_back(ScalarSelectB64PreservesMaskProvenance(32));
cases.push_back(ScalarSelectB64PreservesMaskProvenance(64));
AddCase(ScalarWqmB64SelectsSccDomain);
AddCase(ScalarWqmB64PreservesPartialMasks);
cases.push_back(ScalarWqmB32Masks(32));
cases.push_back(ScalarWqmB32Masks(64));
AddCase(ScalarMaskProvenanceOverlapAndMixedBinary);
cases.push_back(ScalarB32MaskDataflow(32));
cases.push_back(ScalarB32MaskDataflow(64));
AddCase(ScalarLiteral);
AddCase(VectorMoves);
AddCase(VectorVop3MoveAppliesFloatSourceModifiers);
AddCase(VectorIntegerOps);
AddCase(VectorIntegerExtremaRuntimeEdges);
AddCase(VectorFfbhI32NativeAndVop3OnGpu);
AddCase(Vop1SdwaBfrevSourceSelectors);
AddCase(Vop1SdwaFfbhCapturedScalarLowWordSource);
AddCase(Vop1SdwaFfblCapturedHighWordSource);
AddCase(Vop1SdwaNotCapturedByte0Source);
AddCase(Vop1SdwaNotPreservesHighWordDestination);
AddCase(Vop1SdwaNotPartialSourcesAndDestinations);
cases.push_back(Vop2SdwaBitwiseByteDestinations(32));
cases.push_back(Vop2SdwaBitwiseByteDestinations(64));
cases.push_back(Vop2SdwaLshlrevByteDestinations(32));
cases.push_back(Vop2SdwaLshlrevByteDestinations(64));
AddCase(Vop1SdwaMovByteDestinations);
cases.push_back(Vop1SdwaMovByteSources(32));
cases.push_back(Vop1SdwaMovByteSources(64));
AddCase(Vop2SdwaSubNcExactByte2Destination);
AddCase(Vop2SdwaAddNcCapturedHighWordDestination);
AddCase(Vop2SdwaAshrrevCapturedWord0SignExtends);
cases.push_back(Vop2SdwaMaxI32CapturedHighWord(32));
cases.push_back(Vop2SdwaMaxI32CapturedHighWord(64));
cases.push_back(Vop2SdwaMinI32CapturedByte0(32));
cases.push_back(Vop2SdwaMinI32CapturedByte0(64));
cases.push_back(Vop2SdwaMulI24Destinations(32));
cases.push_back(Vop2SdwaMulI24Destinations(64));
cases.push_back(Vop2SdwaMulU24Destinations(32));
cases.push_back(Vop2SdwaMulU24Destinations(64));
AddCase(Vop2SdwaLshrrevCapturedByte1Source);
AddCase(Vop2SdwaSubNcPreservesByteAndWordDestinations);
AddCase(Vop3CvtPkI16I32Captured);
AddCase(Vop3MulLoU16CapturedAndSelectors);
AddCase(Vop3MadI16CapturedSelectorsAndSaturation);
AddCase(Vop3F32ClampFlushesNaN);
AddCase(Vop3Min3U16CapturedAndSelectors);
AddCase(Vop3Med3I16Captured);
AddCase(Vop2SdwaMinU32PreservesWordDestination);
AddCase(VectorShiftCountsMaskLowBits);
AddCase(VectorVop3LshrrevB64Captured);
AddCase(VectorVop3LshlrevB64Captured);
AddCase(VectorVop3IntegerOps);
AddCase(VectorBfeI32SignExtendsField);
AddCase(VectorAlignByteUsesTwoBitByteOffset);
AddCase(VectorCarryAndBitCountOps);
AddCase(VectorMbcntUsesThreadMask);
AddCase(VectorAddcWritesPerLaneCarryOut);
AddCase(VectorAddcUsesPerLaneCarryIn);
AddCase(VectorSubCoCiU32CompactAndVop3);
AddCase(VectorSubrevCoCiU32ExactRawOnGpu);
AddCase(VectorVop3BSubrevCoCiUsesEncodedMasks);
AddCase(VectorVop3BCarryOutWritesSgprMask);
AddCase(VectorVop3BCarryOutUsesEncodedSdst);
AddCase(VectorVop3BSubCoU32UsesRdna2Opcode310);
AddCase(VectorMadU64U32UnsignedCarryOut);
AddCase(VectorLaneAndPackedOps);
AddCase(Vop2PkFmacF16AccumulatesPackedHalvesIndependently);
AddCase(Vop2PkFmacF16DppNegatesBothPackedHalves);
AddCase(Vop3pOpselHiUsesArchitecturalSourceBits);
AddCase(Vop3pIntegerNegationCapturedAndSelectedHalves);
AddCase(Vop3pInt16Ops);
AddCase(CvtPkU8F32PacksSelectedByte);
AddCase(CvtPkrtzF16F32SubnormalRoundsTowardZero);
AddCase(CvtPkrtzF16F32SdwaAndOutputModifiers);
AddCase(PackedMinMaxF16NanAndSignedZeroEdges);
AddCase(VectorMinMaxF16Ops);
AddCase(VectorCvtF16U16ByteSource);
AddCase(VectorCvtU16F16Sdwa);
AddCase(NativeAndSdwa16BitDestinationWrites);
AddCase(VectorMinMaxMed3F16Ops);
AddCase(VectorSpecialF16Ops);
AddCase(VectorFractF16CapturedAndEdges);
AddCase(VectorFractF16Modifiers);
AddCase(VectorCosF16CapturedSdwaAndEdges);
AddCase(VectorSinF16SdwaAndEdges);
AddCase(VectorWritelaneIgnoresExecMask);
AddCase(VectorReadlaneFromInactiveWrittenLane);
AddCase(VectorLaneWave32RuntimeSelectorWraps);
AddCase(VectorReadlaneSelectsTwoKeysWithinWave);
AddCase(VectorPermlanex16);
AddCase(VectorPermlane16FetchInactiveZero);
AddCase(VectorPermlane16FetchInactiveFi);
cases.push_back(VectorDpp8Captured(false));
for (const u32 wave_size : {32u, 64u})
for (const bool fi : {false, true})
cases.push_back(VectorDpp8Captured(true, wave_size, fi));
AddCase(VectorDppQuadPermuteReverse);
AddCase(VectorDppRowXmask);
cases.push_back(VectorDppRowShare(32));
cases.push_back(VectorDppRowShare(64));
cases.push_back(ComputeWave64Minimum(false));
cases.push_back(ComputeWave64Minimum(true));
AddCase(VectorDppBankMaskPreservesDestination);
AddCase(VectorDppBoundsControlZeroPreservesDestination);
AddCase(Vop3FmacF32NegatedSourceAccumulates);
AddCase(VectorMadF32RoundsProduct);
AddCase(VectorMadF32FlushesDenorms);
AddCase(Vop3LdexpSourceModifier);
AddCase(Vop1MoveRelSource);
AddCase(Vop1MoveRelDestination);
AddCase(VectorFloatSpecialOps);
AddCase(CubeIdCapturedNegationAndOutputScale);
AddCase(MadMixF16LiteralHalfSourceUsesOpsel);
AddCase(MadMixF16NegHiIsAbsAndNegIsIndependent);
AddCase(VectorVop3FmaF16UsesRdna2Opcode34b);
AddCase(FloatInlineConstantF16UsesNumericValue);
AddCase(Vop3Int16Ops);
AddCase(VectorFloatControlContractPreservesInfNan);
AddCase(VectorFloatArithmeticOps);
AddCase(VectorMinMaxF32NanAndSignedZeroEdges);
AddCase(VectorMed3F32NanUsesMin3Path);
AddCase(VectorFloatConversionOps);
AddCase(VectorFrexpF32Edges);
AddCase(CvtF32ToIntSaturatesNaNAndOutOfRange);
AddCase(VectorSpecialF32FlushesDenormalInputs);
AddCase(VectorRcpIflagF32IntegerReciprocal);
AddCase(VectorCompareF64Edges);
AddCase(VectorCompareInteger64Edges);
cases.push_back(VectorCompareExecWaveMasks(32));
cases.push_back(VectorCompareExecWaveMasks(64));
cases.push_back(VectorCompare64WaveMasks(32));
cases.push_back(VectorCompare64WaveMasks(64));
cases.push_back(VectorCompare64WaveMasks(32, true));
cases.push_back(VectorCompare64WaveMasks(64, true));
AddCase(VectorF64CapturedScreenSpaceShadows);
AddCase(VectorFractF64CapturedAndEdges);
AddCase(VectorMinMaxF64CapturedAndEdges);
AddCase(VectorRoundF64EdgesAndModifiers);
AddCase(VectorFractF64CapturedChainRuntimeExec);
AddCase(VectorF64ModesModifiersAndExec);
AddCase(VectorF64WideningConversions);
AddCase(VectorSinCosMaxFiniteSpecialCases);
AddCase(VectorCompareF32ExposureGuard);
for (const auto mode : {0xc0u, 0xe0u}) {
cases.push_back(VectorCompareF32DenormalModes(mode));
}
AddCase(VectorCompareOps);
AddCase(VectorVop3CompareEqI64OnGpu);
AddCase(VectorVop3CompareEqU64OnGpu);
AddCase(VectorVop3CompareGtU64OnGpu);
AddCase(VectorVopcCompareLtU64OnGpu);
AddCase(VectorVop3CompareNeU64OnGpu);
AddCase(VectorVopcCmpxNeU64CapturedExecMask);
AddCase(VectorVop3CmpxNeI64CapturedExecMask);
AddCase(VectorCompareClassF32);
AddCase(VectorCompareClassF32ImmediateAndDynamicMasks);
AddCase(VectorVopcSdwaCmpxClassF32CapturedExecMask);
cases.push_back(VectorCmpClassF16(32));
cases.push_back(VectorCmpClassF16(64));
cases.push_back(VectorCmpxClassF16(32));
cases.push_back(VectorCmpxClassF16(64));
AddCase(VectorCompareF16Ops);
AddCase(VectorCompareI16U16Ops);
AddCase(Wave32VccMasksPreserveOtherHalf);
AddCase(Vop2SdwaCndmaskSourceModifier);
AddCase(Vop2SdwaCndmaskCapturedByte3Source);
AddCase(Vop3CndmaskUsesSgprMaskLaneBits);
AddCase(Vop3CndmaskAllowsDataSourceModifier);
AddCase(VectorCompareExecOps);
AddCase(VectorVopcCmpxOrderedCapturedExecMask);
AddCase(VectorVop3FloatCompareNegSourceModifier);
AddCase(VectorVop3CmpxWritesExecMask);
AddCase(VectorVopcSdwaCmpxWritesExecMask);
AddCase(VectorVopcCmpxGtU16CapturedSdwaExecMask);
AddCase(VectorVopcCmpxLtU16CapturedSdwaExecMask);
AddCase(VectorVopcCmpxLtI16CapturedSdwaExecMask);
for (u32 wave_size : {32u, 64u}) {
cases.push_back(VectorVopcCmpxLtI16WaveMasks(wave_size));
}
AddCase(VectorVopcCmpxEqU16SdwaCompactVop3ExecMask);
AddCase(VectorVopcCmpxNeU16SdwaCompactVop3ExecMask);
AddCase(VectorVopcCmpNgtF16CapturedSdwaAndEdges);
AddCase(VectorVopcCmpNltF16CapturedSdwaAndEdges);
AddCase(VectorVopcCmpxNgtF16CapturedSdwaExecMask);
AddCase(VectorVopcCmpxNleF16CapturedSdwaExecMask);
AddCase(VectorCompareInvertedMaskSelect);
AddCase(BranchSelect);
AddCase(SimpleLoop);
AddCase(SharedReturnKeepsSelectedValues);
AddCase(SiblingSharedExitKeepsCapturedConditions);
AddCase(BranchVccnzUsesWaveMask);
AddCase([] { return ScalarBranchRestoresInactiveLanes(32, 32); });
AddCase([] { return ScalarBranchRestoresInactiveLanes(64, 64); });
AddCase([] { return ScalarBranchRestoresInactiveLanes(32, 4); });
AddCase([] { return ScalarBranchRestoresInactiveLanes(64, 4); });
AddCase([] { return ScalarBranchRestoresInactiveLanes(32, 32, true); });
AddCase([] { return ScalarBranchRestoresInactiveLanes(64, 64, true); });
AddCase(BranchVccnzUsesCarryProducedWaveMask);
AddCase(ScalarMemRealtimeCapturedPlaceholder);
AddCase(ScalarMemoryLoadVariants);
AddCase(ScalarBufferOffsetAlignmentAndCarry);
AddCase(ScalarLoadSignedImmediateOffsetAddsSoffset);
AddCase(ScalarLoadAlignsComponentsAndMasksAddress);
AddCase(ScalarLoadAlignsDynamicBase);
cases.push_back(ScalarBufferFromLoopReadlane(32));
cases.push_back(ScalarBufferFromLoopReadlane(64));
AddCase(BufferLoadStore);
AddCase(BufferLoadDwordOffenIdxenUsesVaddrPlusOneOffset);
AddCase(BufferStoreDwordOffenIdxenUsesVaddrPlusOneOffset);
AddCase(BufferLoadDwordNoAddressFlagsIgnoresVaddr);
AddCase(BufferLoadDwordIdxenUsesDescriptorStride);
AddCase(BufferStoreDwordIdxenUsesDescriptorStride);
AddCase(BufferStoreDwordAppliesHostOffset);
cases.push_back(BufferOffsetsUsePackedWords(false));
cases.push_back(BufferOffsetsUsePackedWords(true));
AddCase(BufferLoadVariants);
for (const auto opcode : {ShaderOpcode::BUFFER_LOAD_SHORT_D16, ShaderOpcode::BUFFER_LOAD_SHORT_D16_HI})
cases.push_back(BufferLoadD16Captured(opcode));
for (const u32 wave : {32u, 64u})
cases.push_back(BufferLoadD16Captured(ShaderOpcode::BUFFER_LOAD_UBYTE_D16, wave));
for (const u32 component : {4u, 2u, 0u}) {
cases.push_back(BufferSubwordLoadsAtHostOffset(2, component, false));
}
cases.push_back(BufferSubwordLoadsAtHostOffset(2, 4, true));
cases.push_back(BufferSubwordLoadsAtHostOffset(1, 5, false));
cases.push_back(BufferSubwordLoadsAtHostOffset(1, 5, true));
AddCase(BufferSubwordHostOffsetOverflow);
cases.push_back(BufferSubwordStoresAtHostOffset(2));
cases.push_back(BufferSubwordStoresAtHostOffset(1));
AddCase(BufferLoadDwordx2SnapshotsOverlappingAddress);
AddCase(BufferLoadDwordx3SnapshotsOverlappingAddress);
AddCase(BufferLoadDwordx4SnapshotsOverlappingAddress);
AddCase(BufferLoadDwordx4ZeroesOnlyOutOfBoundsTail);
AddCase(BufferLoadsGpuSelectedDescriptors);
AddCase(BufferLoadDwordGpuSelectedDescriptors);
AddCase(BufferLoadDwordx3GpuSelectedDescriptors);
AddCase(BufferLoadFormatXGpuSelectedDescriptors);
AddCase(BufferLoadFormatXGpuSelectedDescriptors64);
AddCase(BufferLoadUshortGpuSelectedDescriptors);
AddCase(BufferLoadUshortGpuSelectedDescriptors64);
AddCase(BufferLoadUbyteGpuSelectedDescriptors);
AddCase(BufferLoadUbyteGpuSelectedDescriptors64);
AddCase(BufferStoreDwordx4DropsOnlyOutOfBoundsTail);
AddCase(BufferLoadFormatXyzwRejectsPartialRecord);
AddCase(BufferStoreFormatXyzwDropsPartialRecord);
AddCase(BufferLoadFormatXRejectsPartialRecord);
AddCase(BufferStoreFormatXRejectsPartialRecord);
AddCase(BufferLoadFormatXyRejectsPartialRecord);
AddCase(BufferStoreFormatXyRejectsPartialRecord);
AddCase(BufferStoreVariants);
for (bool masked : {false, true}) {
cases.push_back(BufferStoreD16Hi(false, masked, 64));
for (u32 wave_size : {32u, 64u})
cases.push_back(BufferStoreD16Hi(true, masked, wave_size));
}
AddCase(BufferFormatVariants);
AddCase(BufferLoadFormatXyzwSnapshotsOverlappingAddress);
AddCase(BufferLoadFormatXyzwResource32_32FloatAppliesXy01);
AddCase(BufferLoadFormatXyzwResource323232FloatRepeatsSource);
AddCase(BufferLoadFormatXyzwResource8_8_8_8UintAppliesWzy1);
AddCase(BufferLoadFormatXyzwInactiveExecPreservesOverlappingAddress);
AddCase(BufferFormatStoreVariants);
AddCase(BufferStoreFormatXResource16UintWritesHalfword);
AddCase(BufferStoreFormatXResource16UintPreservesAdjacentLanes);
AddCase(BufferStoreFormatXyzwPackedUnormSkinningVectors);
AddCase(BufferStoreFormatXZeroFillsPackedRecord);
AddCase(BufferStoreFormatXyzwFloat16ConvertsComponents);
AddCase(BufferStoreFormatXyzwSnorm16CapturedSkinningVectors);
AddCase(BufferStoreFormatXSnorm16ClampsRoundsAndPreservesHalfwords);
for (auto &test : BufferLoadFormatD16XCases()) {
cases.push_back(std::move(test));
}
for (auto &test : BufferLoadFormatD16XyzwCases()) {
cases.push_back(std::move(test));
}
for (auto &test : BufferStoreFormatD16Cases()) {
cases.push_back(std::move(test));
}
for (auto &test : FormattedStoreConversionCases()) {
cases.push_back(std::move(test));
}
AddCase(BufferLoadFormatXResource8UintZeroExtendsByte);
AddCase(BufferLoadFormatXyResource88UintExtractsBytes);
AddCase(BufferLoadFormatXyResource8888UnormConvertsFirstTwoComponents);
AddCase(BufferStoreFormatXyResource88UintWritesBytes);
AddCase(BufferStoreFormatXyzResource3232UintWritesTwoDwords);
AddCase(BufferStoreFormatXyzwResource323232UintWritesThreeDwords);
AddCase(BufferStoreFormatXyResource32UintWritesOneDword);
AddCase(BufferStoreFormatXyzResource8UintWritesOneByte);
AddCase(BufferStoreFormatXAddTidUsesLaneIndex);
AddCase(BufferStoreFormatXDropsOutOfRangeRecord);
AddCase(TBufferLoadVariants);
AddCase([] { return TBufferCapturedZeroStrideOob(); });
AddCase([] { return TBufferCapturedZeroStrideOob(true); });
AddCase([] { return TBufferCapturedZeroStrideOob(false, true); });
AddCase(BufferZeroStrideOobFormatsAndWidths);
AddCase(TBufferLoadFormatXyzwSnapshotsOverlappingAddress);
AddCase(TBufferLoadFormatXyzwPackedSnapshotsOverlappingAddress);
AddCase(TBufferLoadFormatX8UintZeroExtendsByte);
AddCase(TBufferLoadFormatX8888UintExtractsFirstByte);
AddCase(TBufferLoadFormatXIdxenUsesDescriptorStride);
AddCase(TBufferLoadFormatX16FloatConvertsToFloat);
AddCase(TBufferStoreFormatX8UintWritesOneByte);
AddCase(TBufferStoreFormatXSintWritesSubDword);
AddCase(TBufferLoadFormatXSintSignExtendsSubDword);
AddCase(TBufferLoadFormatXy1616IntegerComponents);
AddCase(TBufferStoreFormatXy1616IntegerComponents);
AddCase(TBufferLoadFormatXyz16161616UintLoadsHalfwords);
AddCase(TBufferLoadFormatXy1616UnormConvertsToFloat);
AddCase(TBufferLoadFormatXy88UnormConvertsToFloat);
AddCase(TBufferLoadFormatXy88SnormConvertsToFloat);
AddCase(TBufferLoadFormatXy8888UnormConvertsFirstTwoComponents);
AddCase(TBufferLoadFormatXyzw8888UintExtractsBytes);
AddCase(TBufferLoadFormatXyzw1010102SnormConvertsToFloat);
AddCase(TBufferLoadFormatXyz111110FloatUnpacks);
AddCase(TBufferLoadFormatXyzw3232FloatZerosMissingComponents);
AddCase(TBufferStoreFormatXyzw3232FloatWritesOnlyPresentComponents);
AddCase(TBufferStoreFormatXy88IntegerComponents);
AddCase(TBufferLoadFormatXy88IntegerComponents);
AddCase(TBufferStoreVariants);
cases.push_back(FlatLoadDlcCaptured(32));
cases.push_back(FlatLoadDlcCaptured(64));
AddCase(FlatStoreSlcCaptured);
AddCase(FlatLoadVariants);
AddCase(FlatSubdwordLoadsApplyByteOffset);
cases.push_back(GlobalLoadShortD16Captured(32));
cases.push_back(GlobalLoadShortD16Captured(64));
AddCase(FlatLoadShortD16AddressSegments);
AddCase(FlatVirtualAddressRebasesGuestAllocation);
AddCase(GlobalSignedImmediateRebasesBeforeSaddr);
AddCase(FlatSegmentIgnoresSaddrAndMasksOffsetMsb);
AddCase(ScratchIsPrivatePerInvocation);
cases.push_back(FlatStackApertures(32));
cases.push_back(FlatStackApertures(64));
cases.push_back(BvhIntersections(true, true, 1));
cases.push_back(BvhIntersections(true, true, 0, true));
for (bool barycentrics : {false, true}) {
for (bool sorted : {false, true}) {
cases.push_back(BvhIntersections(barycentrics, sorted));
}
}
AddCase(DsReadWriteVariants);
for (u32 wave_size : {32u, 64u})
for (bool gds : {false, true})
cases.push_back(DsReadWriteFullRegionAddress(wave_size, gds));
AddCase(DsWriteB16D16HiCapturedUsesHighHalf);
AddCase(DsWriteB8D16HiWritesByteTwo);
AddCase(DsReadU16D16CapturedPreservesHighHalf);
AddCase(DsReadU16D16HiCapturedPreservesLowHalf);
AddCase(DsAppendConsumeUsesEncodedLdsSelector);
cases.push_back(DsAppendAllocatesAcrossWaves(32));
cases.push_back(DsAppendAllocatesAcrossWaves(64));
AddCase(DsAppendUsesEncodedGdsSelector);
for (u32 wave_size : {32, 64}) {
cases.push_back(DsOrderedCountAddressAndExec(wave_size, false));
cases.push_back(DsOrderedCountAddressAndExec(wave_size, true));
}
AddCase(DsAppendConsumeGdsRegionBounds);
AddCase(DsGdsSubdwordAndAtomicWrites);
AddCase(DsReadWrite2Variants);
AddCase(DsWideReadSnapshotsOverlappingAddress);
AddCase(DsReadWrite2EqualOffsetsUseData0);
AddCase(DsWideLdsPartialBounds);
AddCase(DsWideGdsPartialBounds);
AddCase(DsAtomicNoReturnVariants);
for (bool gds : {false, true}) {
cases.push_back(DsMaskedOrBounds(gds));
}
for (u32 wave_size : {32, 64}) {
cases.push_back(DsMaskedOrContention(wave_size));
}
AddCase(DsAtomicReturnVariants);
AddCase(DsAtomicReturn64);
AddCase(DsCompareStoreReturn32);
for (bool add : {false, true}) {
cases.push_back(DsAtomic64Bounds(add));
for (u32 wave_size : {32, 64}) {
cases.push_back(DsAtomic64Contention(add, wave_size));
}
}
AddCase(BufferAtomicAndX2GlcAndExec);
AddCase(BufferAtomicAddX2GlcAndExec);
AddCase(ImageAtomicSignedMinMax<false>);
AddCase(ImageAtomicSignedMinMax<true>);
AddCase(ImageAtomicSignedMinMax<false, true>);
AddCase(ImageAtomicSignedMinMax<true, true>);
for (bool decrement : {false, true}) {
for (bool gds : {false, true}) {
cases.push_back(DsBoundedAtomicBoundaries(decrement, gds));
for (u32 wave_size : {32u, 64u}) {
cases.push_back(DsBoundedAtomicContention(decrement, gds, wave_size));
}
}
}
AddCase(DsMiscVariants);
AddCase(DsFloatMinMaxIgnoresData1);
AddCase([] { return DsFloatMinMaxClasses(false); });
AddCase([] { return DsFloatMinMaxClasses(true); });
AddCase(DsSwizzleInvalidSourceLaneZero);
AddCase(DsPermuteCapturedExecOffsetAndWrap);
AddCase(DsPermuteWave64UsesIndependentHalves);
AddCase(DsBpermuteCapturedExecOffsetAndWrap);
AddCase(DsBpermuteWave64UsesIndependentHalves);
AddCase(Wave64CrossHalfLaneAndLds);
AddCase(Wave64RawMasksAndScalarBranch);
AddCase(Wave64PartialMultidimensionalWorkgroup);
AddCase(Wave64AppendConsumeHighHalf);
AddCase(BufferAtomicVariants);
AddCase(BufferAtomicCmpSwapExactRaw);
AddCase(BufferAtomicGlc0DoesNotReturnOldValue);
AddCase(BufferAtomicFMinExactRawGlcModes);
AddCase(BufferAtomicFMinSpecialValues);
AddCase(BufferAtomicFMinContendedWorkgroup);
AddCase(BufferAtomicFMaxExactRawGlcModes);
AddCase(BufferAtomicFMaxSpecialValues);
AddCase(BufferAtomicFMaxContendedWorkgroup);
AddCase(ImageLoadVariants);
AddCase(ImageLoadR32UintUsesIntegerSampledImage);
AddCase(ImageLoadFmaskUsesNativeSampleMapping);
AddCase(ImageLoadSintUsesSignedSampledImage<false>);
AddCase(ImageLoadSintUsesSignedSampledImage<true>);
AddCase(ImageLoadPackedUintUnpacksAndSwizzles);
AddCase(ImageLoadPackedUnorm<false, false>);
AddCase(ImageLoadPackedUnorm<false, true>);
AddCase(ImageLoadPackedUnorm<true, false>);
AddCase(ImageLoadPackedUnorm<true, true>);
AddCase(ImageSamplePackedUintConvertsSampleAndGather);
AddCase(ImageSamplePackedUnormBilinear<false>);
AddCase(ImageSamplePackedUnormBilinear<true>);
AddCase(ImageSamplePacked8<false>);
AddCase(ImageSamplePacked8<true>);
AddCase(ImageSamplePacked8<false, true>);
AddCase(ImageLoadR128IgnoresAdjacentMaskSgprs);
AddCase(ImageLoad1DUsesScalarCoordinate);
AddCase(ImageGather2DInstructionWith1DDescriptor);
AddCase(ImageLoad1DArrayUsesLayerCoordinate);
AddCase(ImageLoad1DArrayDescriptorUsesSelectedLayer);
AddCase(ImageLoadMipUsesVaddr2Lod2D);
AddCase(ImageLoadMipNsaUsesSelectedAddressVgprs);
AddCase(ImageLoadA16UintCoordsOnGpu);
AddCase(ImageGetResinfoDmaskWidthHeight);
AddCase(ImageGetResinfoDmaskMipLevels);
AddCase(ImageSampleAndGather);
AddCase(ImageCubeGradientsPreserveDerivatives);
AddCase(ImageGatherExplicitLod);
AddCase(ImageGatherLodPerLane<0>);
AddCase(ImageGatherLodPerLane<1>);
AddCase(ImageGatherLodPerLane<2>);
AddCase(ImageGatherLodPerLane<3>);
AddCase(ImageGatherLodPerLane<4>);
AddCase(ImageGatherLodPerLane<5>);
AddCase(ImageGatherLodPerLane<6>);
AddCase(ImageGatherLodPerLane<7>);
AddCase(ImageGatherLodPerLane<8>);
AddCase(ImageD16GatherPacksHalfPairs);
AddCase(ImageSampleA16SamplerCoordsOnGpu);
AddCase(ImageSampleOpcodeAliasUsesNormalCoords);
AddCase(ImageSampleA16OffsetKeepsTexelOffset32BitOnGpu<32>);
AddCase(ImageSampleA16OffsetKeepsTexelOffset32BitOnGpu<64>);
AddCase(ImageSampleOffsetsReadFourNeighbors<32>);
AddCase(ImageSampleOffsetsReadFourNeighbors<64>);
AddCase(ImageSampleA16CompareBiasRdna2AddressOrder);
AddCase(ImageGatherCompareOpcodes);
AddCase(ImageStoreVariants);
AddCase(ImageD16StoreUnpacksHalfPairs);
AddCase(ImageStoreMipSelectsPpsa01340Descriptor);
AddCase(ImageStoreSingleMipBounds<0>);
AddCase(ImageStoreSingleMipBounds<1>);
AddCase(ImageStoreSingleMipBounds<2>);
AddCase(ImageStoreRgbOneUsesInverseSwizzle);
AddCase(ImageStoreDuplicateSelectorUsesInverseSwizzle);
AddCase(ImageStoreBgraUsesInverseSwizzle);
AddCase(ImageStoreYzwxUsesInverseSwizzle);
AddCase(ImageStoreR32FloatUsesFormatlessStorageImage);
AddCase(ImageStoreR32SintUsesRawUintView);
AddCase(ImageStoreR32UintUsesFormatlessStorageImage);
AddCase(ImageStorePackedUintSaturatesChannels);
AddCase(ImageStorePackedUintHonorsSparseDmask);
AddCase(ImageStorePackedUnorm<false>);
AddCase(ImageStorePackedUnorm<true>);
AddCase(ComputeTgSizeSgprUsesWaveMetadata);
AddCase(ImageStoreAndAtomicShareTypedBinding);
AddCase(ImageAtomicCompareSwapGlcAndExec);
AddCase(ImageAtomicSwapReturnsPreviousTexel);
AddCase(ImageStoreAndAtomicUseSeparateBindings);
AddCase(ImageAtomicVariants<>);
AddCase(ImageAtomicVariants<true>);
for (const auto &atomic : ImageAtomicIntegerCases) {
cases.push_back(ImageAtomicIntegerGlcAndExec(atomic, false));
cases.push_back(ImageAtomicIntegerGlcAndExec(atomic, true));
}
AddCase(ImageAtomicFloatGlcAndExec<true>);
AddCase(ImageAtomicFloatGlcAndExec<false>);
AddCase(ImageAtomicFloatSpecialValues<true>);
AddCase(ImageAtomicFloatSpecialValues<false>);
AddCase(ImageAtomicFloatContendedWorkgroup<true>);
AddCase(ImageAtomicFloatContendedWorkgroup<false>);
AddCase(ImageAtomicGlc0DoesNotReturnOldValue);
AddCase(MultipleWorkitemsGlobalId);
AddCase(DispatcherIrreducibleControlFlow);
return cases;
}
std::vector<GraphicsCase> MakeGraphicsCases() {
return {
GraphicsSparseWaveMinimum(),
GraphicsInterpolationExport(),
GraphicsPositionWExport(),
GraphicsPackedHalfCentroid(),
GraphicsPerspectiveSample(false),
GraphicsPerspectiveSample(true),
GraphicsPackedHalfInputAlias(false),
GraphicsPackedHalfInputAlias(true),
GraphicsSmoothRawInputAlias(),
GraphicsSmoothFlatInputAlias(false),
GraphicsSmoothFlatInputAlias(true),
GraphicsSmoothFlatInputAlias(false, true),
GraphicsSmoothFlatInputAlias(true, true),
GraphicsAncillaryLayer(false),
GraphicsAncillaryLayer(true),
GraphicsAncillarySampleId(),
GraphicsFrontFaceBits(false, false),
GraphicsFrontFaceBits(true, false),
GraphicsFrontFaceBits(false, true),
GraphicsFrontFaceBits(true, true),
GraphicsFlatInterpolatorExport(),
GraphicsDsAddtidScratchExport(),
GraphicsDirectSgprPushConstantExport(),
GraphicsInlineSrtScalarPromotionExport(),
GraphicsNullVmExportDiscardsInactiveExec(),
GraphicsMrt0OffVmExportDiscardsInactiveExec(),
GraphicsFinalVmExportSupersedesEarlierVmMask(),
GraphicsBranchPathFinalVmExportDiscardsInactiveExec(),
GraphicsObjectPrimitiveId(false),
GraphicsObjectPrimitiveId(true),
};
}
void CheckComputeLdsLimit(VulkanHarness &vulkan) {
constexpr const char *name = "ComputeLdsLimit";
std::vector<u32> code;
AppendVMovU32(&code, 1, 7);
code.push_back(EncodeDs0(0x0d)); // ds_write_b32 v0, v1
code.push_back(EncodeDs1(0, 1, 0));
AppendEnd(&code);
ShaderMapUserData(reinterpret_cast<uint64_t>(code.data()),
{.type = Prospero::ShaderBinaryType::kCs,
.code_size_bytes = static_cast<u32>(code.size() * sizeof(u32))});
GraphicContext graphics;
graphics.device = vulkan.Device();
graphics.physical_device_properties =
vulkan.RuntimeContext().GetPhysicalDeviceProperties();
auto &limit = graphics.physical_device_properties.limits.maxComputeSharedMemorySize;
// Reproduce a 48-KiB host, or a smaller host limit when the test device requires it.
limit = std::min(limit, 48u * 1024u) & ~511u;
const auto limit_units = static_cast<uint16_t>(limit / 512u);
HW::ComputeShaderInfo regs{};
regs.cs_regs.data_addr = reinterpret_cast<uint64_t>(code.data());
regs.cs_regs.num_thread_x = 1;
regs.cs_regs.num_thread_y = 1;
regs.cs_regs.num_thread_z = 1;
regs.cs_regs.wave_size = 32;
PipelineCache cache(graphics);
ShaderProgram previous;
uint16_t previous_units = 0;
for (const auto units : {static_cast<uint16_t>(limit_units - 1u), limit_units,
uint16_t{112}, uint16_t{128}, uint16_t{128}}) {
regs.cs_regs.lds_size = units;
ShaderComputeInputInfo input{};
const auto program = cache.GetComputeProgram(regs, {}, input);
Require(name, "guest allocation and device backing",
input.lds_size_dwords == units * 128u &&
input.lds_storage == (units > limit_units),
"compute LDS was truncated or chose backing that exceeds the device limit");
Require(name, "pipeline creation",
cache.GetComputePipeline(input, program).pipeline != nullptr,
"full-size LDS shader did not create a Vulkan compute pipeline");
if (previous.id != 0) {
Require(name, "guest-size cache identity",
(program.id == previous.id) == (units == previous_units),
"different LDS sizes shared a shader, or repeating a size recompiled it");
}
previous = program;
previous_units = units;
}
std::printf("[gpu] %-32s ok\n", name);
}
void CheckPs5GameExampleImageClearRuntimeShape() {
const auto MakeCode = [] {
std::vector<u32> code;
AppendVop3(&code, 0x346u, 4, 8, InlineU32(6), Vgpr(0));
for (u32 i = 0; i < 4; i++) {
code.push_back(EncodeVop1(0x01u, i, i + 4u));
}
code.push_back(EncodeMubuf0(0x07u, 0, true, false));
code.push_back(EncodeMubuf1(0, 0, 4));
AppendEnd(&code);
return code;
};
std::array<u32, 8> user_data{};
user_data[0] = 0x00010000u;
user_data[1] = 16u << 16u;
user_data[2] = 64;
user_data[3] =
(static_cast<uint32_t>(Prospero::BufferFormat::k32_32_32_32UInt) << 12u) |
4u | (5u << 3u) | (6u << 6u) | (7u << 9u);
std::fill(user_data.begin() + 4, user_data.end(), 0xff000000u);
ShaderComputeInputInfo compute{};
compute.threads_num[0] = 64;
compute.threads_num[1] = 1;
compute.threads_num[2] = 1;
compute.dispatch_threads_num[0] = 64;
compute.dispatch_threads_num[1] = 1;
compute.dispatch_threads_num[2] = 1;
compute.group_id[0] = true;
compute.dispatch_thread_dimensions = true;
compute.wave_size = 64;
compute.thread_ids_num = 1;
compute.workgroup_register = 8;
u32 scalar_clear = 0x40404040u;
bool clean_scalar = false;
const auto Compile = [&](const std::vector<u32> &code) {
ShaderRecompiler::CompileOptions options;
options.stage = ShaderType::Compute;
options.wave_size = 64;
options.user_data_base = 0;
options.user_data = user_data;
options.input_info.compute = &compute;
options.dump_ir = false;
auto translated = ShaderRecompiler::TranslateProgram(code, options);
auto resource_plan =
ShaderRecompiler::IR::ExtractResourcePlan(translated.program);
ShaderRecompiler::IR::ResourceSnapshot resources;
ShaderRecompiler::IR::ResourceSpecialization specialization;
const ShaderRecompiler::IR::SrtRuntime runtime{
.user_data = options.user_data,
.shader_base = reinterpret_cast<uint64_t>(code.data()),
.userdata = &scalar_clear,
.read_specialization_memory = clean_scalar
? +[](void *data, uint64_t address, std::span<uint32_t> words) {
if (address != reinterpret_cast<uint64_t>(data) || words.size() != 1u) return false;
words[0] = *static_cast<uint32_t *>(data);
return true;
} : nullptr,
};
Require("Ps5GameExampleImageClear", "resource materialization",
ShaderRecompiler::IR::MaterializeResources(
resource_plan, runtime, resources, specialization),
"runtime clear resources could not be materialized");
auto result = ShaderRecompiler::CompileProgram(
std::move(translated), options, specialization);
ValidateSpirv("Ps5GameExampleImageClear", result.spirv);
return resources;
};
const auto code = MakeCode();
auto positive = Compile(code);
auto runtime_resources = positive;
compute.stage.resources = &runtime_resources;
ShaderBufferResource descriptor{};
u32 packed_clear = 0;
uint64_t size = 0;
Require("Ps5GameExampleImageClear", "runtime shape",
ResolveComputeBufferFill(compute, 64, 1, 1, 0x61u, descriptor,
packed_clear, size) &&
descriptor.Base48() == 0x10000u && size == 64u * 16u &&
packed_clear == 0xff000000u,
"exact Prospero runtime binding did not resolve to a complete clear");
auto narrow_exec = code;
narrow_exec.insert(narrow_exec.end() - 3,
EncodeSop1(0x24u, 10, InlineU32(1))); // S_AND_SAVEEXEC_B64.
runtime_resources = Compile(narrow_exec);
Require("Ps5GameExampleImageClear", "guest EXEC narrowing",
!ResolveComputeBufferFill(compute, 64, 1, 1, 0x61u, descriptor,
packed_clear, size),
"a guest predicate narrowing the store was accepted as a complete clear");
auto wrong_index = code;
wrong_index[0] = (wrong_index[0] & ~(0x3ffu << 16u)) | (0x347u << 16u);
runtime_resources = Compile(wrong_index);
Require("Ps5GameExampleImageClear", "add-before-shift address",
!ResolveComputeBufferFill(compute, 64, 1, 1, 0x61u, descriptor,
packed_clear, size),
"V_ADD_LSHL was mistaken for V_LSHL_ADD contiguous coverage");
user_data[7] ^= 1u;
runtime_resources = Compile(code);
Require("Ps5GameExampleImageClear", "non-repeated clear",
!ResolveComputeBufferFill(compute, 64, 1, 1, 0x61u, descriptor,
packed_clear, size),
"non-uniform uint4 data was replaced with a color clear");
runtime_resources = positive;
compute.dispatch_threads_num[0] = 63;
Require("Ps5GameExampleImageClear", "mismatched requested extent",
!ResolveComputeBufferFill(compute, 64, 1, 1, 0x61u, descriptor,
packed_clear, size),
"a dispatch exceeding its requested extent was classified as a complete clear");
runtime_resources.buffers[0].dwords[2] = 65;
compute.dispatch_threads_num[0] = 65;
Require("Ps5GameExampleImageClear", "partial final workgroup",
!ResolveComputeBufferFill(compute, 65, 1, 1, 0x61u, descriptor,
packed_clear, size),
"a nondivisible thread dispatch was classified as a complete clear");
runtime_resources = positive;
compute.dispatch_threads_num[0] = 32;
Require("Ps5GameExampleImageClear", "partial dispatch",
!ResolveComputeBufferFill(compute, 32, 1, 1, 0x61u, descriptor,
packed_clear, size),
"partial buffer coverage was classified as a complete clear");
// Same descriptor/dispatch as GTA3, with the stored value encoded in the
// shader.
user_data = {0x00200000u, 4u << 16u, 64u, 0x14204u};
compute.workgroup_register = 4;
compute.dispatch_thread_dimensions = false;
std::vector<u32> scalar_code;
AppendVop3(&scalar_code, 0x346u, 0, 4, InlineU32(6), Vgpr(0));
scalar_code.push_back(EncodeVop1(0x01u, 1, InlineU32(0)));
scalar_code.push_back(EncodeMubuf0(0x04u, 0, true, false));
scalar_code.push_back(EncodeMubuf1(1, 0, 0));
AppendEnd(&scalar_code);
auto scalar = Compile(scalar_code);
runtime_resources = scalar;
Require("Ps5GameExampleImageClear", "inline scalar fill",
ResolveComputeBufferFill(compute, 1, 1, 1, 0x41u, descriptor,
packed_clear, size) &&
descriptor.Base48() == 0x200000u && size == 256u &&
packed_clear == 0u,
"GTA3's actual scalar store was not recognized from IR");
Require("Ps5GameExampleImageClear", "excess dispatch",
!ResolveComputeBufferFill(compute, 2, 1, 1, 0x41u, descriptor,
packed_clear, size),
"excess invocation coverage was accepted as a fill");
runtime_resources.uniform_fill.bounded_elements = 65;
runtime_resources.buffers[0].dwords[2] = 65;
for (u32 oob = 0; oob < 4; ++oob) {
runtime_resources.buffers[0].dwords[3] = 0x14204u | (oob << 28u);
Require("Ps5GameExampleImageClear", "bounded final workgroup and RDNA2 OOB mode",
ResolveComputeBufferFill(compute, 2, 1, 1, 0x41u, descriptor,
packed_clear, size) == (oob != 3),
"explicit coverage did not respect RDNA2 element bounds");
}
runtime_resources.buffers[0].dwords[3] = 0x14204u;
Require("Ps5GameExampleImageClear", "bounded insufficient dispatch",
!ResolveComputeBufferFill(compute, 1, 1, 1, 0x41u, descriptor,
packed_clear, size),
"a bound was mistaken for actual invocation coverage");
Require("Ps5GameExampleImageClear", "bounded index wrap",
!ResolveComputeBufferFill(compute, 0x04000001u, 1, 1, 0x41u, descriptor,
packed_clear, size),
"a wrapped invocation index was accepted as complete coverage");
runtime_resources.uniform_fill.bounded_elements = 64;
Require("Ps5GameExampleImageClear", "bounded incomplete destination",
!ResolveComputeBufferFill(compute, 2, 1, 1, 0x41u, descriptor,
packed_clear, size),
"a shader bound smaller than the destination was accepted as a fill");
runtime_resources = scalar;
auto alias = scalar.buffers.front();
runtime_resources.buffers.push_back(alias);
Require("Ps5GameExampleImageClear", "aliased scalar input",
!ResolveComputeBufferFill(compute, 1, 1, 1, 0x41u, descriptor,
packed_clear, size),
"a scalar read aliasing the destination was accepted as uniform");
runtime_resources = scalar;
runtime_resources.buffers[0].dwords[3] =
(static_cast<u32>(Prospero::BufferFormat::k32Float) << 12u) | 0x204u;
Require("Ps5GameExampleImageClear", "wrong store format",
!ResolveComputeBufferFill(compute, 1, 1, 1, 0x41u, descriptor,
packed_clear, size),
"a format that changes the stored value was accepted as a uint fill");
// A body change preserving every descriptor and dispatch field must change
// the result.
scalar_code[2] = EncodeVop1(0x01u, 1, InlineU32(7));
runtime_resources = Compile(scalar_code);
Require("Ps5GameExampleImageClear", "changed store value",
ResolveComputeBufferFill(compute, 1, 1, 1, 0x41u, descriptor,
packed_clear, size) &&
packed_clear == 7u,
"clear recognition ignored the shader's actual stored value");
scalar_code[2] = EncodeVop1(0x01u, 1, Vgpr(0));
runtime_resources = Compile(scalar_code);
Require("Ps5GameExampleImageClear", "varying store value",
!ResolveComputeBufferFill(compute, 1, 1, 1, 0x41u, descriptor,
packed_clear, size),
"a varying store with identical resources was treated as a fill");
const auto scalar_address = reinterpret_cast<uint64_t>(&scalar_clear);
user_data[4] = static_cast<u32>(scalar_address);
user_data[5] = static_cast<u32>(scalar_address >> 32u) | (16u << 16u);
user_data[6] = 1;
user_data[7] = 0x4dfacu;
compute.workgroup_register = 8;
scalar_code.clear();
AppendVop3(&scalar_code, 0x346u, 0, 8, InlineU32(6), Vgpr(0));
scalar_code.push_back(EncodeSmem0(0x08u, 106, 2));
scalar_code.push_back(EncodeSmem1(0, 125));
scalar_code.push_back(EncodeVop1(0x01u, 1, 106));
scalar_code.push_back(EncodeMubuf0(0x04u, 0, true, false));
scalar_code.push_back(EncodeMubuf1(1, 0, 0));
AppendEnd(&scalar_code);
clean_scalar = true;
auto loaded = Compile(scalar_code);
runtime_resources = loaded;
Require("Ps5GameExampleImageClear", "scalar-loaded fill",
ResolveComputeBufferFill(compute, 1, 1, 1, 0x41u, descriptor,
packed_clear, size) &&
packed_clear == scalar_clear && descriptor.Base48() == 0x200000u,
"GTA3's scalar input was confused with its destination");
clean_scalar = false;
runtime_resources = Compile(scalar_code);
Require("Ps5GameExampleImageClear", "unavailable clean scalar",
!ResolveComputeBufferFill(compute, 1, 1, 1, 0x41u, descriptor,
packed_clear, size),
"a scalar without a clean reader was replaced by a fill");
std::printf("[host] %-32s ok\n", "Ps5GameExampleImageClear");
}
void CheckTessellationProgram(const char *name, u32 ls_stride, u32 hs_stride) {
using namespace ShaderRecompiler;
const bool multiplied_stride = hs_stride == 112;
const u32 local_address = multiplied_stride ? 18 : 21;
const u32 control_point = multiplied_stride ? 30 : 34;
const u32 relative_patch = multiplied_stride ? 28 : 32;
const u32 output_address = multiplied_stride ? 31 : 35;
const u32 input_address = multiplied_stride ? 25 : 29;
std::array<std::vector<u32>, 3> code;
auto &local = code[0];
// The fused LS addresses one control point using its v3 ordinal.
local.push_back(EncodeVop2(0x0b, local_address, 255u, 3));
local.push_back(ls_stride);
local.push_back(EncodeDs0(0x0e, (3u << 8u) | 1u));
local.push_back(EncodeDs1Ex(0, 5, 2, local_address));
local.push_back(EncodeSop1(0x20, 0, 6));
local.push_back(0xffffffffu); // Shader footer is not executable after LS handoff.
auto &control = code[1];
AppendVop3(&control, 0x365, 0, 193u, InlineU32(0));
AppendVop3(&control, 0x366, 0, 193u, Vgpr(0));
control.push_back(EncodeSopp(0x0a));
control.push_back(EncodeVopc(0xd1, 249u, 3));
control.push_back(EncodeVopcSdwa(0, 0, 0, 6, 1, 0, 0, 0, 0, 0, 0, 0, 1));
AppendVop3(&control, 0x148, control_point, Vgpr(1), InlineU32(8),
InlineU32(5));
AppendVop3(&control, 0x148, relative_patch, Vgpr(1), InlineU32(0),
InlineU32(8));
if (multiplied_stride) {
// The later patch writes 112-byte outputs but reads 108-byte LS records.
// The second multiply overwrites packed v1; only memory consumers identify
// which of the two independently computed addresses supplies each stride.
control.push_back(EncodeVop2(0x0b, 0, 255u, control_point));
control.push_back(hs_stride);
control.push_back(EncodeVop2(0x0b, 1, 255u, control_point));
control.push_back(ls_stride);
AppendVop3(&control, 0x143, output_address, 255u, Vgpr(relative_patch),
Vgpr(0));
control.push_back(3 * hs_stride);
AppendVop3(&control, 0x143, input_address, 255u, Vgpr(relative_patch),
Vgpr(1));
control.push_back(3 * ls_stride);
} else {
control.push_back(EncodeSopk(0x00, 106, 3 * hs_stride));
control.push_back(EncodeVop2(0x0b, 2, 249u, 1));
control.push_back(EncodeVop2Sdwa(106, 6, 0, 6, 0, 0, 0, 0, 0, 0, 0, 1));
AppendVop3(&control, 0x346, output_address, Vgpr(control_point), InlineU32(7),
Vgpr(2));
control.push_back(EncodeVop2(0x0b, input_address, 255u, control_point));
control.push_back(ls_stride);
}
control.push_back(EncodeDs0(0x37, (3u << 8u) | 1u));
control.push_back(EncodeDs1Ex(10, 0, 0, input_address));
control.push_back(EncodeMubuf0(0x1d, 28));
control.push_back(EncodeMubuf1(10, 2, output_address, 2));
// Keep the control-point-dependent value and the last written dword live.
control.push_back(EncodeMubuf0(0x1c, hs_stride - 8));
control.push_back(EncodeMubuf1(0, 2, output_address, 2));
// The game reuses previously written data VGPRs for CP0-only ring addresses.
AppendVMovLiteral(&control, 4, 0x42280000u);
AppendVMovLiteral(&control, 5, 0x422c0000u);
control.push_back(EncodeVopc(0xd4, InlineU32(1), control_point));
control.push_back(EncodeVop2(0x1a, 4, InlineU32(4), relative_patch));
control.push_back(EncodeVop2(0x1a, 6, InlineU32(6), relative_patch));
control.push_back(EncodeVop2(0x26, 5, 255u, 6));
control.push_back(0x7fc0);
for (u32 i = 0; i < 4; i++) {
AppendVMovLiteral(&control, i, std::bit_cast<u32>(static_cast<float>(i + 1)));
}
control.push_back(EncodeMubuf0(0x1e));
control.push_back(EncodeMubuf1(0, 2, 4, 4));
for (u32 i = 0; i < 4; i++) {
control.push_back(EncodeMubuf0(0x1c, i * 16));
control.push_back(EncodeMubuf1(i, 2, 5, 2));
}
AppendEnd(&control);
auto &evaluation = code[2];
AppendVop3(&evaluation, 0x169, 24, 255u, Vgpr(7));
evaluation.push_back(3 * hs_stride);
evaluation.push_back(EncodeMubuf0(0x0d, 28));
evaluation.push_back(EncodeMubuf1(10, 2, 24, 4));
evaluation.push_back(EncodeMubuf0(0x0d, 2 * hs_stride + 28));
evaluation.push_back(EncodeMubuf1(12, 2, 24, 4));
evaluation.push_back(EncodeExp0(0x0c, 0xf));
evaluation.push_back(EncodeExp1(5, 6, 10, 12));
AppendEnd(&evaluation);
ShaderTessellationInputInfo tess{.input_control_points = 3,
.output_control_points = 3,
.domain = 1,
.partitioning = 2,
.output_topology = 2};
AnalyzeTessellationPrograms(local, control, tess);
Require(name, "decoded interface",
tess.ls_stride == ls_stride && tess.hs_stride == hs_stride,
"captured LS and HS address arithmetic must produce distinct strides");
constexpr std::array stages{ShaderType::Local, ShaderType::TessellationControl,
ShaderType::TessellationEvaluation};
for (u32 stage = 0; stage < stages.size(); stage++) {
ShaderVertexInputInfo vertex;
vertex.logical_stage = stages[stage];
vertex.tess = tess;
CompileOptions options;
options.stage = stages[stage];
options.input_info.vertex = &vertex;
auto translated = TranslateProgram(code[stage], options);
std::set<u32> local_offsets, evaluation_offsets, factor_offsets, patch_offsets;
for (const auto *block : translated.program.blocks) {
for (const auto &inst : *block) {
if (inst.GetOpcode() != IR::ValueOpcode::GetTessellationAttribute &&
inst.GetOpcode() != IR::ValueOpcode::SetTessellationAttribute) {
continue;
}
if (!inst.Arg(1).IsImmediate()) {
continue;
}
switch (static_cast<IR::TessellationAttribute>(inst.Arg(0).U32())) {
case IR::TessellationAttribute::LocalOutput:
local_offsets.insert(inst.Arg(1).U32());
break;
case IR::TessellationAttribute::EvaluationInput:
evaluation_offsets.insert(inst.Arg(1).U32());
break;
case IR::TessellationAttribute::Factor:
factor_offsets.insert(inst.Arg(1).U32());
break;
case IR::TessellationAttribute::PatchOutput:
patch_offsets.insert(inst.Arg(1).U32());
break;
default:
break;
}
}
}
Require(name, "separate DS offsets", stage != 0 || local_offsets == std::set<u32>{4, 12},
"DS_WRITE2 must preserve both independently encoded offsets");
Require(name, "offchip component addressing",
stage != 2 ||
evaluation_offsets == std::set<u32>{28, 2 * hs_stride + 28},
"TES control point and component offsets must survive lowering");
Require(name, "triangle factor layout",
stage != 1 || factor_offsets == std::set<u32>{0, 4, 8, 12},
"the four packed guest factors must retain their order");
Require(name, "predicated patch addresses",
stage != 1 || patch_offsets == std::set<u32>{0x7fc0, 0x7fd0, 0x7fe0, 0x7ff0},
"CP0 patch stores must use the address assigned on their active predicate");
auto result = CompileProgram(std::move(translated), options, {});
ValidateSpirv(name, result.spirv);
spvtools::SpirvTools tools(SPV_ENV_VULKAN_1_2);
std::string assembly;
Require(name, "disassembly", tools.Disassemble(result.spirv, &assembly),
"failed to disassemble tessellation shader");
Require(name, "native local inputs",
stage != 0 || (assembly.find("VertexIndex") != std::string::npos &&
assembly.find("InstanceIndex") != std::string::npos),
"LS v2/v5 must read native vertex/instance indices");
Require(name, "control point lane",
stage != 1 || (assembly.find("InvocationId") != std::string::npos &&
assembly.find("SubgroupLocalInvocationId") == std::string::npos),
"HS lane masks must use control point invocation IDs");
Require(name, "control point barrier",
stage != 1 || (assembly.find("OpMemoryModel Logical GLSL450") != std::string::npos &&
assembly.find("OpControlBarrier %uint_2 %uint_4 %uint_0") != std::string::npos),
"HS barriers must match GLSL450 control point synchronization semantics");
Require(name, "factor builtins",
stage != 1 || (assembly.find("TessLevelOuter") != std::string::npos &&
assembly.find("TessLevelInner") != std::string::npos),
"triangle factors require outer[4] and inner[2] Vulkan builtins");
Require(name, "native domain and winding",
stage != 2 || (assembly.find("TessCoord") != std::string::npos &&
assembly.find("Triangles") != std::string::npos &&
assembly.find("SpacingFractionalOdd") != std::string::npos &&
assembly.find("VertexOrderCw") != std::string::npos),
"TES must preserve guest triangle domain, partitioning, and winding");
}
std::printf("[host] %-32s ok\n", name);
}
void CheckTessellationPrograms() {
CheckTessellationProgram("TessellationShiftedStride", 124, 128);
CheckTessellationProgram("TessellationMultipliedStride", 108, 112);
}
void CheckEmbeddedFetchVertexOffset() {
const auto MakeFetch = [](std::initializer_list<std::pair<u32, u32>> adds,
std::optional<std::pair<u32, u32>> late_add = {},
u32 accumulator_vgpr = 0, bool ngg_sad = false,
bool local = false) {
std::vector<u32> code;
code.push_back(EncodeSMovB32(0, InlineU32(0)));
code.push_back(EncodeSmem0(0x02u, 20, 4));
code.push_back(EncodeSmem1(0));
const auto AppendOffsets = [&]() {
for (const auto [sgpr, index_vgpr] : adds) {
if (ngg_sad) {
AppendVop3(&code, 0x15du, accumulator_vgpr, sgpr, InlineU32(0),
Vgpr(index_vgpr));
} else {
AppendVop3B(&code, 0x30fu, accumulator_vgpr, 0, sgpr,
Vgpr(index_vgpr));
}
}
};
if (ngg_sad) {
AppendOffsets();
}
code.push_back(EncodeVop2(0x01u, 0, Vgpr(local ? 5 : 8), local ? 2 : 5));
if (!ngg_sad) {
AppendOffsets();
}
code.push_back(EncodeMubuf0(0x03u, 0, true));
code.push_back(EncodeMubuf1(9, 5, 0));
if (late_add.has_value()) {
AppendVop3B(&code, 0x30fu, 0, 0, late_add->first, Vgpr(late_add->second));
}
if (local) {
AppendVMovU32(&code, 20, 0);
code.push_back(EncodeDs0(0x0e, 1u << 8u));
code.push_back(EncodeDs1Ex(0, 10, 9, 20));
code.push_back(EncodeDs0(0x0e, (3u << 8u) | 2u));
code.push_back(EncodeDs1Ex(0, 12, 11, 20));
code.push_back(EncodeSop1(0x20, 0, 6));
code.push_back(0xffffffffu);
} else {
AppendEnd(&code);
}
return code;
};
const auto Compile = [&](const char *name, const std::vector<u32> &code,
u32 slot10, ShaderType stage = ShaderType::Vertex,
Prospero::BufferFormat format = Prospero::BufferFormat::k32_32_32_32Float) {
std::array<u32, 11> user_data{};
user_data[10] = slot10;
ShaderVertexInputInfo vertex;
vertex.logical_stage = stage;
vertex.tess = {.input_control_points = 3, .output_control_points = 3,
.ls_stride = 124, .hs_stride = 128, .domain = 1,
.partitioning = 2, .output_topology = 2};
vertex.fetch_embedded = true;
vertex.fetch_buffer_reg = 0;
vertex.fetch_attrib_reg = 2;
vertex.resources_num = 1;
vertex.resources[0].fields[3] =
BufferFormat(format) << 12u | DstSel(4, 5, 6, 7);
vertex.resources_dst[0].attr_id = 0;
vertex.resources_dst[0].registers_num = 4;
ShaderRecompiler::CompileOptions options;
options.stage = stage;
options.user_data_base = 8;
options.user_data = user_data;
options.input_info.vertex = &vertex;
auto translated = ShaderRecompiler::TranslateProgram(code, options);
auto resource_plan =
ShaderRecompiler::IR::ExtractResourcePlan(translated.program);
ShaderRecompiler::IR::ResourceSnapshot resources;
ShaderRecompiler::IR::ResourceSpecialization specialization;
const ShaderRecompiler::IR::SrtRuntime runtime{
.user_data = options.user_data,
.shader_base = reinterpret_cast<uint64_t>(code.data()),
};
Require(name, "resource materialization",
ShaderRecompiler::IR::MaterializeResources(
resource_plan, runtime, resources, specialization),
"embedded fetch resources could not be materialized");
auto result = ShaderRecompiler::CompileProgram(
std::move(translated), options, specialization);
Require(name, "fetch rewrite",
result.program.info.vertex_fetch_components[0] == 4,
"encoded fetch sequence was not recognized and rewritten");
return CompiledShader{std::move(result.spirv), std::move(result.program),
std::move(resources), {}};
};
const auto Resolve = [](const CompiledShader &result,
u32 index_offset) {
ShaderVertexInputInfo vertex;
vertex.fetch_embedded = true;
ShaderRecompiler::IR::CompiledShaderInfo program{};
program.user_data_base = result.program.user_data_base;
program.info.vertex_offset_sgpr = result.program.info.vertex_offset_sgpr;
vertex.stage.program = &program;
vertex.stage.resources = &result.resources;
return ResolveDrawOffsets(index_offset, vertex).first;
};
const auto ResolveInstance = [](const CompiledShader &result) {
ShaderVertexInputInfo vertex;
vertex.fetch_embedded = true;
ShaderRecompiler::IR::CompiledShaderInfo program{};
program.user_data_base = result.program.user_data_base;
program.info.instance_offset_sgpr =
result.program.info.instance_offset_sgpr;
vertex.stage.program = &program;
vertex.stage.resources = &result.resources;
return ResolveDrawOffsets(0, vertex).second;
};
for (const auto format : {Prospero::BufferFormat::k32_32_32_32Float,
Prospero::BufferFormat::k32_32_32_32UInt}) {
const auto local = Compile("LocalEmbeddedFetch", MakeFetch({}, {}, 0, false, true),
0, ShaderType::Local, format);
ValidateSpirv("LocalEmbeddedFetch", local.spirv);
Require("LocalEmbeddedFetch", "live vertex attributes",
std::ranges::any_of(local.program.info.inputs, [](const auto &input) {
return input.kind == ShaderRecompiler::IR::StageInputKind::Parameter &&
input.location == 0;
}), "LS indexed fetch must consume native vertex attributes");
if (format == Prospero::BufferFormat::k32_32_32_32UInt) {
spvtools::SpirvTools tools(SPV_ENV_VULKAN_1_2);
std::string assembly;
Require("LocalEmbeddedFetch", "disassembly", tools.Disassemble(local.spirv, &assembly),
"failed to disassemble unsigned LS fetch");
Require("LocalEmbeddedFetch", "unsigned input type",
assembly.find("OpTypePointer Input %v4uint") != std::string::npos &&
assembly.find("OpConvertUToF") == std::string::npos,
"unsigned LS attributes must preserve their native integer bits");
}
}
const auto valid =
Compile("EmbeddedFetchVertexOffset", MakeFetch({{18, 0}}), 7);
Require(
"EmbeddedFetchVertexOffset", "parse",
valid.program.info.vertex_offset_sgpr == 18 && Resolve(valid, 0) == 7 &&
Resolve(valid, 5) == 5,
"canonical fetch offset or register index-offset precedence is wrong");
const auto ngg_code = MakeFetch({{18, 5}}, {}, 5, true);
Require("EmbeddedFetchNggVertexOffset", "encoding",
ngg_code[3] == 0xd55d0005u && ngg_code[4] == 0x04150012u,
"test does not encode the PS5 V_SAD_U32 vertex-offset prolog");
const auto ngg = Compile("EmbeddedFetchNggVertexOffset", ngg_code, 8);
Require("EmbeddedFetchNggVertexOffset", "parse",
ngg.program.info.vertex_offset_sgpr == 18 && Resolve(ngg, 0) == 8 &&
Resolve(ngg, 5) == 5,
"PS5 NGG vertex-index offset or register index-offset precedence is "
"wrong");
const auto ngg_instance =
Compile("EmbeddedFetchNggInstanceOffset",
MakeFetch({{18, 8}}, {}, 8, true), 81);
Require("EmbeddedFetchNggInstanceOffset", "parse",
ngg_instance.program.info.vertex_offset_sgpr == -1 &&
ngg_instance.program.info.instance_offset_sgpr == 18 &&
ResolveInstance(ngg_instance) == 81,
"PS5 NGG instance-index offset was not detected or resolved");
const auto instance_conflict = Compile(
"EmbeddedFetchConflictingInstanceOffset",
MakeFetch({{17, 8}, {18, 8}}, {}, 8, true), 81);
const auto instance_malformed =
Compile("EmbeddedFetchMalformedInstanceOffset",
MakeFetch({{18, 7}}, {}, 7, true), 81);
const auto instance_outside =
Compile("EmbeddedFetchOutsideInstanceOffset",
MakeFetch({{19, 8}}, {}, 8, true), 81);
for (const auto *result :
{&instance_conflict, &instance_malformed, &instance_outside}) {
Require("EmbeddedFetchNggInstanceOffset", "fail closed",
result->program.info.instance_offset_sgpr == -1 &&
ResolveInstance(*result) == 0,
"conflicting, malformed, or out-of-window instance add was "
"classified");
}
const auto pointer = 0x5b7c5100u;
const auto late = Compile("EmbeddedFetchLateOffset",
MakeFetch({}, std::pair<u32, u32>{18, 0}), pointer);
const auto conflict = Compile("EmbeddedFetchConflictingOffset",
MakeFetch({{17, 0}, {18, 0}}), pointer);
const auto malformed =
Compile("EmbeddedFetchMalformedOffset", MakeFetch({{18, 1}}), pointer);
const auto outside =
Compile("EmbeddedFetchOutsideOffset", MakeFetch({{19, 0}}), pointer);
for (const auto *result : {&late, &conflict, &malformed, &outside}) {
Require("EmbeddedFetchVertexOffset", "fail closed",
result->program.info.vertex_offset_sgpr == -1 &&
Resolve(*result, 0) == 0,
"non-prolog, conflicting, malformed, or out-of-window add was "
"classified");
}
std::printf("[host] %-32s ok\n", "EmbeddedFetchVertexOffset");
}
[[noreturn]] void RunReverseRenderTargetDeathCase() {
(void)TextureGetRenderTargetFormat(Prospero::ChannelLayout::k16_16_16_16,
Prospero::ChannelType::kSrgb,
Prospero::ChannelOrder::kStandard);
std::_Exit(0x7f);
}
#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
void CheckRenderTargetFormatContract() {
const auto r8_uint = TextureGetRenderTargetFormat(
Prospero::ChannelLayout::k8, Prospero::ChannelType::kUInt,
Prospero::ChannelOrder::kStandard);
Require("RenderTargetFormat", "R8 uint",
r8_uint.format == vk::Format::eR8Uint &&
r8_uint.bytes_per_element == 1u &&
r8_uint.export_mapping.IsIdentity(),
"R8 uint render-target tuple was rejected");
const auto rgb565 = TextureGetRenderTargetFormat(
Prospero::ChannelLayout::k5_6_5, Prospero::ChannelType::kUNorm,
Prospero::ChannelOrder::kStandard);
Require("RenderTargetFormat", "RGB565 UNorm",
rgb565.format == vk::Format::eB5G6R5UnormPack16 &&
rgb565.bytes_per_element == 2u &&
rgb565.export_mapping.IsIdentity(),
"RGB565 UNorm render-target tuple was rejected");
const auto uint_format = TextureGetRenderTargetFormat(
Prospero::ChannelLayout::k16_16_16_16, Prospero::ChannelType::kUInt,
Prospero::ChannelOrder::kStandard);
Require("RenderTargetFormat", "RGBA16 uint",
uint_format.format == vk::Format::eR16G16B16A16Uint &&
uint_format.bytes_per_element == 8u &&
uint_format.export_mapping.IsIdentity(),
"RGBA16 uint render-target tuple was rejected");
const auto format = TextureGetRenderTargetFormat(
Prospero::ChannelLayout::k16_16_16_16, Prospero::ChannelType::kFloat,
Prospero::ChannelOrder::kReversed);
Require("ReverseRenderTarget", "exact format",
format.format == vk::Format::eR16G16B16A16Sfloat &&
format.bytes_per_element == 8u &&
format.export_mapping == Prospero::ColorMappingAbgr,
"exact reverse RGBA16F render-target tuple was rejected");
Require("ReverseRenderTarget", "write masks",
format.export_mapping.ApplyMask(0x1u) == 0x8u &&
format.export_mapping.ApplyMask(0x2u) == 0x4u &&
format.export_mapping.ApplyMask(0x4u) == 0x2u &&
format.export_mapping.ApplyMask(0x8u) == 0x1u &&
format.export_mapping.ApplyMask(0xfu) == 0xfu,
"reverse RGBA16F component mask was not mapped exactly once");
const auto rg32 = TextureGetRenderTargetFormat(
Prospero::ChannelLayout::k32_32, Prospero::ChannelType::kFloat,
Prospero::ChannelOrder::kReversed);
Require(
"ReverseRenderTarget", "GR32 float",
rg32.format == vk::Format::eR32G32Sfloat &&
rg32.bytes_per_element == 8u &&
rg32.export_mapping == Prospero::ColorMappingGr &&
rg32.export_mapping.ApplyMask(0x1u) == 0x2u &&
rg32.export_mapping.ApplyMask(0x2u) == 0x1u &&
rg32.export_mapping.ApplyMask(0x3u) == 0x3u &&
rg32.export_mapping.ApplyMask(0xfu) == 0xfu,
"reverse GR32F render-target export or write-mask mapping is incorrect");
const auto argb = TextureGetRenderTargetFormat(
Prospero::ChannelLayout::k16_16_16_16, Prospero::ChannelType::kFloat,
Prospero::ChannelOrder::kAltReversed);
Require(
"ReverseRenderTarget", "ARGB float",
argb.format == vk::Format::eR16G16B16A16Sfloat &&
argb.export_mapping == Prospero::ColorMappingArgb &&
argb.export_mapping.ApplyMask(0x1u) == 0x2u &&
argb.export_mapping.ApplyMask(0x2u) == 0x4u &&
argb.export_mapping.ApplyMask(0x4u) == 0x8u &&
argb.export_mapping.ApplyMask(0x8u) == 0x1u,
"alternate-reversed render-target mapping did not invert its ARGB cycle");
char path[MAX_PATH]{};
Require("ReverseRenderTarget", "host",
GetModuleFileNameA(nullptr, path, MAX_PATH) != 0,
"GetModuleFileName failed");
std::string command = std::string("\"") + path + "\" --reverse-rt-death";
std::vector<char> mutable_command(command.begin(), command.end());
mutable_command.push_back('\0');
STARTUPINFOA startup{sizeof(startup)};
PROCESS_INFORMATION process{};
Require("ReverseRenderTarget", "host",
CreateProcessA(nullptr, mutable_command.data(), nullptr, nullptr,
FALSE, CREATE_NO_WINDOW, nullptr, nullptr, &startup,
&process) != 0,
"CreateProcess failed");
Require("ReverseRenderTarget", "host",
WaitForSingleObject(process.hProcess, 10000) == WAIT_OBJECT_0,
"unsupported adjacent render-target tuple timed out");
DWORD exit_code = 0;
const bool exited = GetExitCodeProcess(process.hProcess, &exit_code) != 0;
CloseHandle(process.hThread);
CloseHandle(process.hProcess);
Require("ReverseRenderTarget", "hard failure", exited && exit_code == 321,
"invalid render-target tuple did not retain the fatal guard");
std::printf("[host] %-32s ok\n", "RenderTargetFormat");
}
#endif
[[noreturn]] void RunImageViewDeathCase(const char *kind) {
if (std::strcmp(kind, "sampled-invalid-selector") == 0) {
(void)SelectSampledColorView(vk::Format::eR8G8B8A8Unorm,
vk::Format::eR8G8B8A8Unorm,
DstSel(4, 5, 6, 2));
} else if (std::strcmp(kind, "sampled-incompatible-format") == 0) {
(void)SelectSampledColorView(vk::Format::eR8Unorm, vk::Format::eR16Unorm,
DstSel(4, 0, 0, 1));
} else if (std::strcmp(kind, "sampled-invalid-high") == 0) {
(void)SelectSampledColorView(vk::Format::eR8G8B8A8Unorm,
vk::Format::eR8G8B8A8Unorm,
DstSel(7, 6, 5, 3));
} else if (std::strcmp(kind, "storage-incompatible-format") == 0) {
ValidateStorageColorView(vk::Format::eR8G8B8A8Srgb,
vk::Format::eR16G16B16A16Unorm,
DstSel(4, 5, 6, 7));
} else if (std::strcmp(kind, "volume-mip-count") == 0 ||
std::strcmp(kind, "volume-slice-range") == 0) {
VulkanHarness vulkan;
auto &graphics = vulkan.RuntimeContext();
CommandScheduler scheduler(vulkan.RuntimeRenderer(), graphics);
ImageInfo volume_info{};
volume_info.pixel_format = vk::Format::eR8G8B8A8Unorm;
volume_info.guest_format = Prospero::BufferFormat::k8_8_8_8UNorm;
volume_info.type = Prospero::ImageType::kColor3D;
volume_info.extent = {8, 8, 4};
volume_info.resources = {2, 1};
volume_info.pitch = 8;
volume_info.bytes_per_block = 4;
volume_info.samples = 1;
volume_info.tile_mode = Prospero::TileMode::kLinear;
Libs::Graphics::Image volume(graphics, scheduler, volume_info);
ImageViewInfo view{};
view.format = volume_info.pixel_format;
view.type = vk::ImageViewType::e2DArray;
view.aspect = vk::ImageAspectFlagBits::eColor;
view.base_level = 1;
view.level_count = 1;
view.layer_count = 1;
view.usage = vk::ImageUsageFlagBits::eSampled;
if (std::strcmp(kind, "volume-mip-count") == 0) {
view.base_level = 0;
view.level_count = 2;
} else {
view.base_layer = 1;
view.layer_count = 2;
}
(void)volume.FindView(view);
} else {
ShaderRecompiler::IR::ImageResource resource{};
resource.resource_class = ShaderRecompiler::IR::ImageResourceClass::Storage;
resource.numeric_class = Prospero::TextureNumericClass::Float;
resource.dimension = ShaderRecompiler::Decoder::ImageDimension::Dim2D;
resource.written = true;
if (std::strcmp(kind, "storage-kind") == 0) {
resource.resource_class =
ShaderRecompiler::IR::ImageResourceClass::Sampled;
} else if (std::strcmp(kind, "storage-no-write") == 0) {
resource.written = false;
} else if (std::strcmp(kind, "storage-nonuint-atomic") == 0) {
resource.atomic = true;
} else if (std::strcmp(kind, "storage-compare") == 0) {
resource.depth_compare = true;
} else if (std::strcmp(kind, "storage-dimension") == 0) {
resource.dimension = ShaderRecompiler::Decoder::ImageDimension::Unknown;
} else {
std::_Exit(0x7e);
}
ValidateStorageImageResource(resource);
}
std::_Exit(0x7f);
}
#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
void CheckSampledColorViews() {
Require("SampledColorViews", "identity",
SelectSampledColorView(vk::Format::eR8G8B8A8Unorm,
vk::Format::eR8G8B8A8Unorm,
DstSel(4, 5, 6, 7)) == DstSel(4, 5, 6, 7),
"RGBA did not select the identity view");
uint32_t valid_swizzles = 0;
for (uint32_t swizzle = 0; swizzle <= 0xfffu; swizzle++) {
bool expected = true;
for (uint32_t channel = 0; channel < 4; channel++) {
const auto selector = GetDstSel(swizzle, channel);
if (selector == 2 || selector == 3) {
expected = false;
}
}
const bool valid = IsValidImageSwizzle(swizzle);
const bool supported = IsSupportedSampledColorView(
vk::Format::eR8G8B8A8Unorm, vk::Format::eR8G8B8A8Unorm, swizzle);
Require("SampledColorViews", "exhaustive read swizzle domain",
valid == expected && supported == expected,
"sampled swizzle validator disagreed with the PS5 selector domain");
valid_swizzles += valid;
}
Require("SampledColorViews", "all PS5 read swizzles",
valid_swizzles == 1296 && !IsValidImageSwizzle(DstSel(4, 5, 6, 2)) &&
!IsValidImageSwizzle(DstSel(4, 5, 6, 3)) &&
!IsValidImageSwizzle(0x1000),
"valid PS5 sampled mappings were rejected or reserved selectors were "
"admitted");
const auto arbitrary = DstSel(5, 1, 7, 0);
const auto components = TextureGetComponentMapping(arbitrary, {});
Require("SampledColorViews", "generic Vulkan component mapping",
SelectSampledColorView(vk::Format::eR8G8B8A8Unorm,
vk::Format::eR8G8B8A8Unorm,
arbitrary) == arbitrary &&
components.r == vk::ComponentSwizzle::eG &&
components.g == vk::ComponentSwizzle::eOne &&
components.b == vk::ComponentSwizzle::eA &&
components.a == vk::ComponentSwizzle::eZero,
"arbitrary valid sampled mapping did not use the generic view path");
Require("SampledColorViews", "R8 R001",
SelectSampledColorView(vk::Format::eR8Unorm, vk::Format::eR8Unorm,
DstSel(4, 0, 0, 1)) == DstSel(4, 0, 0, 1),
"R8 did not select its R001 view");
Require("SampledColorViews", "R8 000R",
SelectSampledColorView(vk::Format::eR8Unorm, vk::Format::eR8Unorm,
DstSel(0, 0, 0, 4)) == DstSel(0, 0, 0, 4),
"R8 did not select its 000R component-mapped view");
Require("SampledColorViews", "mutable R8 uint/unorm 000R",
SelectSampledColorView(vk::Format::eR8Uint, vk::Format::eR8Unorm,
DstSel(0, 0, 0, 4)) == DstSel(0, 0, 0, 4),
"compatible R8 integer target sampled view was rejected");
Require("SampledColorViews", "R16G16 RG01",
SelectSampledColorView(vk::Format::eR16G16Sfloat,
vk::Format::eR16G16Sfloat,
DstSel(4, 5, 0, 1)) == DstSel(4, 5, 0, 1),
"R16G16 did not select its RG01 view");
Require("SampledColorViews", "alpha one",
SelectSampledColorView(vk::Format::eR8G8B8A8Unorm,
vk::Format::eR8G8B8A8Unorm,
DstSel(4, 5, 6, 1)) == DstSel(4, 5, 6, 1),
"RGB1 did not select the alpha-one view");
Require("SampledColorViews", "mutable BGRA target",
SelectSampledColorView(vk::Format::eB8G8R8A8Unorm,
vk::Format::eR8G8B8A8Unorm,
DstSel(6, 5, 4, 7)) == DstSel(6, 5, 4, 7),
"BGRA target did not select the exact RGBA/BGRA mutable view");
Require(
"SampledColorViews", "mutable sRGB view of UNORM BGRA target",
SelectSampledColorView(vk::Format::eB8G8R8A8Unorm,
vk::Format::eR8G8B8A8Srgb,
DstSel(6, 5, 4, 7)) == DstSel(6, 5, 4, 7),
"UNORM BGRA target did not select its compatible sRGB RGBA sampled view");
Require("SampledColorViews", "mutable sRGB BGRA target",
SelectSampledColorView(vk::Format::eB8G8R8A8Srgb,
vk::Format::eR8G8B8A8Srgb,
DstSel(6, 5, 4, 7)) == DstSel(6, 5, 4, 7),
"sRGB BGRA target did not select its matching mutable RGBA view");
constexpr auto colorspace_swizzle = DstSel(5, 1, 7, 0);
Require("SampledColorViews", "mutable sRGB/UNORM views",
SelectSampledColorView(vk::Format::eR8G8B8A8Srgb,
vk::Format::eR8G8B8A8Unorm,
DstSel(4, 5, 6, 7)) == DstSel(4, 5, 6, 7) &&
SelectSampledColorView(vk::Format::eB8G8R8A8Unorm,
vk::Format::eB8G8R8A8Srgb,
colorspace_swizzle) == colorspace_swizzle,
"same-order mutable sRGB/UNORM sampled views were rejected");
Require("SampledColorViews", "mutable packed RGB10 view",
SelectSampledColorView(vk::Format::eA2R10G10B10UnormPack32,
vk::Format::eA2B10G10R10UnormPack32,
DstSel(6, 5, 4, 7)) == DstSel(6, 5, 4, 7),
"packed RGB10 target did not select its matching mutable "
"channel-order view");
Require(
"SampledColorViews", "reverse RGBA16F sampled view",
SelectSampledColorView(vk::Format::eR16G16B16A16Sfloat,
vk::Format::eR16G16B16A16Sfloat,
DstSel(7, 6, 5, 4)) == DstSel(7, 6, 5, 4),
"reverse RGBA16F target did not select its reciprocal ABGR sampled view");
Require("SampledColorViews", "mutable integer-class views",
SelectSampledColorView(vk::Format::eR16G16B16A16Sfloat,
vk::Format::eR16G16B16A16Uint,
DstSel(4, 5, 6, 7)) == DstSel(4, 5, 6, 7) &&
SelectSampledColorView(
vk::Format::eR8G8B8A8Unorm, vk::Format::eR8G8B8A8Uint,
DstSel(4, 5, 6, 7)) == DstSel(4, 5, 6, 7) &&
SelectSampledColorView(vk::Format::eR8G8B8A8Uint,
vk::Format::eR8G8B8A8Unorm,
DstSel(4, 5, 6, 7)) == DstSel(4, 5, 6, 7),
"compatible integer render-target sampled view was rejected");
Require("SampledColorViews", "invalid depth format",
!IsSupportedSampledDepthView(vk::Format::eD24UnormS8Uint,
vk::Format::eR8Unorm,
DstSel(4, 4, 4, 4)),
"incompatible sampled depth format was accepted");
Require("SampledColorViews", "invalid depth swizzle",
!IsSupportedSampledDepthView(vk::Format::eD32SfloatS8Uint,
vk::Format::eR32Sfloat,
DstSel(4, 5, 6, 7)),
"incompatible sampled depth swizzle was accepted");
Require("SampledColorViews", "D32 depth target",
IsSupportedSampledDepthView(vk::Format::eD32SfloatS8Uint,
vk::Format::eR32Sfloat,
DstSel(4, 4, 4, 4)),
"D32 depth target did not select its depth-aspect view");
Require("SampledColorViews", "D16 R000 depth target",
IsSupportedSampledDepthView(
vk::Format::eD16Unorm, vk::Format::eR16Unorm, DstSel(4, 0, 0, 0)),
"D16 depth target did not select its R000 depth-aspect view");
Require("SampledColorViews", "D16 RRR1 depth target",
IsSupportedSampledDepthView(
vk::Format::eD16Unorm, vk::Format::eR16Unorm, DstSel(4, 4, 4, 1)),
"D16 depth target rejected replicated depth with constant alpha");
Require("SampledColorViews", "D16S8 R001 depth target",
IsSupportedSampledDepthView(vk::Format::eD16UnormS8Uint,
vk::Format::eR16Unorm,
DstSel(4, 0, 0, 1)),
"D16S8 depth target did not select its R001 depth-aspect view");
Require("SampledColorViews", "promoted D24S8 R001 depth target",
IsSupportedSampledDepthView(vk::Format::eD24UnormS8Uint,
vk::Format::eR16Unorm,
DstSel(4, 0, 0, 1)),
"D24S8 host fallback did not preserve the guest R16 depth view");
Require("SampledColorViews", "promoted D32S8 R001 depth target",
IsSupportedSampledDepthView(vk::Format::eD32SfloatS8Uint,
vk::Format::eR16Unorm,
DstSel(4, 0, 0, 1)),
"D32S8 host fallback did not preserve the guest R16 depth view");
Require("SampledColorViews", "D32S8 R001 depth target",
IsSupportedSampledDepthView(vk::Format::eD32SfloatS8Uint,
vk::Format::eR32Sfloat,
DstSel(4, 0, 0, 1)),
"D32S8 depth target did not select its R001 depth-aspect view");
ShaderRecompiler::IR::ImageResource storage_resource{};
storage_resource.resource_class =
ShaderRecompiler::IR::ImageResourceClass::Storage;
storage_resource.numeric_class = Prospero::TextureNumericClass::Float;
storage_resource.dimension = ShaderRecompiler::Decoder::ImageDimension::Dim2D;
storage_resource.written = true;
Require("SampledColorViews", "storage resource",
IsSupportedStorageImageResource(storage_resource),
"exact storage resource contract was rejected");
storage_resource.dimension = ShaderRecompiler::Decoder::ImageDimension::Dim3D;
storage_resource.read = true;
Require("SampledColorViews", "read-write 3D storage resource",
IsSupportedStorageImageResource(storage_resource),
"basic read-write 3D storage resource was rejected");
storage_resource.dimension =
ShaderRecompiler::Decoder::ImageDimension::Dim2DArray;
storage_resource.read = false;
Require("SampledColorViews", "write-only 2D-array storage resource",
IsSupportedStorageImageResource(storage_resource),
"basic write-only 2D-array storage resource was rejected");
storage_resource.numeric_class = Prospero::TextureNumericClass::Uint;
Require("SampledColorViews", "write-only uint 2D-array storage resource",
IsSupportedStorageImageResource(storage_resource),
"basic write-only uint 2D-array storage resource was rejected");
storage_resource.dimension = ShaderRecompiler::Decoder::ImageDimension::Dim2D;
storage_resource.read = true;
storage_resource.atomic = true;
Require("SampledColorViews", "atomic uint 2D storage resource",
IsSupportedStorageImageResource(storage_resource),
"atomic uint storage resource was rejected");
storage_resource.atomic = false;
storage_resource.mip_mode =
ShaderRecompiler::IR::ImageMipMode::Dynamic;
storage_resource.mip_count = 3;
Require("SampledColorViews", "dynamic storage mip resource",
IsSupportedStorageImageResource(storage_resource),
"dynamic storage mip resource was rejected");
char path[MAX_PATH]{};
Require("SampledColorViews", "host",
GetModuleFileNameA(nullptr, path, MAX_PATH) != 0,
"GetModuleFileName failed");
for (const char *kind :
{"sampled-invalid-selector", "sampled-incompatible-format",
"sampled-invalid-high", "storage-incompatible-format", "storage-kind",
"storage-no-write", "storage-nonuint-atomic", "storage-compare",
"storage-dimension", "volume-mip-count", "volume-slice-range"}) {
std::string command =
std::string("\"") + path + "\" --image-view-death " + kind;
std::vector<char> mutable_command(command.begin(), command.end());
mutable_command.push_back('\0');
STARTUPINFOA startup{sizeof(startup)};
PROCESS_INFORMATION process{};
Require("SampledColorViews", "host",
CreateProcessA(nullptr, mutable_command.data(), nullptr, nullptr,
FALSE, CREATE_NO_WINDOW, nullptr, nullptr, &startup,
&process) != 0,
"CreateProcess failed");
Require("SampledColorViews", "host",
WaitForSingleObject(process.hProcess, 10000) == WAIT_OBJECT_0,
"unsupported view death case timed out");
DWORD exit_code = 0;
const bool exited = GetExitCodeProcess(process.hProcess, &exit_code) != 0;
CloseHandle(process.hThread);
CloseHandle(process.hProcess);
Require("SampledColorViews", "host", exited && exit_code == 321,
std::string(kind) +
" component mapping did not report a fatal error");
}
std::printf("[host] %-32s ok\n", "SampledColorRenderTargetViews");
}
#endif
void CheckSampledDepthResource() {
ShaderRecompiler::IR::ImageResource resource{};
resource.resource_class = ShaderRecompiler::IR::ImageResourceClass::Sampled;
resource.numeric_class = Prospero::TextureNumericClass::Float;
resource.dimension = ShaderRecompiler::Decoder::ImageDimension::Dim2D;
resource.read = true;
resource.depth_compare = true;
Require("SampledDepthResource", "comparison read",
IsSupportedSampledDepthResource(resource),
"basic comparison-sampled depth resource was rejected");
resource.depth_compare = false;
Require("SampledDepthResource", "ordinary read",
IsSupportedSampledDepthResource(resource),
"basic non-comparison depth read was rejected");
const auto basic = resource;
resource.read = false;
Require("SampledDepthResource", "read required",
!IsSupportedSampledDepthResource(resource),
"non-reading depth resource was accepted");
resource = basic;
resource.written = true;
Require("SampledDepthResource", "write rejected",
!IsSupportedSampledDepthResource(resource),
"writable depth resource was accepted");
resource = basic;
resource.atomic = true;
Require("SampledDepthResource", "atomic rejected",
!IsSupportedSampledDepthResource(resource),
"atomic depth resource was accepted");
resource = basic;
resource.mip_mode = ShaderRecompiler::IR::ImageMipMode::Dynamic;
resource.mip_count = 3;
Require("SampledDepthResource", "dynamic mip read",
IsSupportedSampledDepthResource(resource),
"dynamic sampled depth mip resource was rejected");
resource.depth_compare = true;
Require("SampledDepthResource", "dynamic comparison mip read",
IsSupportedSampledDepthResource(resource),
"dynamic comparison depth mip resource was rejected");
resource = basic;
resource.dimension = ShaderRecompiler::Decoder::ImageDimension::Dim2DArray;
Require("SampledDepthResource", "singleton array accepted",
IsSupportedSampledDepthResource(resource),
"array depth resource was rejected");
resource = basic;
resource.numeric_class = Prospero::TextureNumericClass::Uint;
Require("SampledDepthResource", "integer read",
IsSupportedSampledDepthResource(resource),
"read-only uint depth resource was rejected");
resource.depth_compare = true;
Require("SampledDepthResource", "integer comparison rejected",
!IsSupportedSampledDepthResource(resource),
"uint depth resource accepted comparison sampling");
Require("SampledDepthResource", "R32 uint depth view",
IsSupportedSampledDepthFormat(vk::Format::eD32Sfloat,
vk::Format::eR32Uint) &&
!IsSupportedSampledDepthFormat(vk::Format::eR32Uint,
vk::Format::eR32Uint),
"R32 uint depth-view compatibility diverged from its backing role");
std::printf("[host] %-32s ok\n", "SampledDepthResource");
}
void CheckImageTransitionState(RenderContext &renderer) {
constexpr const char *name = "ImageTransitionState";
auto &context = renderer.GetGraphics();
CommandScheduler scheduler(renderer, context);
const auto MakeInfo = [](vk::Format format, uint32_t levels,
uint32_t layers) {
ImageInfo info{};
info.pixel_format = format;
info.guest_format = Prospero::BufferFormat::k8UNorm;
info.type = Prospero::ImageType::kColor2D;
info.extent = {4, 4, 1};
info.resources = {levels, layers};
info.pitch = 4;
info.bytes_per_block = 1;
info.samples = 1;
info.tile_mode = Prospero::TileMode::kLinear;
return info;
};
const auto graphics_stage = vk::PipelineStageFlagBits2::eAllGraphics |
vk::PipelineStageFlagBits2::eComputeShader;
constexpr uint64_t copy_capacity = 128ull << 20;
Require(
name, "buffered-copy capacity bands",
ImageTestAccess::CopyRows(32ull << 10, 4096, copy_capacity) == 4096 &&
ImageTestAccess::CopyRows(32ull << 10, 4097, copy_capacity) == 4096 &&
ImageTestAccess::CopyRows(copy_capacity + 4, 1, copy_capacity) == 0,
"buffered image copy does not split at the fixed scratch capacity");
Image image(context, scheduler, MakeInfo(vk::Format::eR8Unorm, 2, 3));
auto barriers =
image.GetBarriers(vk::ImageLayout::eShaderReadOnlyOptimal,
vk::AccessFlagBits2::eShaderRead, graphics_stage, {});
Require(
name, "initial full state",
barriers.size() == 1 &&
barriers[0].srcStageMask ==
vk::PipelineStageFlagBits2::eAllCommands &&
barriers[0].srcAccessMask == vk::AccessFlagBits2::eNone &&
barriers[0].dstStageMask == graphics_stage &&
barriers[0].dstAccessMask == vk::AccessFlagBits2::eShaderRead &&
barriers[0].oldLayout == vk::ImageLayout::eUndefined &&
barriers[0].newLayout == vk::ImageLayout::eShaderReadOnlyOptimal &&
barriers[0].subresourceRange.aspectMask ==
vk::ImageAspectFlagBits::eColor &&
barriers[0].subresourceRange.levelCount == VK_REMAINING_MIP_LEVELS &&
barriers[0].subresourceRange.layerCount ==
VK_REMAINING_ARRAY_LAYERS &&
image.backing.state.layout ==
vk::ImageLayout::eShaderReadOnlyOptimal &&
image.backing.state.access_mask == vk::AccessFlagBits2::eShaderRead &&
image.backing.state.pl_stage == graphics_stage,
"initial transition did not record or retain pinned state");
Require(name, "repeat read elision",
image
.GetBarriers(vk::ImageLayout::eShaderReadOnlyOptimal,
vk::AccessFlagBits2::eShaderRead, graphics_stage, {})
.empty(),
"identical read-only transition emitted a barrier");
barriers = image.GetBarriers(vk::ImageLayout::eShaderReadOnlyOptimal,
vk::AccessFlagBits2::eShaderRead |
vk::AccessFlagBits2::eTransferRead,
graphics_stage, {});
Require(name, "access-only transition",
barriers.size() == 1 &&
barriers[0].oldLayout == barriers[0].newLayout &&
barriers[0].srcAccessMask == vk::AccessFlagBits2::eShaderRead,
"same-layout access change was lost");
const auto CheckRepeatedWrite = [&](vk::AccessFlagBits2 access,
const char *label) {
image.backing.state = {};
image.backing.subresource_states.clear();
(void)image.GetBarriers(vk::ImageLayout::eGeneral, access, graphics_stage,
{});
Require(
name, label,
image.GetBarriers(vk::ImageLayout::eGeneral, access, graphics_stage, {})
.size() == 1,
"pinned repeated-write dependency was elided");
};
CheckRepeatedWrite(vk::AccessFlagBits2::eTransferWrite,
"repeat transfer write");
CheckRepeatedWrite(vk::AccessFlagBits2::eShaderWrite, "repeat shader write");
CheckRepeatedWrite(vk::AccessFlagBits2::eMemoryWrite, "repeat memory write");
image.backing.state = {};
image.backing.subresource_states.clear();
const auto attachment_access = vk::AccessFlagBits2::eColorAttachmentRead |
vk::AccessFlagBits2::eColorAttachmentWrite;
(void)image.GetBarriers(vk::ImageLayout::eColorAttachmentOptimal,
attachment_access, graphics_stage, {});
Require(name, "pinned attachment-write quirk",
image
.GetBarriers(vk::ImageLayout::eColorAttachmentOptimal,
attachment_access, graphics_stage, {})
.empty(),
"attachment writes diverged from the pinned repeated-write set");
image.backing.state = {};
image.backing.subresource_states.clear();
(void)image.GetBarriers(vk::ImageLayout::eShaderReadOnlyOptimal,
vk::AccessFlagBits2::eShaderRead, graphics_stage, {});
constexpr ImageSubresourceRange mip1_layer0{1, 1, 0, 1};
barriers = image.GetBarriers(vk::ImageLayout::eGeneral,
vk::AccessFlagBits2::eShaderWrite,
graphics_stage, mip1_layer0);
Require(name, "first partial",
barriers.size() == 1 &&
barriers[0].subresourceRange.baseMipLevel == 1 &&
barriers[0].subresourceRange.baseArrayLayer == 0 &&
image.backing.subresource_states.size() == 6 &&
image.backing.subresource_states[3].layout ==
vk::ImageLayout::eGeneral &&
image.backing.subresource_states[3].access_mask ==
vk::AccessFlagBits2::eShaderWrite &&
image.backing.subresource_states[1].layout ==
vk::ImageLayout::eShaderReadOnlyOptimal,
"partial state did not use mip-major indexing");
constexpr ImageSubresourceRange mip0_layer1{0, 1, 1, 1};
barriers = image.GetBarriers(
vk::ImageLayout::eTransferDstOptimal, vk::AccessFlagBits2::eTransferWrite,
vk::PipelineStageFlagBits2::eTransfer, mip0_layer1);
Require(name, "second partial",
barriers.size() == 1 &&
barriers[0].subresourceRange.baseMipLevel == 0 &&
barriers[0].subresourceRange.baseArrayLayer == 1 &&
image.backing.subresource_states[1].layout ==
vk::ImageLayout::eTransferDstOptimal &&
image.backing.subresource_states[1].access_mask ==
vk::AccessFlagBits2::eTransferWrite &&
image.backing.subresource_states[3].layout ==
vk::ImageLayout::eGeneral,
"second partial transition changed the wrong cell");
const ImageSubresourceRange explicit_full{0, 2, 0, 3};
barriers = image.GetBarriers(vk::ImageLayout::eShaderReadOnlyOptimal,
vk::AccessFlagBits2::eShaderRead, graphics_stage,
explicit_full);
bool saw_mip0_layer1 = false;
bool saw_mip1_layer0 = false;
for (const auto &barrier : barriers) {
saw_mip0_layer1 |=
barrier.subresourceRange.baseMipLevel == 0 &&
barrier.subresourceRange.baseArrayLayer == 1 &&
barrier.oldLayout == vk::ImageLayout::eTransferDstOptimal &&
barrier.srcStageMask == vk::PipelineStageFlagBits2::eTransfer;
saw_mip1_layer0 |=
barrier.subresourceRange.baseMipLevel == 1 &&
barrier.subresourceRange.baseArrayLayer == 0 &&
barrier.oldLayout == vk::ImageLayout::eGeneral &&
barrier.srcAccessMask == vk::AccessFlagBits2::eShaderWrite;
}
Require(name, "partial normalization",
barriers.size() == 2 && saw_mip0_layer1 && saw_mip1_layer0 &&
image.backing.subresource_states.empty() &&
image.backing.state.layout ==
vk::ImageLayout::eShaderReadOnlyOptimal &&
image.backing.state.access_mask ==
vk::AccessFlagBits2::eShaderRead &&
image.backing.state.pl_stage == graphics_stage,
"full transition did not reconcile and collapse split state");
barriers = image.GetBarriers(vk::ImageLayout::eTransferSrcOptimal,
vk::AccessFlagBits2::eTransferRead,
vk::PipelineStageFlagBits2::eTransfer, {});
Require(name, "normalized full transition",
barriers.size() == 1 &&
barriers[0].subresourceRange.levelCount ==
VK_REMAINING_MIP_LEVELS &&
barriers[0].subresourceRange.layerCount ==
VK_REMAINING_ARRAY_LAYERS &&
barriers[0].srcStageMask == graphics_stage &&
barriers[0].dstStageMask == vk::PipelineStageFlagBits2::eTransfer,
"normalized state did not return to one full barrier");
auto depth_info = MakeInfo(vk::Format::eD32SfloatS8Uint, 1, 1);
depth_info.guest_format = Prospero::BufferFormat::k32Float;
depth_info.bytes_per_block = 4;
depth_info.tile_mode = Prospero::TileMode::kDepth;
Image depth(context, scheduler, depth_info);
barriers =
depth.GetBarriers(vk::ImageLayout::eDepthStencilReadOnlyOptimal,
vk::AccessFlagBits2::eShaderRead, graphics_stage, {});
Require(name, "depth stencil aspects",
barriers.size() == 1 && barriers[0].subresourceRange.aspectMask ==
(vk::ImageAspectFlagBits::eDepth |
vk::ImageAspectFlagBits::eStencil),
"depth/stencil transition did not use the full image aspect");
HW::Context registers{};
HW::UserConfig user_config{};
HW::Shader shaders{};
scheduler.Begin(registers, user_config, shaders);
image.backing.state = {};
image.backing.subresource_states.clear();
image.Transit(vk::ImageLayout::eGeneral,
vk::AccessFlagBits2::eShaderRead |
vk::AccessFlagBits2::eTransferRead,
{}, scheduler.Current().Handle());
Require(name, "mixed access stages",
image.backing.state.pl_stage ==
(graphics_stage | vk::PipelineStageFlagBits2::eTransfer),
"mixed shader/transfer access omitted a required pipeline stage");
auto source_info = depth_info;
source_info.extent = {2, 2, 1};
source_info.pitch = 2;
source_info.stencil = {0x1000, 4};
auto destination_info = depth_info;
destination_info.resources.levels = 2;
destination_info.stencil = {0x2000, 16};
Image source(context, scheduler, source_info);
Image destination(context, scheduler, destination_info);
StreamBuffer upload(context, scheduler, MemoryUsage::Upload, 4096);
StreamBuffer download(context, scheduler, MemoryUsage::Download, 4096);
const auto [upload_data, upload_offset] = upload.Map(128, 4);
Require(name, "depth/stencil upload map", upload_data != nullptr,
"combined depth/stencil upload allocation failed");
constexpr std::array<uint32_t, 4> expected_depth{0x3e800000u, 0x3f000000u,
0x3f400000u, 0x3f800000u};
constexpr std::array<uint8_t, 4> expected_stencil{0x12, 0x34, 0x56, 0x78};
std::memcpy(upload_data, expected_depth.data(), sizeof(expected_depth));
std::memcpy(upload_data + 64, expected_stencil.data(),
sizeof(expected_stencil));
upload.Commit();
std::array<vk::BufferImageCopy, 2> source_copies{};
source_copies[0].bufferOffset = upload_offset;
source_copies[0].imageSubresource = {vk::ImageAspectFlagBits::eDepth, 0, 0,
1};
source_copies[0].imageExtent = {2, 2, 1};
source_copies[1].bufferOffset = upload_offset + 64;
source_copies[1].imageSubresource = {vk::ImageAspectFlagBits::eStencil, 0, 0,
1};
source_copies[1].imageExtent = {2, 2, 1};
source.Upload(source_copies, upload.Handle(), upload_offset, 128);
destination.CopyMip(source, 1, 0);
const auto [download_data, download_offset] = download.Map(128, 4);
Require(name, "depth/stencil download map", download_data != nullptr,
"combined depth/stencil download allocation failed");
download.Commit();
auto destination_copies = source_copies;
destination_copies[0].bufferOffset = download_offset;
destination_copies[0].imageSubresource.mipLevel = 1;
destination_copies[1].bufferOffset = download_offset + 64;
destination_copies[1].imageSubresource.mipLevel = 1;
destination.Download(destination_copies, download.Handle(), download_offset,
128);
auto buffered_info = MakeInfo(vk::Format::eR8Unorm, 1, 1);
buffered_info.extent = {8, 4, 1};
buffered_info.pitch = 8;
buffered_info.mip_layout[0] = {0, 32, 8, 4};
Image buffered_source(context, scheduler, buffered_info);
Image buffered_destination(context, scheduler, buffered_info);
constexpr std::array<uint8_t, 32> buffered_expected{
0x00, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88, 0x99, 0xaa,
0xbb, 0xcc, 0xdd, 0xee, 0xff, 0x10, 0x21, 0x32, 0x43, 0x54, 0x65,
0x76, 0x87, 0x98, 0xa9, 0xba, 0xcb, 0xdc, 0xed, 0xfe, 0x0f};
const auto [buffered_upload_data, buffered_upload_offset] =
upload.Map(buffered_expected.size(), 4);
Require(name, "buffered-copy upload map", buffered_upload_data != nullptr,
"buffered-copy source allocation failed");
std::memcpy(buffered_upload_data, buffered_expected.data(),
buffered_expected.size());
upload.Commit();
vk::BufferImageCopy buffered_region{};
buffered_region.bufferOffset = buffered_upload_offset;
buffered_region.bufferRowLength = buffered_info.pitch;
buffered_region.bufferImageHeight = buffered_info.extent.height;
buffered_region.imageSubresource = {vk::ImageAspectFlagBits::eColor, 0, 0, 1};
buffered_region.imageExtent = buffered_info.extent;
buffered_source.Upload(std::span{&buffered_region, 1}, upload.Handle(),
buffered_upload_offset, buffered_expected.size());
Buffer copy_scratch(context, scheduler, MemoryUsage::DeviceLocal, 0, AllFlags,
16);
StreamBuffer copy_parameters(context, scheduler, MemoryUsage::Stream, 4096);
TileManager copy_tiler(context, scheduler, copy_parameters);
buffered_destination.CopyImageWithBuffer(buffered_source, copy_scratch, copy_tiler);
const auto [buffered_download_data, buffered_download_offset] =
download.Map(buffered_expected.size(), 4);
Require(name, "buffered-copy download map", buffered_download_data != nullptr,
"buffered-copy destination allocation failed");
download.Commit();
buffered_region.bufferOffset = buffered_download_offset;
buffered_destination.Download(std::span{&buffered_region, 1},
download.Handle(), buffered_download_offset,
buffered_expected.size());
scheduler.Finish();
download.Invalidate(download_offset, 128);
download.Invalidate(buffered_download_offset, buffered_expected.size());
Require(name, "depth/stencil mip contents",
std::memcmp(download_data, expected_depth.data(),
sizeof(expected_depth)) == 0 &&
std::memcmp(download_data + 64, expected_stencil.data(),
sizeof(expected_stencil)) == 0,
"combined depth/stencil mip copy lost an aspect");
Require(name, "buffered-copy chunk contents",
std::memcmp(buffered_download_data, buffered_expected.data(),
buffered_expected.size()) == 0,
"real buffered image copy lost a row across its scratch boundary");
std::printf("[host] %-32s ok\n", name);
}
void CheckDepthTextureEncoding() {
const ShaderTextureResource disabled_sampler_tweaks{{
0x05135600u,
0xc1600000u,
0x010dc1dfu,
0x91800924u,
0x00000000u,
0x00000000u,
0x00000000u,
0x00000000u,
}};
Require("DepthTextureEncoding", "disabled sampler tweaks",
disabled_sampler_tweaks.PerfMod5() == 0 &&
IsSupportedDepthTextureEncoding(disabled_sampler_tweaks),
"valid sampler modulation factor zero was rejected");
const ShaderTextureResource uncompressed_msaa{{
0x00705d00u,
0xc1600000u,
0x010dc1dfu,
0xe1810924u,
0x00000000u,
0x00700010u,
0x00000000u,
0x00000000u,
}};
Require("DepthTextureEncoding", "uncompressed 2x MSAA depth",
IsSupportedDepthTextureEncoding(uncompressed_msaa),
"valid uncompressed MSAA depth descriptor required an HTILE "
"compatibility flag");
auto r128_msaa = uncompressed_msaa;
std::fill(r128_msaa.fields + 4, r128_msaa.fields + 8, 0u);
Require("DepthTextureEncoding", "R128 uncompressed 2x MSAA depth",
IsSupportedDepthTextureEncoding(r128_msaa, true),
"valid R128 MSAA depth descriptor required omitted dwords");
const ShaderTextureResource compressed_descriptor{{
0x0104d500u,
0xc1600000u,
0x021bc3bfu,
0x91800924u,
0x00000000u,
0x00700000u,
0x80280000u,
0x000106d4u,
}};
Require("DepthTextureEncoding", "compressed HTILE descriptor",
IsSupportedDepthTextureEncoding(compressed_descriptor),
"valid compressed depth descriptor required a prior depth target");
const ShaderTextureResource disabled_metadata{{
0x20018100u, 0xc1600000u, 0x021bc1dfu, 0x91800204u,
0x00000000u, 0x00700000u, 0x00000000u, 0x0020037fu,
}};
auto retained_metadata_address = compressed_descriptor;
retained_metadata_address.fields[6] &= 0xff000000u;
Require("DepthTextureEncoding", "disabled metadata with retained address",
IsSupportedDepthTextureEncoding(disabled_metadata) &&
IsSupportedDepthTextureEncoding(retained_metadata_address),
"uncompressed depth rejected inactive metadata address bits");
const ShaderTextureResource first_use_depth{{
0x0225fc00u, 0x01600000u, 0x00000000u, 0x91800924u,
0x00000000u, 0x00700000u, 0x80280000u, 0x000225fdu,
}};
Require("DepthTextureEncoding", "PPSA30490 first-use 1x1 depth",
IsSupportedDepthTextureEncoding(first_use_depth),
"captured comparison texture required tracked HTILE metadata");
auto missing_metadata = compressed_descriptor;
missing_metadata.fields[6] &= 0x00ffffffu;
missing_metadata.fields[7] = 0;
auto out_of_range_metadata = compressed_descriptor;
out_of_range_metadata.fields[7] = TRACKER_ADDRESS_SIZE >> 16u;
auto unaligned_metadata = compressed_descriptor;
unaligned_metadata.fields[6] ^= 1u << 24u;
auto dcc_only_control = compressed_descriptor;
dcc_only_control.fields[6] |= 1u << 22u;
auto non_msaa_iterate_256 = compressed_descriptor;
non_msaa_iterate_256.fields[6] |= 1u << 10u;
Require(
"DepthTextureEncoding", "invalid HTILE encoding",
!IsSupportedDepthTextureEncoding(missing_metadata) &&
!IsSupportedDepthTextureEncoding(out_of_range_metadata) &&
!IsSupportedDepthTextureEncoding(unaligned_metadata) &&
!IsSupportedDepthTextureEncoding(dcc_only_control) &&
!IsSupportedDepthTextureEncoding(non_msaa_iterate_256),
"compressed sampled depth accepted an unsupported address or control");
std::printf("[host] %-32s ok\n", "DepthTextureEncoding");
}
ShaderRecompiler::IR::ImageResource BasicStorageTextureResource() {
ShaderRecompiler::IR::ImageResource resource{};
resource.resource_class = ShaderRecompiler::IR::ImageResourceClass::Storage;
resource.numeric_class = Prospero::TextureNumericClass::Float;
resource.dimension = ShaderRecompiler::Decoder::ImageDimension::Dim3D;
resource.read = true;
resource.written = true;
return resource;
}
ShaderTextureResource BasicStorageTextureDescriptor() {
return {{0x00785d00u, 0x04700000u, 0x00080008u, 0xa1b00facu, 0x00000020u,
0x00700000u, 0x00000000u, 0x00000000u}};
}
ShaderRecompiler::IR::ImageResource BasicLinearStorageTextureResource() {
auto resource = BasicStorageTextureResource();
resource.dimension = ShaderRecompiler::Decoder::ImageDimension::Dim2D;
resource.read = false;
return resource;
}
ShaderTextureResource BasicLinearStorageTextureDescriptor() {
return {{0x04bcc401u, 0xc3800000u, 0x021bc3bfu, 0x900003acu, 0x00000000u,
0x00700000u, 0x00000000u, 0x00000000u}};
}
ShaderRecompiler::IR::ImageResource BasicBgraStorageTextureResource() {
auto resource = BasicStorageTextureResource();
resource.dimension = ShaderRecompiler::Decoder::ImageDimension::Dim2D;
resource.read = false;
return resource;
}
ShaderTextureResource BasicBgraStorageTextureDescriptor() {
return {{0x007c6500u, 0xc3800000u, 0x010dc1dfu, 0x91b00f2eu, 0x00000000u,
0x00700000u, 0x00000000u, 0x00000000u}};
}
ShaderTextureResource Ppsa06228R11G11B10StorageTextureDescriptor() {
return {{0x10c6b500u, 0xc2400000u, 0x010dc1dfu, 0x91b003acu, 0x00000000u,
0x00700000u, 0x00000000u, 0x00000000u}};
}
ShaderTextureResource Ppsa01530MaxMipStorageTextureDescriptor() {
return {{0x04a42900u, 0xc3e00000u, 0x000fc00fu, 0x91b0022cu, 0x00000000u,
0x00700050u, 0x00000000u, 0x00000000u}};
}
ShaderTextureResource Ppsa01340MipRangeStorageTextureDescriptor() {
return {{0x0294dc00u, 0xc4700000u, 0x00b3c13fu, 0x91b31facu, 0x00000000u,
0x00700030u, 0xb07b0000u, 0x0002ac3cu}};
}
ShaderTextureResource Ppsa01340OverwideMipStorageTextureDescriptor() {
return {{0x0346c000u, 0xc4b00000u, 0x007fc07fu, 0x90990facu, 0x00000000u,
0x00700080u, 0x00000000u, 0x00000000u}};
}
ShaderRecompiler::IR::ImageResource Ppsa01530MaxMipStorageTextureResource() {
auto resource = BasicBgraStorageTextureResource();
resource.numeric_class = Prospero::TextureNumericClass::Uint;
return resource;
}
ShaderTextureResource Ppsa02527R16FloatStorageTextureDescriptor() {
return {{0x00ce3500u, 0xc0d00000u, 0x010dc1dfu, 0x91b00204u, 0x00000000u,
0x00700000u, 0x00000000u, 0x00000000u}};
}
ShaderTextureResource Ppsa02527R32FloatStorageTextureDescriptor() {
return {{0x00cea900u, 0xc1600000u, 0x0086c0efu, 0x91b00204u, 0x00000000u,
0x00700000u, 0x00000000u, 0x00000000u}};
}
ShaderTextureResource Ppsa02527R8UnormStorageTextureDescriptor() {
return {{0x00c7d500u, 0xc0100000u, 0x0086c0efu, 0x91b00204u, 0x00000000u,
0x00700000u, 0x00000000u, 0x00000000u}};
}
ShaderTextureResource BasicYzwxStorageTextureDescriptor() {
return {{0x00627801u, 0xc4d00000u, 0x0001c001u, 0x900009f5u, 0x00000000u,
0x00700000u, 0x00000000u, 0x00000000u}};
}
ShaderRecompiler::IR::ImageResource BasicArrayStorageTextureResource() {
auto resource = BasicLinearStorageTextureResource();
resource.dimension = ShaderRecompiler::Decoder::ImageDimension::Dim2DArray;
return resource;
}
ShaderTextureResource BasicArrayStorageTextureDescriptor() {
return {{0x20179000u, 0x03800000u, 0x00000000u, 0xd1b00f2eu, 0x00000000u,
0x00700000u, 0x00000000u, 0x00000000u}};
}
ShaderRecompiler::IR::ImageResource BasicUintArrayStorageTextureResource() {
auto resource = BasicArrayStorageTextureResource();
resource.numeric_class = Prospero::TextureNumericClass::Uint;
return resource;
}
ShaderTextureResource BasicUintArrayStorageTextureDescriptor() {
return {{0x20179200u, 0x01400000u, 0x00000000u, 0xd1b00204u, 0x00000000u,
0x00700000u, 0x00000000u, 0x00000000u}};
}
ShaderTextureResource Standard4KBUintArrayStorageTextureDescriptor() {
return {{0x006c6600u, 0x01400000u, 0x00000000u, 0xd0500204u, 0x00000000u,
0x00700000u, 0x00000000u, 0x00000000u}};
}
ShaderTextureResource Standard64KBStorageTextureDescriptor() {
return {{0x011fab00u, 0xc3800000u, 0x0003c003u, 0xd0900facu, 0x00000000u,
0x00700000u, 0x00000000u, 0x00000000u}};
}
ShaderRecompiler::IR::ImageResource Ppsa14053DepthTileStorageTextureResource() {
return BasicUintArrayStorageTextureResource();
}
ShaderTextureResource Ppsa14053DepthTileStorageTextureDescriptor() {
return {{0x20144c00u, 0x00500000u, 0x00000000u, 0xd1800204u, 0x00000000u,
0x00700000u, 0x00000000u, 0x00000000u}};
}
ShaderTextureResource Ppsa10112D16StorageTextureDescriptor() {
return {{0x205b9000u, 0xc0700000u, 0x0021803bu, 0xd1800204u, 0x00000000u,
0x00700000u, 0x00000000u, 0x00000000u}};
}
ShaderRecompiler::IR::ImageResource BasicUintVolumeStorageTextureResource() {
auto resource = BasicStorageTextureResource();
resource.numeric_class = Prospero::TextureNumericClass::Uint;
resource.read = false;
return resource;
}
ShaderTextureResource BasicUintVolumeStorageTextureDescriptor() {
return {{0x20180600u, 0xc0b00000u, 0x0003c003u, 0xa0000004u, 0x0000000fu,
0x00700000u, 0x00000000u, 0x00000000u}};
}
ShaderRecompiler::IR::ImageResource AtomicStorageTextureResource() {
auto resource = BasicLinearStorageTextureResource();
resource.numeric_class = Prospero::TextureNumericClass::Uint;
resource.read = true;
resource.atomic = true;
return resource;
}
ShaderTextureResource AtomicStorageTextureDescriptor() {
return {{0x304bb700u, 0xc1400000u, 0x0000001fu, 0x91b00204u, 0x00000000u,
0x00700000u, 0x00000000u, 0x00000000u}};
}
[[noreturn]] void RunStorageTextureDescriptorDeathCase(const char *kind) {
auto resource = BasicStorageTextureResource();
auto descriptor = BasicStorageTextureDescriptor();
if (std::strcmp(kind, "linear-rgb1-read") == 0) {
resource = BasicLinearStorageTextureResource();
descriptor = BasicLinearStorageTextureDescriptor();
resource.read = true;
} else if (std::strcmp(kind, "bgra-read") == 0) {
resource = BasicBgraStorageTextureResource();
descriptor = BasicBgraStorageTextureDescriptor();
resource.read = true;
} else if (std::strcmp(kind, "r16-float-read") == 0) {
resource = BasicBgraStorageTextureResource();
descriptor = Ppsa02527R16FloatStorageTextureDescriptor();
resource.read = true;
} else if (std::strcmp(kind, "r8-unorm-read") == 0) {
resource = BasicBgraStorageTextureResource();
descriptor = Ppsa02527R8UnormStorageTextureDescriptor();
resource.read = true;
} else if (std::strcmp(kind, "yzwx-read") == 0) {
resource = BasicLinearStorageTextureResource();
descriptor = BasicYzwxStorageTextureDescriptor();
resource.read = true;
} else if (std::strcmp(kind, "reserved-swizzle") == 0) {
resource = BasicLinearStorageTextureResource();
descriptor = BasicLinearStorageTextureDescriptor();
descriptor.fields[3] =
(descriptor.fields[3] & ~0xfffu) | DstSel(4, 5, 6, 2);
} else if (std::strcmp(kind, "resource") == 0) {
resource.written = false;
} else if (std::strcmp(kind, "type") == 0) {
descriptor.fields[3] =
(descriptor.fields[3] & 0x0fffffffu) |
(static_cast<uint32_t>(Prospero::ImageType::kColor2D) << 28u);
} else if (std::strcmp(kind, "standard256b-volume") == 0) {
descriptor.fields[3] =
(descriptor.fields[3] & ~(0x1fu << 20u)) |
(static_cast<uint32_t>(Prospero::TileMode::kStandard256B) << 20u);
} else if (std::strcmp(kind, "inverted-mip-range") == 0) {
descriptor.fields[3] |= 1u << 12u;
descriptor.fields[5] |= 1u << 4u;
} else if (std::strcmp(kind, "swizzle") == 0) {
descriptor.fields[3] =
(descriptor.fields[3] & ~0xfffu) | DstSel(4, 5, 6, 1);
} else if (std::strcmp(kind, "array-base-out-of-range") == 0) {
resource = BasicArrayStorageTextureResource();
descriptor = BasicArrayStorageTextureDescriptor();
descriptor.fields[4] |= 1u << 16u;
} else if (std::strcmp(kind, "reserved") == 0) {
descriptor.fields[1] |= 1u << 29u;
} else if (std::strcmp(kind, "uint-format") == 0) {
descriptor.fields[1] =
(descriptor.fields[1] & ~0x1ff00000u) |
(static_cast<uint32_t>(Prospero::BufferFormat::k8_8_8_8UInt) << 20u);
} else if (std::strcmp(kind, "uint-resource-float-format") == 0) {
resource = BasicUintArrayStorageTextureResource();
descriptor = BasicArrayStorageTextureDescriptor();
} else if (std::strcmp(kind, "atomic-format") == 0) {
resource = AtomicStorageTextureResource();
descriptor = AtomicStorageTextureDescriptor();
descriptor.fields[1] =
(descriptor.fields[1] & ~0x1ff00000u) |
(static_cast<uint32_t>(Prospero::BufferFormat::k8UInt) << 20u);
} else if (std::strcmp(kind, "depth-tile-read") == 0) {
resource = Ppsa14053DepthTileStorageTextureResource();
descriptor = Ppsa14053DepthTileStorageTextureDescriptor();
resource.read = true;
} else if (std::strcmp(kind, "depth-tile-extent") == 0) {
resource = Ppsa14053DepthTileStorageTextureResource();
descriptor = Ppsa14053DepthTileStorageTextureDescriptor();
resource.dimension = ShaderRecompiler::Decoder::ImageDimension::Dim2D;
descriptor.fields[3] =
(descriptor.fields[3] & 0x0fffffffu) |
(static_cast<uint32_t>(Prospero::ImageType::kColor2D) << 28u);
descriptor.fields[4] |= 1u;
} else if (std::strcmp(kind, "depth-tile-fmask") == 0) {
resource = BasicArrayStorageTextureResource();
descriptor = Ppsa10112D16StorageTextureDescriptor();
descriptor.fields[1] =
(descriptor.fields[1] & ~(0x1ffu << 20u)) |
(static_cast<uint32_t>(Prospero::BufferFormat::kFmask8_S4_F4) << 20u);
} else {
std::_Exit(0x7e);
}
ValidateStorageTexture(resource, descriptor, 0x10000);
std::_Exit(0x7f);
}
#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
void CheckBasicStorageTextureDescriptor() {
const auto descriptor = BasicStorageTextureDescriptor();
Require("BasicStorageTexture", "descriptor",
descriptor.Base40() == 0x785d0000ull &&
descriptor.Width5() + 1u == 33 &&
descriptor.Height5() + 1u == 33 && descriptor.Depth() + 1u == 33,
"basic 3D storage descriptor fixture is malformed");
ValidateStorageTexture(BasicStorageTextureResource(), descriptor, 0x10000);
const ShaderTextureResource extended{{0x204aca00u, 0xc4700000u, 0x000fc00fu,
0xa1b00facu, 0x0000003fu, 0x00700000u,
0x006b0000u, 0x00204b0au}};
auto extended_resource = BasicStorageTextureResource();
extended_resource.read = false;
Require("BasicStorageTexture", "extended descriptor",
extended.fields[6] != 0 && extended.fields[7] != 0,
"extended 3D storage descriptor fixture is malformed");
ValidateStorageTexture(extended_resource, extended, 0x400000);
const auto linear = BasicLinearStorageTextureDescriptor();
Require("BasicStorageTexture", "linear descriptor",
linear.Base40() == 0x4bcc40100ull && linear.Width5() + 1u == 3840 &&
linear.Height5() + 1u == 2160 && linear.Depth() + 1u == 1 &&
linear.Format() == Prospero::BufferFormat::k8_8_8_8UNorm &&
linear.TileMode() == Prospero::TileMode::kLinear,
"PPSA07429 linear 2D storage descriptor fixture is malformed");
ValidateStorageTexture(BasicLinearStorageTextureResource(), linear,
0x1fa4000);
const auto bgra = BasicBgraStorageTextureDescriptor();
Require("BasicStorageTexture", "BGRA descriptor",
bgra.Base40() == 0x7c650000ull && bgra.Width5() + 1u == 1920 &&
bgra.Height5() + 1u == 1080 && bgra.Depth() + 1u == 1 &&
bgra.Format() == Prospero::BufferFormat::k8_8_8_8UNorm &&
bgra.TileMode() == Prospero::TileMode::kRenderTarget &&
bgra.DstSelXYZW() == DstSel(6, 5, 4, 7),
"PPSA02604 BGRA 2D storage descriptor fixture is malformed");
ValidateStorageTexture(BasicBgraStorageTextureResource(), bgra, 0x870000);
const ShaderTextureResource r128{
{0x0202e500u, 0xc8200000u, 0x010dc1dfu, 0x91b00facu, 0, 0, 0, 0}};
auto r128_resource = BasicBgraStorageTextureResource();
r128_resource.r128 = true;
Require("BasicStorageTexture", "PPSA01736 R128 descriptor",
r128.Base40() == 0x202e50000ull && r128.Width5() + 1u == 1920 &&
r128.Height5() + 1u == 1080 && r128.Depth() + 1u == 1 &&
r128.Format() == Prospero::BufferFormat::k8_8_8_8Srgb &&
r128.TileMode() == Prospero::TileMode::kRenderTarget &&
r128.DstSelXYZW() == DstSel(4, 5, 6, 7),
"PPSA01736 R128 storage descriptor fixture is malformed");
ValidateStorageTexture(r128_resource, r128, 0x870000);
const auto r11g11b10 = Ppsa06228R11G11B10StorageTextureDescriptor();
Require("BasicStorageTexture", "PPSA06228 R11G11B10 descriptor",
r11g11b10.Base40() == 0x10c6b50000ull &&
r11g11b10.Width5() + 1u == 1920 &&
r11g11b10.Height5() + 1u == 1080 && r11g11b10.Depth() + 1u == 1 &&
r11g11b10.Format() == Prospero::BufferFormat::k11_11_10Float &&
r11g11b10.TileMode() == Prospero::TileMode::kRenderTarget &&
r11g11b10.DstSelXYZW() == DstSel(4, 5, 6, 1),
"PPSA06228 R11G11B10 storage descriptor fixture is malformed");
ValidateStorageTexture(BasicBgraStorageTextureResource(), r11g11b10,
0x870000);
ValidateStorageColorView(vk::Format::eB8G8R8A8Unorm,
vk::Format::eB10G11R11UfloatPack32,
r11g11b10.DstSelXYZW());
const auto max_mip = Ppsa01530MaxMipStorageTextureDescriptor();
Require("BasicStorageTexture", "PPSA01530 max-mip descriptor",
max_mip.Base40() == 0x4a4290000ull && max_mip.Width5() + 1u == 64 &&
max_mip.Height5() + 1u == 64 && max_mip.Depth() + 1u == 1 &&
max_mip.BaseLevel() == 0 && max_mip.LastLevel() == 0 &&
max_mip.MaxMip() == 5 &&
max_mip.Format() == Prospero::BufferFormat::k32_32UInt &&
max_mip.TileMode() == Prospero::TileMode::kRenderTarget &&
max_mip.DstSelXYZW() == DstSel(4, 5, 0, 1),
"PPSA01530 max-mip storage descriptor fixture is malformed");
ValidateStorageTexture(Ppsa01530MaxMipStorageTextureResource(), max_mip,
0x20000);
auto mip_one = max_mip;
mip_one.fields[3] |= (1u << 12u) | (1u << 16u);
Require("BasicStorageTexture", "PPSA01530 mip-one descriptor",
mip_one.BaseLevel() == 1 && mip_one.LastLevel() == 1 &&
mip_one.MaxMip() == 5,
"PPSA01530 mip-one storage descriptor fixture is malformed");
ValidateStorageTexture(Ppsa01530MaxMipStorageTextureResource(), mip_one,
0x20000);
const auto mip_range = Ppsa01340MipRangeStorageTextureDescriptor();
Require("BasicStorageTexture", "PPSA01340 mip-range descriptor",
mip_range.Base40() == 0x294dc0000ull &&
mip_range.Width5() + 1u == 1280 &&
mip_range.Height5() + 1u == 720 && mip_range.Depth() + 1u == 1 &&
mip_range.BaseLevel() == 1 && mip_range.LastLevel() == 3 &&
mip_range.MaxMip() == 3 &&
mip_range.Format() == Prospero::BufferFormat::k16_16_16_16Float &&
mip_range.TileMode() == Prospero::TileMode::kRenderTarget &&
mip_range.DstSelXYZW() == DstSel(4, 5, 6, 7),
"PPSA01340 mip-range storage descriptor fixture is malformed");
ValidateStorageTexture(BasicBgraStorageTextureResource(), mip_range,
0xa30000);
const auto overwide_mip = Ppsa01340OverwideMipStorageTextureDescriptor();
Require(
"BasicStorageTexture", "PPSA01340 over-wide mip view",
overwide_mip.Base40() == 0x346c00000ull &&
overwide_mip.Width5() + 1u == 512 &&
overwide_mip.Height5() + 1u == 512 && overwide_mip.BaseLevel() == 0 &&
overwide_mip.LastLevel() == 9 && overwide_mip.MaxMip() == 8 &&
overwide_mip.Format() == Prospero::BufferFormat::k32_32_32_32UInt &&
overwide_mip.TileMode() == Prospero::TileMode::kStandard64KB,
"captured over-wide storage mip view is malformed");
TileSizeAlign overwide_size{};
TileGetTextureTotalSize(overwide_mip.Format(), overwide_mip.Width5() + 1u,
overwide_mip.Height5() + 1u, 1,
overwide_mip.MaxMip() + 1u, overwide_mip.TileMode(),
false, overwide_size);
Require("BasicStorageTexture", "PPSA01340 over-wide mip footprint",
overwide_size.size == 0x560000 && overwide_size.align == 0x10000,
"captured nine-level Standard64KB footprint changed");
ValidateStorageTexture(Ppsa01530MaxMipStorageTextureResource(), overwide_mip,
overwide_size.size);
const auto r16_float = Ppsa02527R16FloatStorageTextureDescriptor();
Require("BasicStorageTexture", "PPSA02527 R16F descriptor",
r16_float.Base40() == 0xce350000ull &&
r16_float.Width5() + 1u == 1920 &&
r16_float.Height5() + 1u == 1080 && r16_float.Depth() + 1u == 1 &&
r16_float.Format() == Prospero::BufferFormat::k16Float &&
r16_float.TileMode() == Prospero::TileMode::kRenderTarget &&
r16_float.DstSelXYZW() == DstSel(4, 0, 0, 1),
"PPSA02527 R16F 2D storage descriptor fixture is malformed");
ValidateStorageTexture(BasicBgraStorageTextureResource(), r16_float,
0x480000);
const auto r32_float = Ppsa02527R32FloatStorageTextureDescriptor();
Require("BasicStorageTexture", "PPSA02527 R32F descriptor",
r32_float.Base40() == 0xcea90000ull &&
r32_float.Width5() + 1u == 960 &&
r32_float.Height5() + 1u == 540 && r32_float.Depth() + 1u == 1 &&
r32_float.Format() == Prospero::BufferFormat::k32Float &&
r32_float.TileMode() == Prospero::TileMode::kRenderTarget &&
r32_float.DstSelXYZW() == DstSel(4, 0, 0, 1),
"PPSA02527 R32F 2D storage descriptor fixture is malformed");
ValidateStorageTexture(BasicBgraStorageTextureResource(), r32_float,
0x280000);
const auto r8_unorm = Ppsa02527R8UnormStorageTextureDescriptor();
Require("BasicStorageTexture", "PPSA02527 R8 UNORM descriptor",
r8_unorm.Base40() == 0xc7d50000ull && r8_unorm.Width5() + 1u == 960 &&
r8_unorm.Height5() + 1u == 540 && r8_unorm.Depth() + 1u == 1 &&
r8_unorm.Format() == Prospero::BufferFormat::k8UNorm &&
r8_unorm.TileMode() == Prospero::TileMode::kRenderTarget &&
r8_unorm.DstSelXYZW() == DstSel(4, 0, 0, 1),
"PPSA02527 R8 UNORM 2D storage descriptor fixture is malformed");
ValidateStorageTexture(BasicBgraStorageTextureResource(), r8_unorm, 0xc0000);
const auto yzwx = BasicYzwxStorageTextureDescriptor();
Require("BasicStorageTexture", "YZWX descriptor",
yzwx.Base40() == 0x62780100ull && yzwx.Width5() + 1u == 8 &&
yzwx.Height5() + 1u == 8 && yzwx.Depth() + 1u == 1 &&
yzwx.Format() == Prospero::BufferFormat::k32_32_32_32Float &&
yzwx.TileMode() == Prospero::TileMode::kLinear &&
yzwx.DstSelXYZW() == DstSel(5, 6, 7, 4),
"PPSA04181 linear YZWX storage descriptor fixture is malformed");
ValidateStorageTexture(BasicLinearStorageTextureResource(), yzwx, 0x800);
auto all_swizzles = yzwx;
all_swizzles.fields[1] =
(all_swizzles.fields[1] & ~0x1ff00000u) |
(static_cast<uint32_t>(Prospero::BufferFormat::k16_16_16_16Float) << 20u);
uint32_t valid_storage_swizzles = 0;
for (uint32_t swizzle = 0; swizzle <= 0xfffu; swizzle++) {
if (!IsValidImageSwizzle(swizzle)) {
continue;
}
all_swizzles.fields[3] = (all_swizzles.fields[3] & ~0xfffu) | swizzle;
ValidateStorageTexture(BasicLinearStorageTextureResource(), all_swizzles,
0x800);
ValidateStorageColorView(vk::Format::eR16G16B16A16Sfloat,
vk::Format::eR16G16B16A16Sfloat, swizzle);
ValidateStorageColorView(vk::Format::eR8G8B8A8Srgb,
vk::Format::eR8G8B8A8Unorm, swizzle);
ValidateStorageColorView(vk::Format::eB8G8R8A8Srgb,
vk::Format::eR8G8B8A8Unorm, swizzle);
valid_storage_swizzles++;
}
Require("BasicStorageTexture", "all write swizzles",
valid_storage_swizzles == 1296,
"valid write-only storage image mappings were rejected");
const ShaderTextureResource cube{{0x025bca00u, 0xc4700000u, 0x003fc03fu,
0xb1b00facu, 0x00000005u, 0x00700080u,
0x00000000u, 0x00000000u}};
auto cube_resource = BasicArrayStorageTextureResource();
cube_resource.cube = true;
Require("BasicStorageTexture", "PPSA07429 cube descriptor",
cube.Base40() == 0x25bca0000ull && cube.Width5() + 1u == 256 &&
cube.Height5() + 1u == 256 && cube.Depth() + 1u == 6 &&
cube.BaseArray5() == 0 && cube.MaxMip() == 8 &&
cube.Type() == Prospero::ImageType::kCube &&
cube.Format() == Prospero::BufferFormat::k16_16_16_16Float &&
cube.TileMode() == Prospero::TileMode::kRenderTarget &&
cube.DstSelXYZW() == DstSel(4, 5, 6, 7),
"PPSA07429 cube storage descriptor fixture is malformed");
ValidateStorageTexture(cube_resource, cube, 0x420000);
const auto array = BasicArrayStorageTextureDescriptor();
Require("BasicStorageTexture", "2D-array descriptor",
array.Base40() == 0x2017900000ull && array.Width5() + 1u == 1 &&
array.Height5() + 1u == 1 && array.Depth() + 1u == 1 &&
array.BaseArray5() == 0 &&
array.Type() == Prospero::ImageType::kColor2DArray &&
array.Format() == Prospero::BufferFormat::k8_8_8_8UNorm &&
array.TileMode() == Prospero::TileMode::kRenderTarget &&
array.DstSelXYZW() == DstSel(6, 5, 4, 7),
"PPSA21268 2D-array storage descriptor fixture is malformed");
ValidateStorageTexture(BasicArrayStorageTextureResource(), array, 0x10000);
const ShaderTextureResource mip_array{{0x20268d00u, 0xc4700000u, 0x001fc01fu,
0xd1b11facu, 0x00000000u, 0x00700070u,
0x00000000u, 0x00000000u}};
Require("BasicStorageTexture", "PPSA14457 mip-one 2D-array descriptor",
mip_array.BaseLevel() == 1 && mip_array.LastLevel() == 1 &&
mip_array.MaxMip() == 7 &&
mip_array.Type() == Prospero::ImageType::kColor2DArray &&
mip_array.Depth() == 0 && mip_array.BaseArray5() == 0,
"PPSA14457 mip-one 2D-array storage descriptor fixture is malformed");
ValidateStorageTexture(BasicArrayStorageTextureResource(), mip_array,
0x30000);
const auto uint_array = BasicUintArrayStorageTextureDescriptor();
Require("BasicStorageTexture", "uint 2D-array descriptor",
uint_array.Base40() == 0x2017920000ull &&
uint_array.Width5() + 1u == 1 && uint_array.Height5() + 1u == 1 &&
uint_array.Depth() + 1u == 1 && uint_array.BaseArray5() == 0 &&
uint_array.Type() == Prospero::ImageType::kColor2DArray &&
uint_array.Format() == Prospero::BufferFormat::k32UInt &&
uint_array.TileMode() == Prospero::TileMode::kRenderTarget &&
uint_array.DstSelXYZW() == DstSel(4, 0, 0, 1),
"PPSA21268 uint 2D-array storage descriptor fixture is malformed");
ValidateStorageTexture(BasicUintArrayStorageTextureResource(), uint_array,
0x10000);
const ShaderTextureResource standard256b{{
0x204e4900u,
0x43c00000u,
0x00004000u,
0x90100facu,
0x00000000u,
0x00700000u,
0x00000000u,
0x00000000u,
}};
auto standard256b_resource = BasicBgraStorageTextureResource();
standard256b_resource.numeric_class = Prospero::TextureNumericClass::Uint;
TileSizeAlign standard256b_size{};
TileGetTextureTotalSize(standard256b.Format(), standard256b.Width5() + 1u,
standard256b.Height5() + 1u,
standard256b.Depth() + 1u, 1, standard256b.TileMode(),
false, standard256b_size);
Require("BasicStorageTexture", "Standard256B uint 2D descriptor",
standard256b.Width5() + 1u == 2 && standard256b.Height5() + 1u == 2 &&
standard256b.Depth() + 1u == 1 &&
standard256b.Type() == Prospero::ImageType::kColor2D &&
standard256b.Format() == Prospero::BufferFormat::k8_8_8_8UInt &&
standard256b.TileMode() == Prospero::TileMode::kStandard256B &&
standard256b.DstSelXYZW() == DstSel(4, 5, 6, 7) &&
standard256b_size.size == 0x100 &&
standard256b_size.align == 0x100,
"captured PPSA08511 Standard256B storage descriptor is malformed");
ValidateStorageTexture(standard256b_resource, standard256b,
standard256b_size.size);
const auto standard4kb_array = Standard4KBUintArrayStorageTextureDescriptor();
TileSizeAlign standard4kb_size{};
TileGetTextureTotalSize(standard4kb_array.Format(), 1, 1, 1, 1,
standard4kb_array.TileMode(), false,
standard4kb_size);
Require(
"BasicStorageTexture", "Standard4KB uint 2D-array descriptor",
standard4kb_array.Type() == Prospero::ImageType::kColor2DArray &&
standard4kb_array.Format() == Prospero::BufferFormat::k32UInt &&
standard4kb_array.TileMode() == Prospero::TileMode::kStandard4KB &&
standard4kb_array.DstSelXYZW() == DstSel(4, 0, 0, 1) &&
standard4kb_size.size == 0x1000 && standard4kb_size.align == 0x1000,
"captured Standard4KB uint 2D-array descriptor is malformed");
ValidateStorageTexture(BasicUintArrayStorageTextureResource(),
standard4kb_array, standard4kb_size.size);
auto based_standard4kb_array = standard4kb_array;
based_standard4kb_array.fields[0] = 0x006c6800u;
based_standard4kb_array.fields[4] = 0x00010001u;
TileSizeAlign based_standard4kb_size{};
TileGetTextureTotalSize(based_standard4kb_array.Format(), 1, 1,
based_standard4kb_array.Depth() + 1u, 1,
based_standard4kb_array.TileMode(), false,
based_standard4kb_size);
Require("BasicStorageTexture", "based Standard4KB array view",
based_standard4kb_array.Base40() == 0x6c680000ull &&
based_standard4kb_array.BaseArray5() == 1 &&
based_standard4kb_array.Depth() == 1 &&
based_standard4kb_size.size == 0x2000 &&
based_standard4kb_size.align == 0x1000,
"captured based Standard4KB array view is malformed");
ValidateStorageTexture(BasicUintArrayStorageTextureResource(),
based_standard4kb_array, based_standard4kb_size.size);
const auto standard64kb = Standard64KBStorageTextureDescriptor();
TileSizeAlign standard64kb_size{};
TileGetTextureTotalSize(standard64kb.Format(), standard64kb.Width5() + 1u,
standard64kb.Height5() + 1u,
standard64kb.Depth() + 1u, 1, standard64kb.TileMode(),
false, standard64kb_size);
Require("BasicStorageTexture", "Standard64KB 2D descriptor",
standard64kb.Type() == Prospero::ImageType::kColor2DArray &&
standard64kb.Format() == Prospero::BufferFormat::k8_8_8_8UNorm &&
standard64kb.TileMode() == Prospero::TileMode::kStandard64KB &&
standard64kb.DstSelXYZW() == DstSel(4, 5, 6, 7) &&
standard64kb_size.size == 0x10000 &&
standard64kb_size.align == 0x10000,
"captured Standard64KB storage descriptor is malformed");
ValidateStorageTexture(BasicBgraStorageTextureResource(), standard64kb,
standard64kb_size.size);
const auto uint_volume = BasicUintVolumeStorageTextureDescriptor();
Require("BasicStorageTexture", "uint 3D descriptor",
uint_volume.Base40() == 0x2018060000ull &&
uint_volume.Width5() + 1u == 16 &&
uint_volume.Height5() + 1u == 16 &&
uint_volume.Depth() + 1u == 16 && uint_volume.BaseArray5() == 0 &&
uint_volume.Type() == Prospero::ImageType::kColor3D &&
uint_volume.Format() == Prospero::BufferFormat::k16UInt &&
uint_volume.TileMode() == Prospero::TileMode::kLinear &&
uint_volume.DstSelXYZW() == DstSel(4, 0, 0, 0),
"PPSA21268 uint 3D storage descriptor fixture is malformed");
ValidateStorageTexture(BasicUintVolumeStorageTextureResource(), uint_volume,
0x10000);
const ShaderTextureResource tiled_uint_volume{
{0x1ac0e530u, 0xc0b00000u, 0x0003c003u, 0xa0500004u, 0x0000000fu,
0x00700000u, 0x00000000u, 0x00000000u}};
Require("BasicStorageTexture", "tiled uint 3D descriptor",
tiled_uint_volume.Width5() + 1u == 16 &&
tiled_uint_volume.Height5() + 1u == 16 &&
tiled_uint_volume.Depth() + 1u == 16 &&
tiled_uint_volume.Type() == Prospero::ImageType::kColor3D &&
tiled_uint_volume.TileMode() == Prospero::TileMode::kStandard4KB,
"captured tiled uint 3D storage descriptor fixture is malformed");
ValidateStorageTexture(BasicUintVolumeStorageTextureResource(),
tiled_uint_volume, 0x2000);
const auto depth_tile = Ppsa14053DepthTileStorageTextureDescriptor();
Require("BasicStorageTexture", "PPSA14053 depth-tile descriptor",
depth_tile.Base40() == 0x20144c0000ull &&
depth_tile.Width5() + 1u == 1 && depth_tile.Height5() + 1u == 1 &&
depth_tile.Depth() + 1u == 1 && depth_tile.BaseArray5() == 0 &&
depth_tile.Type() == Prospero::ImageType::kColor2DArray &&
depth_tile.Format() == Prospero::BufferFormat::k8UInt &&
depth_tile.TileMode() == Prospero::TileMode::kDepth &&
depth_tile.DstSelXYZW() == DstSel(4, 0, 0, 1),
"PPSA14053 write-only depth-tile storage descriptor fixture is "
"malformed");
ValidateStorageTexture(Ppsa14053DepthTileStorageTextureResource(), depth_tile,
0x10000);
const auto d16_depth_tile = Ppsa10112D16StorageTextureDescriptor();
Require("BasicStorageTexture", "PPSA10112 D16 depth-tile descriptor",
d16_depth_tile.Base40() == 0x205b900000ull &&
d16_depth_tile.Width5() + 1u == 240 &&
d16_depth_tile.Height5() + 1u == 135 &&
d16_depth_tile.Depth() + 1u == 1 &&
d16_depth_tile.Type() == Prospero::ImageType::kColor2DArray &&
d16_depth_tile.Format() == Prospero::BufferFormat::k16UNorm &&
d16_depth_tile.TileMode() == Prospero::TileMode::kDepth &&
d16_depth_tile.DstSelXYZW() == DstSel(4, 0, 0, 1),
"PPSA10112 writable D16 depth-plane descriptor fixture is malformed");
const auto d16_pitch =
TileGetTexturePitch(d16_depth_tile.Format(), d16_depth_tile.Width5() + 1u,
d16_depth_tile.TileMode());
TileSizeAlign d16_size{};
TileGetTextureTotalSize(d16_depth_tile.Format(), d16_depth_tile.Width5() + 1u,
d16_depth_tile.Height5() + 1u,
d16_depth_tile.Depth() + 1u, 1,
d16_depth_tile.TileMode(), false, d16_size);
Require("BasicStorageTexture", "PPSA10112 D16 depth-tile footprint",
d16_pitch == 256 && d16_size.size == 0x20000 &&
d16_size.align == 0x10000,
"PPSA10112 writable D16 depth-plane footprint is incorrect");
ValidateStorageTexture(BasicArrayStorageTextureResource(), d16_depth_tile,
d16_size.size);
auto depth_tile_r32 = depth_tile;
depth_tile_r32.fields[1] =
(depth_tile_r32.fields[1] & ~(0x1ffu << 20u)) |
(static_cast<uint32_t>(Prospero::BufferFormat::k32UInt) << 20u);
depth_tile_r32.fields[3] =
(depth_tile_r32.fields[3] & ~(0xfu << 28u)) |
(static_cast<uint32_t>(Prospero::ImageType::kColor2D) << 28u);
auto depth_tile_r32_resource = Ppsa14053DepthTileStorageTextureResource();
depth_tile_r32_resource.dimension =
ShaderRecompiler::Decoder::ImageDimension::Dim2D;
ValidateStorageTexture(depth_tile_r32_resource, depth_tile_r32, 0x10000);
Require("BasicStorageTexture", "R32_UINT replicated write mapping",
IsValidImageSwizzle(DstSel(4, 4, 4, 4)),
"single-channel replicated destination selection was rejected");
const auto atomic = AtomicStorageTextureDescriptor();
Require("BasicStorageTexture", "atomic R32_UINT descriptor",
atomic.Width5() + 1u == 128 && atomic.Height5() + 1u == 1 &&
atomic.Depth() + 1u == 1 &&
atomic.Type() == Prospero::ImageType::kColor2D &&
atomic.Format() == Prospero::BufferFormat::k32UInt &&
atomic.DstSelXYZW() == DstSel(4, 0, 0, 1),
"PPSA22102 image-atomic descriptor fixture is malformed");
ValidateStorageTexture(AtomicStorageTextureResource(), atomic, 0x10000);
char path[MAX_PATH]{};
Require("BasicStorageTexture", "host",
GetModuleFileNameA(nullptr, path, MAX_PATH) != 0,
"GetModuleFileName failed");
for (const char *kind : {"resource",
"type",
"standard256b-volume",
"inverted-mip-range",
"swizzle",
"linear-rgb1-read",
"bgra-read",
"r16-float-read",
"r8-unorm-read",
"yzwx-read",
"reserved-swizzle",
"array-base-out-of-range",
"reserved",
"uint-format",
"uint-resource-float-format",
"atomic-format",
"depth-tile-read",
"depth-tile-extent",
"depth-tile-fmask"}) {
std::string command = std::string("\"") + path +
"\" --storage-texture-descriptor-death " + kind;
std::vector<char> mutable_command(command.begin(), command.end());
mutable_command.push_back('\0');
STARTUPINFOA startup{sizeof(startup)};
PROCESS_INFORMATION process{};
Require("BasicStorageTexture", "host",
CreateProcessA(nullptr, mutable_command.data(), nullptr, nullptr,
FALSE, CREATE_NO_WINDOW, nullptr, nullptr, &startup,
&process) != 0,
"CreateProcess failed");
Require("BasicStorageTexture", "host",
WaitForSingleObject(process.hProcess, 10000) == WAIT_OBJECT_0,
"descriptor death case timed out");
DWORD exit_code = 0;
const bool exited = GetExitCodeProcess(process.hProcess, &exit_code) != 0;
CloseHandle(process.hThread);
CloseHandle(process.hProcess);
Require("BasicStorageTexture", "host", exited && exit_code == 321,
std::string(kind) +
" storage descriptor did not report a fatal error");
}
std::printf("[host] %-32s ok\n", "BasicStorageTextureDescriptor");
}
#endif
void CheckStorageTextureLinearUploadLayout() {
constexpr auto format = Prospero::BufferFormat::k8_8_8_8UNorm;
constexpr uint32_t width = 3840;
constexpr uint32_t height = 2160;
constexpr uint32_t depth = 1;
constexpr auto tile = Prospero::TileMode::kLinear;
const auto pitch = TileGetTexturePitch(format, width, tile);
TileSizeAlign total{};
TileGetTextureTotalSize(format, width, height, depth, 1, tile, false, total);
const auto layout =
MakeTilingImage(format, width, height, 1, depth, tile, total.size, false);
const auto regions = layout.BufferCopies();
Require("StorageTextureLinearUpload", "layout",
pitch == width && total.size == 0x1fa4000 && total.align == 256 &&
layout.tile_mode == tile && layout.pitch == width &&
layout.linear_slice_stride == total.size && regions.size() == 1 &&
regions[0].bufferOffset == 0 &&
regions[0].imageExtent.width == width &&
regions[0].imageExtent.height == height &&
regions[0].bufferRowLength == width,
"linear RGBA8 storage upload lost Prospero pitch or allocation size");
std::printf("[host] %-32s ok\n", "StorageTextureLinearUpload");
}
void CheckStorageTextureDepthTileUploadLayout() {
constexpr auto format = Prospero::BufferFormat::k8UInt;
constexpr uint32_t width = 1;
constexpr uint32_t height = 1;
constexpr uint32_t depth = 1;
constexpr auto tile = Prospero::TileMode::kDepth;
const auto pitch = TileGetTexturePitch(format, width, tile);
TileSizeAlign slice{};
TileSizeAlign total{};
TileSizeOffset level{};
TilePaddedSize padded{};
TileGetTextureSize(format, width, height, 1, tile, &slice, &level, &padded);
TileGetTextureTotalSize(format, width, height, depth, 1, tile, false, total);
const auto layout =
MakeTilingImage(format, width, height, 1, depth, tile, total.size, false);
const auto regions = layout.BufferCopies();
Require("StorageTextureDepthTileUpload", "PPSA14053 layout",
pitch == 256 && padded.width == 256 && padded.height == 256 &&
slice.size == 0x10000 && slice.align == 0x10000 &&
level.size == slice.size && level.offset == 0 &&
total.size == slice.size && total.align == slice.align &&
layout.tile_mode == tile &&
layout.tiling.block.family == TileBlockFamily::Depth64KB &&
layout.pitch == pitch && layout.linear_slice_stride == 0 &&
layout.tiled_slice_stride == total.size &&
layout.mip_layout[0].linear_size == pitch &&
layout.mip_layout[0].size == total.size && regions.size() == 1 &&
regions[0].bufferOffset == 0 &&
regions[0].imageExtent.width == width &&
regions[0].imageExtent.height == height &&
regions[0].bufferRowLength == pitch,
"1x1 R8_UINT depth tile lost its 64 KiB source footprint");
std::printf("[host] %-32s ok\n", "StorageTextureDepthTileUpload");
}
void CheckImageSamplerSpecialization() {
ShaderSamplerResource filtered_sampler{};
filtered_sampler.fields[2] =
(1u << 20u) | (3u << 22u) | (2u << 24u) | (2u << 26u);
filtered_sampler.SetPointFiltering();
Require("ImageSamplerSpecialization", "forced point sampler",
filtered_sampler.XyMagFilter() == 0u &&
filtered_sampler.XyMinFilter() == 0u &&
filtered_sampler.ZFilter() == 1u &&
filtered_sampler.MipFilter() == 1u,
"point-only image format retained a filtered sampler mode");
ShaderSamplerResource base_mip_sampler{};
base_mip_sampler.fields[2] = (3u << 20u) | (1u << 22u) | (2u << 24u);
base_mip_sampler.SetPointFiltering();
Require("ImageSamplerSpecialization", "base-mip point sampler",
base_mip_sampler.XyMagFilter() == 0u &&
base_mip_sampler.XyMinFilter() == 0u &&
base_mip_sampler.ZFilter() == 1u &&
base_mip_sampler.MipFilter() == 0u,
"forcing point filtering enabled mipmapping on a base-mip sampler");
ShaderRecompiler::IR::Program mixed_sampler_program;
mixed_sampler_program.resource_tracking_complete = true;
ShaderRecompiler::IR::ImageResource sampled_image;
sampled_image.resource_class =
ShaderRecompiler::IR::ImageResourceClass::Sampled;
sampled_image.dimension = ShaderRecompiler::Decoder::ImageDimension::Dim2D;
sampled_image.read = true;
mixed_sampler_program.info.images = {sampled_image, sampled_image,
sampled_image, sampled_image};
mixed_sampler_program.info.samplers.push_back({0u, 4u});
mixed_sampler_program.info.sampled_pairs = {
{0u, 0u, 8u}, {1u, 0u, 12u}, {2u, 0u, 16u}, {3u, 0u, 20u}};
ShaderRecompiler::IR::MemoryInfo signed_memory;
signed_memory.kind = ShaderRecompiler::IR::ResourceKind::Image;
signed_memory.resource = 2u;
signed_memory.sampler = 0u;
mixed_sampler_program.memory_info.push_back(signed_memory);
mixed_sampler_program.block_storage.push_back(
std::make_unique<ShaderRecompiler::IR::Block>());
auto *mixed_sampler_block = mixed_sampler_program.block_storage.back().get();
mixed_sampler_program.blocks.push_back(mixed_sampler_block);
using ShaderRecompiler::IR::Value;
using ShaderRecompiler::IR::ValueOpcode;
auto &image_handle = mixed_sampler_block->AppendNewInst(
ValueOpcode::GetImageResource,
{Value(0u), Value(0u), Value(0u), Value(0u), Value(0u), Value(0u),
Value(0u), Value(0u)});
auto &sampler_handle = mixed_sampler_block->AppendNewInst(
ValueOpcode::GetSamplerResource,
{Value(0u), Value(0u), Value(0u), Value(0u)});
auto &image_address = mixed_sampler_block->AppendNewInst(
ValueOpcode::MakeImageAddress,
{Value(0u), Value(0u), Value(0u), Value(0u), Value(0u), Value(0u),
Value(0u), Value(0u), Value(0u), Value(0u), Value(0u), Value(0u),
Value(0u)});
const ShaderRecompiler::IR::MemoryFlags signed_memory_flags{0u, 16u};
uint64_t signed_memory_flag_bits = 0;
std::memcpy(&signed_memory_flag_bits, &signed_memory_flags,
sizeof(signed_memory_flags));
mixed_sampler_block->AppendNewInst(
ValueOpcode::ImageSampleRaw,
{Value(&image_handle), Value(&sampler_handle), Value(&image_address), Value(true)},
signed_memory_flag_bits);
ShaderRecompiler::IR::DescriptorValue native_image_descriptor{};
native_image_descriptor.dword_count = 8;
native_image_descriptor.dwords[0] = 0x1000u;
native_image_descriptor.dwords[1] =
static_cast<uint32_t>(Prospero::BufferFormat::k32Float) << 20u;
native_image_descriptor.dwords[2] = 3u | (3u << 14u);
native_image_descriptor.dwords[3] =
DstSel(4, 5, 6, 7) |
(static_cast<uint32_t>(Prospero::ImageType::kColor2D) << 28u);
auto packed_image_descriptor = native_image_descriptor;
packed_image_descriptor.dwords[0] = 0x2000u;
packed_image_descriptor.dwords[1] =
static_cast<uint32_t>(Prospero::BufferFormat::k11_11_10UInt) << 20u;
auto signed_image_descriptor = native_image_descriptor;
signed_image_descriptor.dwords[0] = 0x3000u;
signed_image_descriptor.dwords[1] =
static_cast<uint32_t>(Prospero::BufferFormat::k32SInt) << 20u;
auto unsigned_image_descriptor = native_image_descriptor;
unsigned_image_descriptor.dwords[0] = 0x4000u;
unsigned_image_descriptor.dwords[1] =
static_cast<uint32_t>(Prospero::BufferFormat::k32UInt) << 20u;
ShaderRecompiler::IR::DescriptorValue sampler_descriptor{};
sampler_descriptor.dword_count = 4;
const std::array descriptors{native_image_descriptor, packed_image_descriptor,
signed_image_descriptor, unsigned_image_descriptor,
sampler_descriptor};
mixed_sampler_program.descriptor_sources.resize(descriptors.size());
for (u32 source = 0; source < descriptors.size(); source++) {
auto &destination = mixed_sampler_program.descriptor_sources[source];
destination.dword_count = descriptors[source].dword_count;
for (u32 dword = 0; dword < destination.dword_count; dword++) {
destination.dwords[dword] = Value(descriptors[source].dwords[dword]);
}
}
for (u32 image = 0; image < mixed_sampler_program.info.images.size(); image++) {
mixed_sampler_program.info.images[image].source = image;
}
mixed_sampler_program.info.samplers[0].source = 4;
mixed_sampler_program.srt_plan_complete = true;
auto mixed_sampler_plan =
ShaderRecompiler::IR::ExtractResourcePlan(mixed_sampler_program);
ShaderRecompiler::IR::ResourceSnapshot mixed_sampler_snapshot;
ShaderRecompiler::IR::ResourceSpecialization mixed_sampler_specialization;
Require("ImageSamplerSpecialization", "resource materialization",
ShaderRecompiler::IR::MaterializeResources(
mixed_sampler_plan, {}, mixed_sampler_snapshot,
mixed_sampler_specialization),
"mixed sampler descriptors could not be materialized");
ShaderRecompiler::IR::ApplyResourceSpecialization(
mixed_sampler_program, mixed_sampler_specialization);
Require("ImageSamplerSpecialization", "mixed sampler variant",
mixed_sampler_program.info.samplers.size() == 2u &&
!mixed_sampler_program.info.samplers[0].force_point_filtering &&
!mixed_sampler_program.info.samplers[0].integer_border &&
mixed_sampler_program.info.samplers[1].force_point_filtering &&
mixed_sampler_program.info.samplers[1].integer_border &&
mixed_sampler_program.info.sampled_pairs[0].sampler == 0u &&
mixed_sampler_program.info.sampled_pairs[1].sampler == 1u &&
mixed_sampler_program.info.sampled_pairs[2].sampler == 1u &&
mixed_sampler_program.info.sampled_pairs[3].sampler == 1u &&
mixed_sampler_program.memory_info[0].sampler == 1u &&
mixed_sampler_snapshot.samplers.size() == 1u &&
std::ranges::all_of(mixed_sampler_program.info.samplers,
[](const auto& sampler) {
return sampler.source == 4u && sampler.snapshot_index == 0u;
}),
"a shared float/unsigned/signed sampler lost its border or filtering "
"class in the compiled bindings, instruction or descriptor snapshot");
for (const bool scalar : {false, true}) {
using namespace ShaderRecompiler::IR;
const char *name = scalar ? "ScalarMaterialImageDomain" : "GpuMaterialImageDomain";
constexpr u32 count = 70, table_base = 0x1000, material_base = 0x2000;
std::vector<u32> memory(4096);
for (u32 i = 0; i < count; ++i) {
auto descriptor = native_image_descriptor;
descriptor.dwords[0] += i * 0x100u;
std::copy(descriptor.dwords.begin(), descriptor.dwords.end(),
memory.begin() + table_base / 4u + i * 8u);
memory[material_base / 4u + i] = (i << 4u) | 3u;
}
memory[material_base / 4u + count] = (7u << 4u) | 15u; // Duplicate after shift.
memory[material_base / 4u + count + 1u] = 0x80000010u; // key<<5 wraps to entry1.
memory[material_base / 4u + count + 2u] = count << 4u; // Scalar table OOB is null.
std::array<u32, 64> user_data{};
const std::array<u32, 8> buffers{table_base, 32u << 16u, count, 0x5204u,
material_base, 4u << 16u, count + 3u, 0x5204u};
std::copy(buffers.begin(), buffers.end(), user_data.begin());
const auto output = MakeStructuredStorageBufferData(
0u, 64u, false, static_cast<u32>(Prospero::BufferFormat::k32UInt));
std::copy_n(output.begin(), 4, user_data.begin() + 48);
user_data[48] = 0x3000u;
if (scalar) user_data[30] = 1u;
std::vector<u32> code{EncodeVopc(0xc5, InlineU32(0), 0),
EncodeSop1(0x24, 24, 106), EncodeSopp(0x08, 0)};
const auto append_scalar_key = [&](bool numeric_use) {
code.insert(code.end(), {EncodeVop1(0x02, 20, Vgpr(0)),
EncodeSop2(0x00, 20, 20, 21),
EncodeSop2(0x26, 20, 20, InlineU32(3)),
EncodeSop2(0x1e, 20, 20, InlineU32(2)),
EncodeSop2(0x00, 20, 20, InlineU32(8)),
EncodeSmem0(0x08, 20, 2), EncodeSmem1(0, 20)});
if (numeric_use) code.push_back(EncodeVop1(0x01, 17, 20));
code.push_back(EncodeSop2(0x20, 20, 20, InlineU32(4)));
};
if (scalar) {
append_scalar_key(true);
} else {
code.insert(code.end(), {EncodeSop1(0x04, 26, 126), EncodeVop1(0x01, 16, InlineU32(1))});
// An earlier descriptor waterfall separates the EXECZ witness from the material load.
const auto previous_loop = code.size();
code.insert(code.end(), {EncodeSop1(0x04, 28, 126),
EncodeVop1(0x02, 22, Vgpr(0)), EncodeVopc(0xd2, 22, 0),
EncodeVop1(0x01, 16, InlineU32(0)),
EncodeSop1(0x04, 126, 28), EncodeVopc(0xd5, InlineU32(0), 16)});
code.push_back(EncodeSopp(0x09, static_cast<int16_t>(previous_loop - code.size() - 1u)));
code.insert(code.end(), {EncodeSop1(0x04, 126, 26),
EncodeMubuf0(0x0c, 0, true, false), EncodeMubuf1(15, 1, 0),
EncodeVop2(0x16, 15, InlineU32(4), 15),
EncodeVop1(0x01, 16, InlineU32(1))});
}
const auto image_loop = code.size();
if (!scalar)
code.insert(code.end(), {EncodeSop1(0x04, 28, 126),
EncodeVop1(0x02, 20, Vgpr(15)), EncodeVopc(0xd2, 20, 15)});
code.insert(code.end(), {EncodeSop2(0x1e, 20, 20, InlineU32(5)),
EncodeSmem0(0x0b, 12), EncodeSmem1(0, 20)});
const auto table_read_end = code.size();
code.insert(code.end(), {EncodeMimg0(0x24, 1), EncodeMimg1(4, 0, 3, 2)});
AppendBufferStoreDword(&code, 4, 0);
if (!scalar) {
code.insert(code.end(), {EncodeVop1(0x01, 16, InlineU32(0)),
EncodeSop1(0x04, 126, 28), EncodeVopc(0xd5, InlineU32(0), 16)});
code.push_back(EncodeSopp(0x09, static_cast<int16_t>(image_loop - code.size() - 1u)));
} else {
// Crossed exits introduce stale s16:19 Phi arms before the later s16:23 samples.
code.push_back(EncodeVop1(0x01, 5, InlineU32(0)));
code.push_back(EncodeSopc(0x06, 30, InlineU32(0)));
const auto skip_load = code.size();
code.push_back(EncodeSopp(0x05, 0));
append_scalar_key(false);
code.insert(code.end(), {EncodeSop2(0x1e, 20, 20, InlineU32(5)),
EncodeSmem0(0x0b, 16), EncodeSmem1(0, 20),
EncodeSopc(0x06, 31, InlineU32(0))});
const auto later_sample = code.size();
code.push_back(EncodeSopp(0x05, 0));
code.insert(code.end(), {EncodeMimg0(0x24, 1), EncodeMimg1(4, 0, 4, 2)});
const auto early_exit = code.size();
code.push_back(EncodeSopp(0x02, 0));
code[later_sample] = EncodeSopp(0x05, static_cast<int16_t>(code.size() - later_sample - 1u));
code.insert(code.end(), {EncodeMimg0(0x24, 1), EncodeMimg1(4, 0, 4, 2),
EncodeMimg0(0x24, 1), EncodeMimg1(5, 0, 4, 2)});
const auto leave_samples = code.size();
code.push_back(EncodeSopp(0x02, 0));
code[skip_load] = EncodeSopp(0x05, static_cast<int16_t>(code.size() - skip_load - 1u));
code.push_back(EncodeVop1(0x01, 4, InlineU32(0)));
code[early_exit] = EncodeSopp(0x02, static_cast<int16_t>(code.size() - early_exit - 1u));
code[leave_samples] = EncodeSopp(0x02, static_cast<int16_t>(code.size() - leave_samples - 1u));
AppendBufferStoreDword(&code, 4, 0);
AppendBufferStoreDword(&code, 5, 0);
AppendBufferStoreDword(&code, 17, 0);
}
code[2] = EncodeSopp(0x08, static_cast<int16_t>(code.size() - 3u));
AppendEnd(&code);
ShaderComputeInputInfo compute{};
compute.wave_size = 32;
compute.threads_num[0] = 32;
compute.thread_ids_num = 1;
ShaderRecompiler::CompileOptions options;
options.stage = ShaderType::Compute;
options.wave_size = 32;
options.user_data = user_data;
options.input_info.compute = &compute;
auto translated = ShaderRecompiler::TranslateProgram(code, options);
auto plan = ExtractResourcePlan(translated.program);
ResourceSnapshot snapshot;
ResourceSpecialization specialization;
const SrtRuntime runtime{.user_data = user_data, .read_memory = ReadTestMemory,
.userdata = &memory, .read_specialization_memory = ReadTestMemory};
Require(name, "bounded native material plan",
plan.capture_specialization_reads && !translated.program.has_address_writes &&
MaterializeResources(plan, runtime, snapshot, specialization) &&
snapshot.images.size() == count + 1u,
"GPU-selected material descriptors did not expand beyond native root capacity");
const auto offset = specialization.images[0].indirect_mapping_offset;
const auto ordinal = [&](u32 byte_offset) {
const auto word = byte_offset >> 2u;
return word < snapshot.flattened_srt[offset]
? snapshot.flattened_srt[offset + 1u + word] : 0u;
};
Require(name, "dense table map and coherent snapshot",
specialization.images[0].indirect_search_iterations == 0u &&
snapshot.flattened_srt[offset] == (count - 1u) * 8u + 1u &&
ordinal(32u) != 0u &&
ordinal(count * 32u) == 0u && snapshot.images[0].dwords[0] == 0u &&
snapshot.specialization_reads ==
std::vector<std::pair<uint64_t, uint64_t>>{{table_base, count * 32u}},
"image table lost its dense offsets, explicit null or single coherent read");
const auto previous = specialization;
memory[table_base / 4u + 8u] += 0x10000u;
Require(name, "runtime descriptor refresh",
MaterializeResources(plan, runtime, snapshot, specialization) &&
specialization == previous && snapshot.images[ordinal(32u)].dwords[0] == 0x11100u,
"descriptor addresses were frozen or entered the shader specialization");
auto compiled = ShaderRecompiler::CompileProgram(std::move(translated), options, specialization);
ValidateSpirv(name, compiled.spirv);
if (!scalar) {
user_data[4] += 1u;
Require(name, "GPU material reads remain on GPU",
MaterializeResources(plan, runtime, snapshot, specialization) &&
specialization == previous && snapshot.specialization_reads ==
std::vector<std::pair<uint64_t, uint64_t>>{{table_base, count * 32u}},
"GPU material addressing reentered host descriptor selection");
user_data[4] -= 1u;
}
auto dirty = runtime;
dirty.read_specialization_memory = +[](void*, uint64_t, std::span<u32>) { return false; };
Require(name, "unreadable image table rejection",
!MaterializeResources(plan, dirty, snapshot, specialization),
"GPU-selected texture materialization bypassed strict read provenance");
if (scalar) {
code.insert(code.begin() + table_read_end, EncodeVop1(0x01, 18, 12));
code.pop_back();
AppendBufferStoreDword(&code, 18, 0);
code[2] = EncodeSopp(0x08, static_cast<int16_t>(code.size() - 3u));
AppendEnd(&code);
} else {
code[2] = EncodeSopp(0x00, 0); // GPU selectors do not need a host nonempty-EXEC witness.
}
auto retained = ShaderRecompiler::TranslateProgram(code, options);
auto retained_plan = ExtractResourcePlan(retained.program);
Require(name, "retained descriptor consumers",
MaterializeResources(retained_plan, runtime, snapshot, specialization),
"retained GPU descriptor reads could not use the shared image table");
compiled = ShaderRecompiler::CompileProgram(std::move(retained), options, specialization);
ValidateSpirv(name, compiled.spirv);
}
std::printf("[host] %-32s ok\n", "ImageSpecializationPipelineId");
}
void CheckResourcePlanHandoff() {
constexpr const char *name = "ResourcePlanHandoff";
using namespace ShaderRecompiler::IR;
for (const bool numeric_read : {false, true}) {
std::vector<u32> code{EncodeSmem0(0x02, 8), EncodeSmem1(0, 125),
EncodeSmem0(0x08, 12, 4), EncodeSmem1(0, 125)};
if (numeric_read) {
code.insert(code.end(), {EncodeSmem0(0x00, 13), EncodeSmem1(16, 125)});
AppendVop3(&code, 0x125u, 0, 12, 13); // v_add_nc_u32 v0, s12, s13
} else {
code.push_back(EncodeVop1(0x01, 0, 12));
}
code.insert(code.end(), {EncodeMubuf0(0x1c, 0, false, false),
EncodeMubuf1(0, 2, 0)});
AppendEnd(&code);
std::array<u32, 2> user_data{};
std::vector<u32> memory{0x1000u, 0u, 16u, 0x30000000u | DstSel(4, 5, 6, 7), 7u};
ShaderComputeInputInfo compute{};
ShaderRecompiler::CompileOptions options;
options.stage = ShaderType::Compute;
options.user_data = user_data;
options.input_info.compute = &compute;
auto translated = ShaderRecompiler::TranslateProgram(code, options);
auto plan = ExtractResourcePlan(translated.program);
ResourceSnapshot snapshot;
ResourceSpecialization specialization;
const SrtRuntime runtime{.user_data = user_data,
.read_memory = ReadTestMemory,
.userdata = &memory,
.read_specialization_memory = ReadTestMemory};
Require(name, "initial descriptors",
MaterializeResources(plan, runtime, snapshot, specialization) &&
snapshot.buffers.size() == 1 && snapshot.buffers[0].dwords[0] == 0x1000u,
"SRT buffer descriptor was not materialized");
auto compiled = ShaderRecompiler::CompileProgram(std::move(translated), options,
specialization);
ValidateSpirv(name, compiled.spirv);
const auto &bindings = compiled.program.bindings;
Require(name, "GPU data requirements",
compiled.program.info.user_data_registers.empty() && bindings.descriptor_counts[0] == 1 &&
(bindings.descriptor_counts[static_cast<size_t>(DescriptorBindingKind::FlattenedSrt)] != 0) ==
numeric_read,
"descriptor-only shader retained SRT uploads or live scalar data was removed");
memory[0] = 0x2000u;
memory[4] = 13u;
Require(name, "independent host plan",
MaterializeResources(plan, runtime, snapshot, specialization) &&
snapshot.buffers[0].dwords[0] == 0x2000u &&
(!numeric_read || snapshot.flattened_srt.back() == 13u),
"compiled shader cleanup invalidated or froze the host resource plan");
}
std::printf("[host] %-32s ok\n", name);
}
void CheckNativeImageDescriptorTypes() {
using ShaderRecompiler::IR::DescriptorBindingKind;
using ShaderRecompiler::IR::FirstImageBinding;
using ShaderRecompiler::IR::FirstStorageImageBinding;
for (uint32_t value = FirstImageBinding;
value < static_cast<uint32_t>(DescriptorBindingKind::Samplers);
value++) {
const auto expected = value < FirstStorageImageBinding
? vk::DescriptorType::eSampledImage
: vk::DescriptorType::eStorageImage;
Require("NativeImageDescriptorTypes", "image binding",
NativeDescriptorType(static_cast<DescriptorBindingKind>(value)) ==
expected,
"generated image binding has the wrong Vulkan descriptor type");
}
Require("NativeImageDescriptorTypes", "non-image bindings",
NativeDescriptorType(DescriptorBindingKind::Samplers) ==
vk::DescriptorType::eSampler &&
NativeDescriptorType(DescriptorBindingKind::Buffers) ==
vk::DescriptorType::eStorageBuffer &&
NativeDescriptorType(DescriptorBindingKind::Gds) ==
vk::DescriptorType::eStorageBuffer &&
NativeDescriptorType(DescriptorBindingKind::BdaPagetable) ==
vk::DescriptorType::eStorageBuffer &&
NativeDescriptorType(DescriptorBindingKind::FaultBuffer) ==
vk::DescriptorType::eStorageBuffer &&
NativeDescriptorType(DescriptorBindingKind::FlattenedSrt) ==
vk::DescriptorType::eStorageBuffer &&
NativeDescriptorType(DescriptorBindingKind::ShaderData) ==
vk::DescriptorType::eStorageBuffer,
"non-image binding has the wrong Vulkan descriptor type");
std::printf("[host] %-32s ok\n", "NativeImageDescriptorTypes");
}
#if KYTY_PLATFORM != KYTY_PLATFORM_WINDOWS
void CheckShaderRecompilerFatalContracts() {
ExpectFatal("InvalidDescriptorBindingRejection", [] {
(void)NativeDescriptorType(
ShaderRecompiler::IR::DescriptorBindingKind::Count);
});
ExpectFatal("WritableFlatStoreRejection", [] {
const auto test = FlatStoreVariants();
(void)CompileCase(test);
});
ExpectFatal("GdsOrB64Rejection", [] {
TestCase test;
test.name = "GdsOrB64Rejection";
test.code = {EncodeDs0(0x4a, 0, true), EncodeDs1(0, 3, 7)};
AppendEnd(&test.code);
test.opcodes = {ShaderOpcode::DS_OR_B64, ShaderOpcode::S_ENDPGM};
(void)CompileCase(test);
});
{
ShaderRecompiler::IR::Program program;
program.resource_tracking_complete = true;
ShaderRecompiler::IR::ImageResource image;
image.resource_class = ShaderRecompiler::IR::ImageResourceClass::Storage;
image.numeric_class = Prospero::TextureNumericClass::Uint;
image.dimension = ShaderRecompiler::Decoder::ImageDimension::Dim2D;
image.read = true;
image.written = true;
image.atomic = true;
program.info.images.push_back(image);
ShaderRecompiler::IR::DescriptorValue descriptor{};
descriptor.dword_count = 8;
descriptor.dwords[0] = 0x2000u;
descriptor.dwords[1] =
static_cast<uint32_t>(Prospero::BufferFormat::k11_11_10UInt) << 20u;
descriptor.dwords[2] = 3u | (3u << 14u);
descriptor.dwords[3] =
DstSel(4, 5, 6, 7) |
(static_cast<uint32_t>(Prospero::ImageType::kColor2D) << 28u);
image.source = 0;
program.info.images[0].source = 0;
program.descriptor_sources.resize(1);
program.descriptor_sources[0].dword_count = descriptor.dword_count;
for (u32 dword = 0; dword < descriptor.dword_count; dword++) {
program.descriptor_sources[0].dwords[dword] =
ShaderRecompiler::IR::Value(descriptor.dwords[dword]);
}
program.srt_plan_complete = true;
auto plan = ShaderRecompiler::IR::ExtractResourcePlan(program);
ShaderRecompiler::IR::ResourceSnapshot snapshot;
ShaderRecompiler::IR::ResourceSpecialization specialization;
Require("PackedAtomicImageRejection", "resource materialization",
!ShaderRecompiler::IR::MaterializeResources(
plan, {}, snapshot, specialization),
"packed atomic image descriptor was accepted");
}
}
#endif
void CheckStorageTextureVolumeUploadLayout() {
constexpr auto format = Prospero::BufferFormat::k16_16_16_16Float;
constexpr uint32_t width = 33;
constexpr uint32_t height = 33;
constexpr uint32_t depth = 33;
const auto pitch =
TileGetTexturePitch(format, width, Prospero::TileMode::kRenderTarget);
TileSizeAlign total{};
TileGetTextureTotalSize(format, width, height, depth, 1,
Prospero::TileMode::kRenderTarget, true, total);
const auto layout =
MakeTilingImage(format, width, height, 1, depth,
Prospero::TileMode::kRenderTarget, total.size, true);
const auto regions = layout.BufferCopies();
Require("StorageTextureVolumeUpload", "layout",
pitch == 128 && total.size == 0x210000 &&
layout.linear_slice_stride == 0 &&
layout.tiled_slice_stride == 0x10000 &&
layout.mip_layout[0].linear_size == 0x2208 &&
layout.mip_layout[0].size == 0x10000 && regions.size() == 1 &&
regions[0].imageExtent.depth == depth,
"3D render-target upload did not preserve its compact linear layout");
Require("StorageTextureVolumeUpload", "GPU records",
layout.resources.levels == 1 && layout.MipExtent(0).depth == depth,
"3D render-target canonical mip record lost its depth");
for (const uint32_t z : {0u, 1u, depth - 1u}) {
const auto &mip = layout.mip_layout[0];
Require("StorageTextureVolumeUpload", "slice offsets",
mip.linear_offset + z * mip.linear_size == uint64_t{z} * 0x2208 &&
mip.offset + z * layout.tiled_slice_stride ==
uint64_t{z} * 0x10000 &&
mip.surface_z + z == z && mip.linear_pitch == width,
"volume slice lost its linear stride, block slice, or Z swizzle");
}
std::printf("[host] %-32s ok\n", "StorageTextureVolumeUpload");
}
void CheckStorageTextureVolumeMipRegions() {
constexpr auto format = Prospero::BufferFormat::k8_8_8_8UNorm;
constexpr uint32_t width = 8;
constexpr uint32_t height = 4;
constexpr uint32_t depth = 5;
constexpr uint32_t levels = 3;
constexpr auto tile = Prospero::TileMode::kLinear;
TileSizeAlign total{};
TileGetTextureTotalSize(format, width, height, depth, levels, tile, true,
total);
const auto layout = MakeTilingImage(format, width, height, levels, depth,
tile, total.size, true);
const auto copies = layout.BufferCopies();
bool valid = copies.size() == levels;
for (uint32_t level = 0; level < levels && valid; ++level) {
const auto &copy = copies[level];
const auto &mip = layout.mip_layout[level];
const auto active = layout.MipExtent(level);
valid &= copy.imageSubresource.mipLevel == level &&
copy.imageOffset.z == 0 &&
copy.imageExtent.width == active.width &&
copy.imageExtent.height == active.height &&
copy.imageExtent.depth == active.depth &&
copy.bufferOffset == mip.linear_offset &&
copy.bufferRowLength == mip.linear_pitch &&
uint64_t{copy.bufferRowLength} * copy.bufferImageHeight *
sizeof(uint32_t) ==
layout.linear_slice_stride;
}
const auto saved_mips = layout.mip_layout;
const auto prefix = layout.BufferCopies(1024, 1);
const auto again = layout.BufferCopies();
valid &= prefix.size() == 1 &&
prefix[0].bufferOffset == 1024 + layout.mip_layout[0].linear_offset &&
copies[0].bufferOffset == 768 && copies[1].bufferOffset == 256 &&
copies[2].bufferOffset == 0 && again == copies &&
layout.mip_layout == saved_mips;
Require("StorageTextureVolumeMipRegions", "per-mip depth", valid,
"direction-neutral image copies did not shrink depth or preserve "
"Vulkan Z coordinates");
std::printf("[host] %-32s ok\n", "StorageTextureVolumeMipRegions");
}
void CheckStandard64RenderTargetTileRoundTrip() {
constexpr auto format = Prospero::BufferFormat::k32Float;
constexpr auto tile = Prospero::TileMode::kStandard64KB;
constexpr uint32_t observed_width = 3840;
constexpr uint32_t observed_height = 2160;
const auto observed_pitch = TileGetTexturePitch(format, observed_width, tile);
TileSizeAlign observed{};
TileGetTextureSize(format, observed_width, observed_height, 1, tile,
&observed, nullptr, nullptr);
Require("Standard64RenderTarget", "observed layout",
observed_pitch == 3840 && observed.size == 0x1fe0000 &&
observed.align == 0x10000,
"PPSA02721 Standard64KB render-target footprint changed");
constexpr uint32_t width = 257;
constexpr uint32_t height = 131;
const auto pitch = TileGetTexturePitch(format, width, tile);
TileSizeAlign storage{};
TileGetTextureSize(format, width, height, 1, tile, &storage, nullptr,
nullptr);
Require("Standard64RenderTarget", "partial layout",
pitch == 384 && storage.size == 0x60000 && storage.align == 0x10000,
"partial Standard64KB footprint was not padded in 128x128 blocks");
std::printf("[host] %-32s ok\n", "Standard64RenderTarget");
}
#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
void CheckStorageTextureGpuOwnedRebindState() {
constexpr uintptr_t base = 0x0000000200200000ull;
constexpr uint64_t size = 0x10000;
const auto guest_memory = Libs::LibKernel::Memory::AllocateRuntimeMemory(
base, size, Common::VirtualMemory::Mode::ReadWrite,
"storage_texture_gpu_owned_rebind", true);
auto *memory = reinterpret_cast<uint8_t *>(guest_memory);
Require("StorageTextureGpuOwnedRebind", "allocation", guest_memory == base,
"fixed guest-owner allocation failed");
PageManager page_manager;
MemoryTracker tracker(page_manager);
tracker.ForEachUploadRange(
base, size, true, [](uint64_t, uint64_t) noexcept {}, []() noexcept {});
uint64_t readable = 0;
uint64_t mapped = 0;
MEMORY_BASIC_INFORMATION protection{};
Require(
"StorageTextureGpuOwnedRebind", "owned",
tracker.IsRegionGpuModified(base, size) &&
(!HostMemoryQueryReadable(base, size, readable) || readable < size) &&
HostMemoryQueryRange(base, size, HostMemoryAccess::Mapped, mapped) &&
mapped == size &&
VirtualQuery(memory, &protection, sizeof(protection)) != 0 &&
protection.Protect == PAGE_NOACCESS,
"GPU-owned storage pages remained host-readable or lost tracker "
"identity");
tracker.UnmarkRegionAsGpuModified(base, size);
readable = 0;
Require("StorageTextureGpuOwnedRebind", "clean readback",
!tracker.IsRegionGpuModified(base, size) &&
!tracker.IsRegionCpuModified(base, size) &&
HostMemoryQueryReadable(base, size, readable) && readable == size,
"clean storage readback did not publish readable coherent backing");
tracker.MarkRegionAsGpuModified(base, size);
readable = 0;
Require(
"StorageTextureGpuOwnedRebind", "clean reclaim",
tracker.IsRegionGpuModified(base, size) &&
!tracker.IsRegionCpuModified(base, size) &&
(!HostMemoryQueryReadable(base, size, readable) || readable < size),
"clean storage rebind did not reclaim GPU ownership without an upload");
tracker.UnmarkRegionAsGpuModified(base, size);
tracker.MarkRegionAsCpuModified(base, size);
uint32_t dirty_ranges = 0;
bool upload_called = false;
tracker.ForEachUploadRange(
base, size, true, [&](uint64_t, uint64_t) noexcept { dirty_ranges++; },
[&]() noexcept { upload_called = true; });
readable = 0;
Require(
"StorageTextureGpuOwnedRebind", "dirty refresh",
dirty_ranges == 1 && upload_called &&
tracker.IsRegionGpuModified(base, size) &&
!tracker.IsRegionCpuModified(base, size) &&
(!HostMemoryQueryReadable(base, size, readable) || readable < size),
"CPU-dirty storage rebind did not refresh once and reclaim GPU "
"ownership");
tracker.UnmarkRegionAsGpuModified(base, size);
tracker.UntrackMemory(base, size);
Require("StorageTextureGpuOwnedRebind", "free",
Libs::LibKernel::Memory::FreeGuestMemory(base, size),
"guest-owner free failed");
std::printf("[host] %-32s ok\n", "StorageTextureGpuOwnedRebind");
}
#endif
void CheckNativeMsaaState() {
Require("NativeMsaaState", "sample encoding",
render_sample_count(0) == 1 && render_sample_count(1) == 2 &&
render_sample_count(2) == 4 && render_sample_count(3) == 8 &&
render_sample_count(4) == 0,
"PS5 sample encodings were not mapped exactly");
Require("NativeMsaaState", "Vulkan sample mapping",
vulkan_sample_count(1) == vk::SampleCountFlagBits::e1 &&
vulkan_sample_count(2) == vk::SampleCountFlagBits::e2 &&
vulkan_sample_count(4) == vk::SampleCountFlagBits::e4 &&
vulkan_sample_count(8) == vk::SampleCountFlagBits::e8 &&
vulkan_sample_count(3) == vk::SampleCountFlagBits{},
"native sample counts were not mapped exactly to Vulkan");
TileSizeAlign color{};
const auto color_pitch = TileGetRenderTargetPitch(1920, 8, 3);
Require("NativeMsaaState", "8x color footprint",
color_pitch == 1920 &&
TileGetRenderTargetSize(1920, 1080, color_pitch, 8, color, 3) &&
color.align == 0x10000 && color.size == 0x07f80000,
"8x R16G16B16A16 color footprint was not preserved");
TileSizeAlign depth{};
TileSizeAlign stencil{};
TileSizeAlign htile{};
Require("NativeMsaaState", "8x depth/stencil footprint",
TileGetDepthPitch(1920, 4, 3) == 1920 &&
TileGetDepthSize(1920, 1080, 0, Prospero::DepthFormat::kZ32F,
Prospero::StencilFormat::k8UInt, true, stencil,
htile, depth, 3) &&
depth.align == 0x10000 && depth.size == 0x03fc0000 &&
stencil.align == 0x10000 && stencil.size == 0x010e0000 &&
htile.align == 0x8000 && htile.size == 0x00030000,
"8x depth/stencil or fragment-independent HTile footprint regressed");
std::printf("[host] %-32s ok\n", "NativeMsaaState");
}
void CheckDepthHtileStencilCompatibility() {
Require("DepthHtileStencilCompatibility", "inactive stencil plane",
depth_htile_stencil_acceleration_compatible(false, true, false),
"inactive Hi-Stencil state was treated as an attachment");
Require("DepthHtileStencilCompatibility", "disabled acceleration",
depth_htile_stencil_acceleration_compatible(true, false, true),
"disabled Hi-Stencil state was rejected");
Require("DepthHtileStencilCompatibility", "PS5 stencil plus HTile",
depth_htile_stencil_acceleration_compatible(true, true, false),
"valid PS5 Hi-Stencil attachment was rejected");
Require("DepthHtileStencilCompatibility", "missing HTile metadata",
!depth_htile_stencil_acceleration_compatible(true, false, false),
"active Hi-Stencil without HTile metadata was admitted");
std::printf("[host] %-32s ok\n", "DepthHtileStencilCompatibility");
}
void CheckPs5DepthRegisterDecoding() {
constexpr uint32_t captured_z = 0x22900803u;
constexpr uint32_t captured_stencil = 0x00100801u;
const auto z = HW::DepthZInfo::Decode(captured_z);
const auto stencil = HW::DepthStencilInfo::Decode(captured_stencil);
Require(
"Ps5DepthRegisterDecoding", "captured texture-compatible state",
z.format == Prospero::DepthFormat::kZ32F && z.num_samples == 0 &&
z.htile_acceleration && !z.expclear_enabled &&
!z.partially_resident && z.max_mip_level == 0 &&
z.z_compare_base == Prospero::ZCompareBase::kZMin &&
z.texture_compatibility ==
Prospero::TextureCompatiblePlaneCompression::kEnable &&
z.HasValidTextureCompatibility() &&
stencil.format == Prospero::StencilFormat::k8UInt &&
!stencil.htile_stencil_disabled && !stencil.expclear_enabled &&
!stencil.partially_resident &&
stencil.texture_compatibility ==
Prospero::TextureCompatibleStencil::kEnable &&
stencil.HasValidTextureCompatibility(),
"valid PS5 depth/stencil aggregate encodings were not decoded exactly");
const auto malformed_z = HW::DepthZInfo::Decode(0x00000803u);
const auto malformed_stencil = HW::DepthStencilInfo::Decode(0x00100001u);
Require("Ps5DepthRegisterDecoding", "malformed aggregate state",
!malformed_z.HasValidTextureCompatibility() &&
!malformed_stencil.HasValidTextureCompatibility(),
"partial texture-compatible aggregate encodings were accepted");
std::printf("[host] %-32s ok\n", "Ps5DepthRegisterDecoding");
}
void CheckDepthAttachmentWrites() {
RenderDepthInfo target{};
target.desc.view_info.format = vk::Format::eD32SfloatS8Uint;
Require("DepthAttachmentWrites", "disabled depth test",
!target.AttachmentWriteAspects(),
"disabled depth testing claimed a depth write");
target.depth_test_enable = true;
target.depth_write_enable = true;
target.depth_compare_op = vk::CompareOp::eLess;
Require("DepthAttachmentWrites", "depth write",
target.AttachmentWriteAspects() == vk::ImageAspectFlagBits::eDepth,
"write-enabled depth testing did not claim depth");
target.depth_test_enable = false;
target.depth_write_enable = false;
target.depth_load_clear_enable = true;
Require("DepthAttachmentWrites", "depth clear",
target.AttachmentWriteAspects() == vk::ImageAspectFlagBits::eDepth,
"depth attachment clear did not claim depth");
target.depth_load_clear_enable = false;
target.stencil_test_enable = true;
target.stencil_front = {vk::StencilOp::eKeep, vk::StencilOp::eKeep,
vk::StencilOp::eKeep, vk::CompareOp::eAlways, 0xff, 0xff, 0};
target.stencil_back = target.stencil_front;
Require("DepthAttachmentWrites", "stencil keep",
!target.AttachmentWriteAspects(),
"KEEP-only stencil state claimed a stencil write");
target.stencil_front.failOp = vk::StencilOp::eZero;
Require("DepthAttachmentWrites", "unreachable stencil fail",
!target.AttachmentWriteAspects(),
"ALWAYS comparison claimed an unreachable fail operation");
target.stencil_front.failOp = vk::StencilOp::eKeep;
target.stencil_front.passOp = vk::StencilOp::eReplace;
Require("DepthAttachmentWrites", "stencil pass write",
target.AttachmentWriteAspects() == vk::ImageAspectFlagBits::eStencil,
"write-capable stencil pass did not claim stencil");
target.stencil_front.writeMask = 0;
Require("DepthAttachmentWrites", "back-face keep",
!target.AttachmentWriteAspects(),
"masked front write or KEEP-only back face claimed stencil");
target.stencil_test_enable = false;
target.stencil_clear_enable = true;
Require("DepthAttachmentWrites", "stencil clear",
target.AttachmentWriteAspects() == vk::ImageAspectFlagBits::eStencil,
"stencil attachment clear did not claim stencil");
target.desc.view_info.format = vk::Format::eD32Sfloat;
Require("DepthAttachmentWrites", "missing stencil aspect",
!target.AttachmentWriteAspects(),
"depth-only format claimed a stencil write");
std::printf("[host] %-32s ok\n", "DepthAttachmentWrites");
}
void CheckDepthFeedbackAspects() {
RenderDepthInfo target{};
target.desc.view_info.format = vk::Format::eD32SfloatS8Uint;
target.desc.view_info.base_level = 1;
target.desc.view_info.level_count = 2;
target.desc.view_info.base_layer = 3;
target.desc.view_info.layer_count = 2;
target.depth_test_enable = true;
target.depth_write_enable = true;
target.depth_compare_op = vk::CompareOp::eAlways;
target.stencil_test_enable = true;
target.stencil_front = {vk::StencilOp::eKeep, vk::StencilOp::eReplace,
vk::StencilOp::eKeep, vk::CompareOp::eAlways, 0xff, 0xff, 0};
target.stencil_back = target.stencil_front;
auto sampled = target.desc.view_info;
const auto feedback = [&](const ImageViewInfo& view) {
return DepthFeedbackAspects(target.AttachmentWriteAspects(),
target.desc.view_info, view);
};
sampled.aspect = vk::ImageAspectFlagBits::eDepth;
Require("DepthFeedbackAspects", "sampled writable depth",
feedback(sampled) == vk::ImageAspectFlagBits::eDepth,
"overlapping depth sampling did not request depth feedback");
sampled.aspect = vk::ImageAspectFlagBits::eStencil;
Require("DepthFeedbackAspects", "sampled writable stencil",
feedback(sampled) == vk::ImageAspectFlagBits::eStencil,
"overlapping stencil sampling did not request stencil feedback");
sampled.aspect = vk::ImageAspectFlagBits::eDepth | vk::ImageAspectFlagBits::eStencil;
Require("DepthFeedbackAspects", "both writable aspects",
feedback(sampled) == sampled.aspect,
"combined depth/stencil sampling lost a writable aspect");
target.depth_write_enable = false;
Require("DepthFeedbackAspects", "read-only depth",
feedback(sampled) == vk::ImageAspectFlagBits::eStencil,
"read-only depth was included in stencil feedback");
target.depth_write_enable = true;
target.stencil_test_enable = false;
Require("DepthFeedbackAspects", "read-only stencil",
feedback(sampled) == vk::ImageAspectFlagBits::eDepth,
"read-only stencil was included in depth feedback");
sampled.aspect = vk::ImageAspectFlagBits::eStencil;
Require("DepthFeedbackAspects", "sampled read-only aspect",
!feedback(sampled),
"sampling a read-only stencil requested feedback for depth writes");
sampled.aspect = vk::ImageAspectFlagBits::eDepth;
sampled.base_level = 3;
Require("DepthFeedbackAspects", "disjoint mip",
!feedback(sampled),
"disjoint mip ranges requested depth feedback");
sampled.base_level = 2;
sampled.base_layer = 5;
Require("DepthFeedbackAspects", "disjoint layer",
!feedback(sampled),
"disjoint layer ranges requested depth feedback");
sampled.base_layer = 4;
Require("DepthFeedbackAspects", "overlapping range boundary",
feedback(sampled) == vk::ImageAspectFlagBits::eDepth,
"overlapping mip and layer boundaries missed depth feedback");
target.depth_write_enable = false;
target.depth_load_clear_enable = true;
Require("DepthFeedbackAspects", "depth clear only",
feedback(sampled) == vk::ImageAspectFlagBits::eDepth,
"a sampled depth attachment clear missed feedback");
target.depth_load_clear_enable = false;
target.stencil_clear_enable = true;
sampled.aspect = vk::ImageAspectFlagBits::eStencil;
Require("DepthFeedbackAspects", "stencil clear only",
feedback(sampled) == vk::ImageAspectFlagBits::eStencil,
"a sampled stencil attachment clear missed feedback");
std::printf("[host] %-32s ok\n", "DepthFeedbackAspects");
}
void CheckDynamicRenderingState() {
RenderState first{};
first.width = 64;
first.height = 32;
first.num_layers = 2;
first.num_color_attachments = 1;
first.color_attachments[0].image_layout =
vk::ImageLayout::eColorAttachmentOptimal;
first.color_attachments[0].clear_value = {1, 2, 3, 4};
first.color_attachments[0].is_clear = true;
auto changed = first;
Require("DynamicRenderingState", "identical state", first == changed,
"fieldwise RenderState identity rejected an identical scope");
changed.color_attachments[0].is_clear = false;
Require("DynamicRenderingState", "clear identity", first != changed,
"clear/load state did not participate in rendering identity");
changed = first;
changed.num_layers = 1;
Require("DynamicRenderingState", "layer identity", first != changed,
"layer count did not participate in rendering identity");
changed = first;
changed.color_attachments[0].image_layout = vk::ImageLayout::eGeneral;
Require("DynamicRenderingState", "layout identity", first != changed,
"attachment layout did not participate in rendering identity");
PipelineRenderingState rgba{};
rgba.color_count = 1;
rgba.color_formats[0] = vk::Format::eR8G8B8A8Unorm;
auto uint_color = rgba;
uint_color.color_formats[0] = vk::Format::eR8G8B8A8Uint;
Require("DynamicRenderingState", "pipeline color format identity",
rgba != uint_color,
"distinct dynamic-rendering formats can alias one pipeline key");
auto depth = rgba;
depth.depth_format = vk::Format::eD32SfloatS8Uint;
depth.stencil_format = vk::Format::eD32SfloatS8Uint;
Require("DynamicRenderingState", "pipeline depth/stencil identity",
rgba != depth,
"depth/stencil formats did not participate in pipeline identity");
RenderDepthInfo attachment{};
attachment.desc.view_info.format = vk::Format::eD32SfloatS8Uint;
Require("DynamicRenderingState", "read-only depth/stencil layout",
depth_attachment_layout(attachment) ==
vk::ImageLayout::eDepthStencilReadOnlyOptimal,
"fully read-only depth/stencil used a writable layout");
attachment.depth_load_clear_enable = true;
Require("DynamicRenderingState", "deferred depth clear layout",
depth_attachment_layout(attachment) ==
vk::ImageLayout::eDepthAttachmentStencilReadOnlyOptimal,
"a depth load clear used a read-only attachment layout");
attachment.depth_test_enable = true;
attachment.depth_write_enable = true;
Require("DynamicRenderingState", "depth-write stencil-read layout",
depth_attachment_layout(attachment) ==
vk::ImageLayout::eDepthAttachmentStencilReadOnlyOptimal,
"depth-only writes did not retain read-only stencil");
attachment.depth_load_clear_enable = false;
attachment.depth_write_enable = false;
attachment.stencil_clear_enable = true;
Require("DynamicRenderingState", "depth-read stencil-write layout",
depth_attachment_layout(attachment) ==
vk::ImageLayout::eDepthReadOnlyStencilAttachmentOptimal,
"stencil-only writes did not retain read-only depth");
attachment.depth_write_enable = true;
Require("DynamicRenderingState", "writable depth/stencil layout",
depth_attachment_layout(attachment) ==
vk::ImageLayout::eDepthStencilAttachmentOptimal,
"combined depth/stencil writes did not use the writable layout");
std::printf("[host] %-32s ok\n", "DynamicRenderingState");
}
void CheckDepthTargetFootprints() {
TileSizeAlign stencil{};
TileSizeAlign htile{};
TileSizeAlign depth{};
struct AttachmentFormatCase {
const char *name;
Prospero::DepthFormat depth_format;
Prospero::StencilFormat stencil_format;
vk::Format expected;
};
constexpr AttachmentFormatCase attachment_cases[] = {
{"Z16", Prospero::DepthFormat::kZ16, Prospero::StencilFormat::kInvalid,
vk::Format::eD16Unorm},
{"Z16S8", Prospero::DepthFormat::kZ16, Prospero::StencilFormat::k8UInt,
vk::Format::eD16UnormS8Uint},
{"Z32", Prospero::DepthFormat::kZ32F, Prospero::StencilFormat::kInvalid,
vk::Format::eD32Sfloat},
{"Z32S8", Prospero::DepthFormat::kZ32F, Prospero::StencilFormat::k8UInt,
vk::Format::eD32SfloatS8Uint},
{"invalid depth", static_cast<Prospero::DepthFormat>(2u),
Prospero::StencilFormat::k8UInt, vk::Format::eUndefined},
{"invalid stencil", Prospero::DepthFormat::kZ16,
static_cast<Prospero::StencilFormat>(2u), vk::Format::eUndefined},
};
for (const auto &test : attachment_cases) {
Require("DepthTargetFootprints", test.name,
DepthAttachmentFormat(test.depth_format, test.stencil_format) ==
test.expected,
"PS5 depth/stencil attachment mapping changed");
}
Require("DepthTargetFootprints", "1920x1080 Z16S8 without HTile",
TileGetDepthSize(1920, 1080, 0, Prospero::DepthFormat::kZ16,
Prospero::StencilFormat::k8UInt, false, stencil,
htile, depth) &&
depth.size == 0x480000 && depth.align == 0x10000 &&
stencil.size == 0x280000 && stencil.align == 0x10000 &&
htile.size == 0,
"captured Z16S8 footprint disagrees with Prospero block rules");
struct TargetFormatCase {
const char *name;
vk::Format host_format;
Prospero::BufferFormat guest_format;
uint32_t bytes_per_element;
bool stencil;
bool supported;
bool readback;
};
constexpr TargetFormatCase target_cases[] = {
{"D16", vk::Format::eD16Unorm, Prospero::BufferFormat::k16UNorm, 2, false,
true, true},
{"D16S8", vk::Format::eD16UnormS8Uint, Prospero::BufferFormat::k16UNorm,
2, true, true, true},
{"D16 via D24S8", vk::Format::eD24UnormS8Uint,
Prospero::BufferFormat::k16UNorm, 2, true, true, true},
{"D16 via D32S8", vk::Format::eD32SfloatS8Uint,
Prospero::BufferFormat::k16UNorm, 2, true, true, true},
{"D32", vk::Format::eD32Sfloat, Prospero::BufferFormat::k32Float, 4,
false, true, true},
{"D32S8", vk::Format::eD32SfloatS8Uint, Prospero::BufferFormat::k32Float,
4, true, true, true},
{"D16 plus stencil mismatch", vk::Format::eD16Unorm,
Prospero::BufferFormat::k16UNorm, 2, true, false, false},
{"fallback without stencil", vk::Format::eD24UnormS8Uint,
Prospero::BufferFormat::k16UNorm, 2, false, false, false},
{"D32 guest via D24", vk::Format::eD24UnormS8Uint,
Prospero::BufferFormat::k32Float, 4, true, false, false},
{"D16 byte mismatch", vk::Format::eD16Unorm,
Prospero::BufferFormat::k16UNorm, 4, false, false, false},
};
for (const auto &test : target_cases) {
ImageInfo target{};
target.pixel_format = test.host_format;
target.guest_format = test.guest_format;
target.bytes_per_block = test.bytes_per_element;
target.stencil = test.stencil ? GuestRange{0x10000, 0x10000} : GuestRange{};
Require("DepthTargetFootprints", test.name,
IsSupportedDepthTargetFormat(target) == test.supported &&
IsSupportedDepthPlaneReadback(target) == test.readback,
"host/guest depth format or exact readback policy changed");
}
ImageInfo compressed_stencil{};
compressed_stencil.pixel_format = vk::Format::eD32SfloatS8Uint;
compressed_stencil.guest_format = Prospero::BufferFormat::k32Float;
compressed_stencil.bytes_per_block = 4;
compressed_stencil.stencil = {0x10000, 0x10000};
compressed_stencil.metadata.stencil_compressed = true;
Require("DepthTargetFootprints", "compressed stencil depth-plane readback",
IsSupportedDepthTargetFormat(compressed_stencil) &&
IsSupportedDepthPlaneReadback(compressed_stencil),
"stencil metadata incorrectly blocked independent depth-plane "
"readback");
Require("DepthTargetFootprints", "lexicographic subresource comparison",
ImageSubresources{2, 1} > ImageSubresources{1, 4} &&
ImageSubresources{1, 4} < ImageSubresources{2, 1},
"subresource ordering diverged from required overlap semantics");
struct TransferPlaneCase {
const char *name;
vk::Format attachment;
vk::Format transfer;
uint32_t bytes;
};
constexpr TransferPlaneCase transfer_cases[] = {
{"D16S8 transfer", vk::Format::eD16UnormS8Uint, vk::Format::eD16Unorm, 2},
{"D24S8 transfer", vk::Format::eD24UnormS8Uint,
vk::Format::eX8D24UnormPack32, 4},
{"D32S8 transfer", vk::Format::eD32SfloatS8Uint, vk::Format::eD32Sfloat,
4},
{"invalid transfer", vk::Format::eR16Unorm, vk::Format::eUndefined, 0},
};
for (const auto &test : transfer_cases) {
Require("DepthTargetFootprints", test.name,
DepthAspectTransferFormat(test.attachment) == test.transfer &&
DepthAspectTransferBytes(test.attachment) == test.bytes,
"combined depth transfer-plane layout changed");
}
struct PromotionCase {
const char *name;
uint16_t source;
uint32_t d24;
uint32_t d32;
};
constexpr PromotionCase promotion_cases[] = {
{"zero", 0, 0, 0},
{"midpoint", 0x8000, 0x00800080, 0x3f000080},
{"maximum", 0xffff, 0x00ffffff, 0x3f800000},
};
for (const auto &test : promotion_cases) {
Require("DepthTargetFootprints", test.name,
EncodeD16AsD24(test.source) == test.d24 &&
EncodeD16AsD32(test.source) == test.d32,
"D16 host promotion changed the represented depth value");
}
Require("DepthTargetFootprints", "640x360 Z32S8 without HTile",
TileGetDepthSize(640, 360, 0, Prospero::DepthFormat::kZ32F,
Prospero::StencilFormat::k8UInt, false, stencil,
htile, depth),
"valid non-HTile depth/stencil footprint was rejected");
Require(
"DepthTargetFootprints", "640x360 Prospero block sizes",
depth.size == 0xf0000 && depth.align == 0x10000 &&
stencil.size == 0x60000 && stencil.align == 0x10000 &&
htile.size == 0 && htile.align == 0,
"non-HTile depth/stencil footprint disagrees with Prospero block rules");
const auto depth_pitch = TileGetTexturePitch(Prospero::BufferFormat::k32Float,
640, Prospero::TileMode::kDepth);
TileSizeAlign texture_depth{};
TileGetTextureTotalSize(Prospero::BufferFormat::k32Float, 640, 360, 1, 1,
Prospero::TileMode::kDepth, false, texture_depth);
Require("DepthTargetFootprints", "640x360 generic depth tile",
depth_pitch == 640 && texture_depth.size == 0xf0000 &&
texture_depth.align == 0x10000,
"generic depth texture sizing bypassed 64 KiB block padding");
Require(
"DepthTargetFootprints", "960x540 Z32S8 with HTile",
TileGetDepthSize(960, 540, 0, Prospero::DepthFormat::kZ32F,
Prospero::StencilFormat::k8UInt, true, stencil, htile,
depth) &&
depth.size == 0x280000 && depth.align == 0x10000 &&
stencil.size == 0xc0000 && stencil.align == 0x10000 &&
htile.size == 0x10000 && htile.align == 0x8000,
"generic Prospero HTile block calculation rejected the title footprint");
Require("DepthTargetFootprints", "known HTile extent",
TileGetDepthSize(1280, 720, 0, Prospero::DepthFormat::kZ32F,
Prospero::StencilFormat::k8UInt, true, stencil,
htile, depth) &&
depth.size == 0x3c0000 && stencil.size == 0xf0000 &&
htile.size == 0x20000,
"validated 1280x720 HTile footprint regressed");
Require("DepthTargetFootprints", "PPSA06228 3840x2160 Z32S8 with HTile",
TileGetDepthSize(3840, 2160, 0, Prospero::DepthFormat::kZ32F,
Prospero::StencilFormat::k8UInt, true, stencil,
htile, depth) &&
depth.size == 0x1fe0000 && depth.align == 0x10000 &&
stencil.size == 0x870000 && stencil.align == 0x10000 &&
htile.size == 0xa0000 && htile.align == 0x8000,
"captured 3840x2160 depth/stencil/HTile footprint disagrees with "
"Prospero rules");
Require("DepthTargetFootprints", "invalid depth format",
!TileGetDepthSize(960, 540, 0, static_cast<Prospero::DepthFormat>(2u),
Prospero::StencilFormat::k8UInt, true, stencil,
htile, depth),
"unsupported depth format was silently admitted");
Require("DepthTargetFootprints", "invalid stencil format",
!TileGetDepthSize(960, 540, 0, Prospero::DepthFormat::kZ32F,
static_cast<Prospero::StencilFormat>(2u), true,
stencil, htile, depth),
"unsupported stencil format was silently admitted");
Require("DepthTargetFootprints", "invalid extent",
!TileGetDepthSize(0, 540, 0, Prospero::DepthFormat::kZ32F,
Prospero::StencilFormat::k8UInt, true, stencil,
htile, depth) &&
!TileGetDepthSize(16385, 540, 0, Prospero::DepthFormat::kZ32F,
Prospero::StencilFormat::k8UInt, true, stencil,
htile, depth),
"invalid HTile extent was silently admitted");
std::printf("[host] %-32s ok\n", "DepthTargetFootprints");
}
struct SlotLifetimeProbe {
explicit SlotLifetimeProbe(bool *destroyed) : destroyed(destroyed) {
if (destroyed == nullptr || *destroyed) {
EXIT("fence-lifetime probe has invalid construction state\n");
}
}
~SlotLifetimeProbe() { *destroyed = true; }
bool *destroyed = nullptr;
};
void CheckSlotVectorLifetime() {
bool destroyed = false;
Common::SlotVector<SlotLifetimeProbe> slots;
const auto first = slots.insert(&destroyed);
Require("SlotVectorLifetime", "inserted",
slots.is_allocated(first) && slots.try_get(first) != nullptr &&
!destroyed,
"slot insertion did not expose its cache-owned object");
slots.erase(first);
const bool first_destroyed = destroyed;
destroyed = false;
const auto second = slots.insert(&destroyed);
Require("SlotVectorLifetime", "generation",
first_destroyed && !destroyed && first.index == second.index &&
first != second && !slots.is_allocated(first) &&
slots.is_allocated(second),
"reused slot accepted a stale cache resource id");
std::printf("[host] %-32s ok\n", "SlotVectorLifetime");
}
void CheckEmbeddedFetchLaneSpill() {
std::vector<u32> code;
code.push_back(EncodeSMovB32(0, InlineU32(0)));
code.push_back(EncodeSmem0(0x02u, 20, 4));
code.push_back(EncodeSmem1(0)); // s_load_dwordx4 s[20:23], s[8:9]
AppendVop3(&code, 0x361u, 6, 20, InlineU32(0)); // v_writelane_b32 v6, s20, 0
code.push_back(EncodeSMovB32(20, InlineU32(0)));
AppendVop3(&code, 0x360u, 20, Vgpr(6),
InlineU32(0)); // v_readlane_b32 s20, v6, 0
code.push_back(EncodeVop2(0x01u, 0, Vgpr(8), 5));
code.push_back(EncodeMubuf0(0x03u, 0, true));
code.push_back(EncodeMubuf1(9, 5, 0));
AppendEnd(&code);
std::array<u32, 11> user_data{};
ShaderVertexInputInfo vertex;
vertex.fetch_embedded = true;
vertex.fetch_buffer_reg = 0;
vertex.resources_num = 1;
vertex.resources[0].fields[3] =
BufferFormat(Prospero::BufferFormat::k32_32_32_32Float) << 12u | DstSel(4, 5, 6, 7);
vertex.resources_dst[0].attr_id = 0;
vertex.resources_dst[0].registers_num = 4;
ShaderRecompiler::CompileOptions options;
options.stage = ShaderType::Vertex;
options.user_data_base = 8;
options.user_data = user_data;
options.input_info.vertex = &vertex;
auto translated = ShaderRecompiler::TranslateProgram(code, options);
auto resource_plan =
ShaderRecompiler::IR::ExtractResourcePlan(translated.program);
ShaderRecompiler::IR::ResourceSnapshot resources;
ShaderRecompiler::IR::ResourceSpecialization specialization;
const ShaderRecompiler::IR::SrtRuntime runtime{
.user_data = options.user_data,
.shader_base = reinterpret_cast<uint64_t>(code.data()),
};
Require("EmbeddedFetchLaneSpill", "resource materialization",
ShaderRecompiler::IR::MaterializeResources(
resource_plan, runtime, resources, specialization),
"lane-spilled fetch resources could not be materialized");
auto result = ShaderRecompiler::CompileProgram(
std::move(translated), options, specialization);
Require("EmbeddedFetchLaneSpill", "fetch rewrite",
result.program.info.vertex_fetch_components[0] == 4,
"lane-spilled fetch descriptor was not recognized and rewritten");
std::printf("[host] %-32s ok\n", "EmbeddedFetchLaneSpill");
}
void CheckReferenceClockScale() {
uint64_t value = 0;
Require("ReferenceClockScale", "zero",
Sync::ScaleReferenceClock(0, 3000000000ull, value) && value == 0,
"zero host tick did not produce a zero GPU clock");
Require("ReferenceClockScale", "fractional second",
Sync::ScaleReferenceClock(1500000000ull, 3000000000ull, value) &&
value == 50000000ull,
"host half-second did not scale to 50,000,000 ticks");
Require("ReferenceClockScale", "whole and fractional",
Sync::ScaleReferenceClock(3750000000ull, 3000000000ull, value) &&
value == 125000000ull,
"host 1.25 seconds did not scale to 125,000,000 ticks");
Require("ReferenceClockScale", "monotonic floor",
Sync::ScaleReferenceClock(3750000001ull, 3000000000ull, value) &&
value == 125000000ull,
"sub-reference-tick increment did not use a monotonic floor");
Require("ReferenceClockScale", "guards",
!Sync::ScaleReferenceClock(1, 0, value) &&
!Sync::ScaleReferenceClock(UINT64_MAX, 1, value),
"invalid frequency or overflow was accepted");
std::printf("[host] %-32s ok\n", "ReferenceClockScale");
}
void CheckClipControlDepthClipState() {
HW::ClipControl clip;
Require("ClipControlDepthClipState", "default", clip.IsZClipEnabled(),
"default paired Z clipping was not enabled");
clip.min_z_clip_disable = true;
Require("ClipControlDepthClipState", "asymmetric near",
!clip.IsZClipEnabled(),
"near-plane disable did not disable paired host Z clipping");
clip.min_z_clip_disable = false;
clip.max_z_clip_disable = true;
Require("ClipControlDepthClipState", "asymmetric far", !clip.IsZClipEnabled(),
"far-plane disable did not disable paired host Z clipping");
clip.min_z_clip_disable = true;
Require("ClipControlDepthClipState", "both disabled", !clip.IsZClipEnabled(),
"paired Z-clip disable was not represented");
std::printf("[host] %-32s ok\n", "ClipControlDepthClipState");
}
void CheckVulkan13FeatureRequirements() {
const auto features = WindowContext::RequiredVulkan13Features();
Require(
"Vulkan13FeatureRequirements", "production requirements",
features.sType == vk::StructureType::ePhysicalDeviceVulkan13Features &&
features.pNext == nullptr && features.dynamicRendering == VK_TRUE &&
features.synchronization2 == VK_TRUE,
"production device creation did not require dynamic rendering and "
"synchronization2 from one feature declaration");
std::printf("[host] %-32s ok\n", "Vulkan13FeatureRequirements");
}
void CheckPm4SyntheticOcclusionCounterDump(RenderContext &renderer) {
GraphicsInitJmpTables();
CommandProcessor processor(renderer, 0);
constexpr uint64_t untouched = 0x1122334455667788ull;
constexpr uint64_t ready_bit = 1ull << 63u;
alignas(16) std::array<std::array<uint64_t, 2>, 16> results;
for (auto &pair : results) {
pair.fill(untouched);
}
const auto dump = [&](uint64_t *address) {
const auto raw_address = reinterpret_cast<uint64_t>(address);
std::array<uint32_t, 4> packet{
KYTY_PM4(4, Pm4::IT_EVENT_WRITE, Pm4::R_ZERO), 0x00000139u,
static_cast<uint32_t>(raw_address),
static_cast<uint32_t>(raw_address >> 32u)};
Pm4Execution execution;
return processor.Process(execution, packet);
};
const auto begin_result = dump(&results[0][0]);
bool begin_written = begin_result == Pm4ProcessResult::Complete;
for (const auto &pair : results) {
begin_written &= pair[0] == ready_bit && pair[1] == untouched;
}
const auto end_result = dump(&results[0][1]);
bool end_written = end_result == Pm4ProcessResult::Complete;
uint64_t visible_samples = 0;
for (const auto &pair : results) {
end_written &= pair[0] == ready_bit && pair[1] == (ready_bit | 1u);
visible_samples += pair[1] - pair[0];
}
Require("Pm4SyntheticOcclusionCounterDump", "always-visible result",
begin_written && end_written && visible_samples == results.size(),
"EVENT_WRITE did not publish valid nonzero begin/end counters to "
"every PS5 DB");
std::printf("[host] %-32s ok\n", "Pm4SyntheticOcclusionCounterDump");
}
void CheckPm4Predication(RenderContext &renderer) {
constexpr const char *name = "Pm4Predication";
constexpr uint64_t ready = 1ull << 63u;
GraphicsInitJmpTables();
CommandProcessor processor(renderer, 0);
processor.BufferInit();
alignas(16) std::array<uint64_t, 32> query{};
alignas(16) uint64_t boolean = 0;
uint32_t predicated = 0;
uint32_t unconditional = 0;
const auto commands = [&](uint32_t op, uint32_t condition, uint32_t wait,
const void *source) {
const auto address = reinterpret_cast<uint64_t>(source);
const auto predicated_address = reinterpret_cast<uint64_t>(&predicated);
const auto unconditional_address = reinterpret_cast<uint64_t>(&unconditional);
return std::array<uint32_t, 14>{
// Captured packet form: c0022000 00010100 <query low> <query high>.
0xc0022000u, (op << 16u) | (wait << 12u) | (condition << 8u),
static_cast<uint32_t>(address), static_cast<uint32_t>(address >> 32u),
KYTY_PM4(5, Pm4::IT_WRITE_DATA, 0) | 1u, 0,
static_cast<uint32_t>(predicated_address),
static_cast<uint32_t>(predicated_address >> 32u), 11,
KYTY_PM4(5, Pm4::IT_WRITE_DATA, 0), 0,
static_cast<uint32_t>(unconditional_address),
static_cast<uint32_t>(unconditional_address >> 32u), 22};
};
const auto check = [&](const char *stage, uint32_t op, uint32_t condition,
uint32_t wait, const void *source, bool skip) {
predicated = unconditional = 0;
const auto packet = commands(op, condition, wait, source);
const auto tick = renderer.GetCommandScheduler().CurrentTick();
Pm4Execution execution;
Require(name, stage,
processor.Process(execution, packet) == Pm4ProcessResult::Complete &&
processor.ShouldSkipPredicatedPackets() == skip &&
predicated == (skip ? 0u : 11u) && unconditional == 22,
"predicate polarity, tagged packet execution, or packet consumption is wrong");
Require(name, stage, renderer.GetCommandScheduler().CurrentTick() == tick,
"predication unnecessarily submitted and drained the host queue");
};
struct Case {
uint64_t delta;
uint32_t condition;
bool skip;
};
constexpr Case cases[]{{1, 1, false}, {1, 0, true},
{0, 1, true}, {0, 0, false}};
for (const auto &test : cases) {
query.fill(ready | 0x344u);
for (size_t db = 0; db < 16; db++) {
query[db * 2 + 1] += test.delta;
}
boolean = test.delta;
for (uint32_t wait : {0u, 1u}) {
check("ready Z-pass", 1, test.condition, wait, query.data(), test.skip);
check("bool preservation", 3, test.condition, wait, &boolean, test.skip);
}
}
query.fill(ready | 0x344u);
query[31]++;
check("only last DB visible", 1, 1, 0, query.data(), false);
check("inverse last DB visibility", 1, 0, 0, query.data(), true);
check("clear", 0, 0, 0, nullptr, false);
// Every begin AND end counter must be ready; no-wait must discard old skip state.
for (size_t missing = 0; missing < query.size(); missing++) {
query.fill(ready | 0x344u);
query[missing] &= ~ready;
for (uint32_t condition : {0u, 1u}) {
boolean = 0;
processor.SetPredication(1, 3, 0, &boolean, 0);
check("unavailable no-wait", 1, condition, 1, query.data(), false);
}
}
// Retry the same packet when its last DB's begin or end arrives later.
for (size_t missing : {30u, 31u}) {
query.fill(ready | 0x344u);
query[31]++;
query[missing] &= ~ready;
boolean = 0;
processor.SetPredication(1, 3, 0, &boolean, 0);
predicated = unconditional = 0;
const auto packet = commands(1, 1, 0, query.data());
Pm4Execution execution;
for (int retry = 0; retry < 2; retry++) {
Require(name, "pending wait suspends",
processor.Process(execution, packet) == Pm4ProcessResult::Blocked &&
!execution.MadeProgress() && predicated == 0 && unconditional == 0 &&
processor.ShouldSkipPredicatedPackets(),
"pending query advanced its packet or changed the prior predicate");
}
query[missing] |= ready;
Require(name, "ready wait resumes",
processor.Process(execution, packet) == Pm4ProcessResult::Complete &&
execution.MadeProgress() && predicated == 11 && unconditional == 22 &&
!processor.ShouldSkipPredicatedPackets(),
"ready query did not reevaluate and execute the suspended packet's suffix");
}
processor.BufferWait();
std::printf("[host] %-32s ok\n", name);
}
void CheckPm4StencilInfoValueLane(RenderContext &renderer) {
CommandProcessor processor(renderer, 0);
constexpr std::array<uint32_t, 2> payload{0x00100801u, 0x28000000u};
const auto consumed = HwCtxSetStencilInfo(
processor, 0xC0016900u, Pm4::DB_STENCIL_INFO, payload.data(), 1);
const auto &stencil = processor.GetCtx().GetDepthStencilInfo();
Require("Pm4StencilInfoValueLane", "standalone register value",
consumed == 1 && stencil.format == Prospero::StencilFormat::k8UInt &&
stencil.texture_compatibility ==
Prospero::TextureCompatibleStencil::kEnable &&
!stencil.expclear_enabled && !stencil.htile_stencil_disabled,
"standalone DB_STENCIL_INFO did not decode its sole payload value");
std::printf("[host] %-32s ok\n", "Pm4StencilInfoValueLane");
}
void CheckPm4NativeTargetGeometryRegisters(RenderContext &renderer) {
GraphicsInitJmpTables();
CommandProcessor packet(renderer, 0);
CommandProcessor indirect(renderer, 0);
const auto write_context = [](CommandProcessor &processor, uint32_t offset,
uint32_t value) {
std::array<uint32_t, 3> command{
KYTY_PM4(3, Pm4::IT_SET_CONTEXT_REG, Pm4::R_ZERO), offset, value};
Pm4Execution execution;
return processor.Process(execution, command) == Pm4ProcessResult::Complete;
};
constexpr uint32_t color_slot = 7;
constexpr uint32_t width_minus_one = 1919;
constexpr uint32_t height_minus_one = 1079;
constexpr uint32_t depth_minus_one = 5;
constexpr uint32_t max_mip = 3;
constexpr uint32_t dimension_3d = 2;
constexpr uint32_t attrib2_value =
height_minus_one | (width_minus_one << 14u) | (max_mip << 28u);
constexpr uint32_t attrib3_value =
depth_minus_one |
(static_cast<uint32_t>(Prospero::TileMode::kStandard64KB) << 14u) |
(dimension_3d << 24u) | (1u << 26u) | (1u << 30u);
const bool color_packets_complete =
write_context(packet, Pm4::CB_COLOR7_ATTRIB2, attrib2_value) &&
write_context(packet, Pm4::CB_COLOR7_ATTRIB3, attrib3_value);
g_hw_ctx_indirect_func[Pm4::CB_COLOR7_ATTRIB2](
indirect, Pm4::CB_COLOR7_ATTRIB2, attrib2_value);
g_hw_ctx_indirect_func[Pm4::CB_COLOR7_ATTRIB3](
indirect, Pm4::CB_COLOR7_ATTRIB3, attrib3_value);
const auto color_matches = [](const HW::RenderTarget &target) {
return target.attrib2.width == width_minus_one &&
target.attrib2.height == height_minus_one &&
target.attrib2.num_mip_levels == max_mip &&
target.attrib3.depth == depth_minus_one &&
target.attrib3.tile_mode == Prospero::TileMode::kStandard64KB &&
target.attrib3.dimension == dimension_3d &&
target.attrib3.metadata_pipe_aligned;
};
Require("Pm4NativeTargetGeometry", "sparse color geometry",
color_packets_complete &&
color_matches(packet.GetCtx().GetRenderTarget(color_slot)) &&
color_matches(indirect.GetCtx().GetRenderTarget(color_slot)) &&
!color_matches(packet.GetCtx().GetRenderTarget(0)) &&
!color_matches(indirect.GetCtx().GetRenderTarget(0)),
"ATTRIB2/ATTRIB3 did not provide native color extent and layout");
constexpr uint32_t depth_size_value =
width_minus_one | (height_minus_one << 16u);
const bool depth_packet_complete =
write_context(packet, Pm4::DB_DEPTH_SIZE_XY, depth_size_value);
g_hw_ctx_indirect_func[Pm4::DB_DEPTH_SIZE_XY](indirect, Pm4::DB_DEPTH_SIZE_XY,
depth_size_value);
const auto depth_matches = [](const HW::DepthRenderTarget &target) {
return target.size.valid && target.size.x_max == width_minus_one &&
target.size.y_max == height_minus_one;
};
Require("Pm4NativeTargetGeometry", "native depth geometry",
depth_packet_complete &&
depth_matches(packet.GetCtx().GetDepthRenderTarget()) &&
depth_matches(indirect.GetCtx().GetDepthRenderTarget()),
"DB_DEPTH_SIZE_XY did not provide the native depth extent");
bool legacy_slots_are_unhandled = true;
for (uint32_t slot = 0; slot < 8; slot++) {
const auto stride = slot * 15u;
const std::array legacy_color_slots{
Pm4::CB_COLOR0_BASE + stride + 1u,
Pm4::CB_COLOR0_BASE + stride + 2u,
Pm4::CB_COLOR0_CMASK + stride + 1u,
Pm4::CB_COLOR0_FMASK + stride + 1u,
};
for (const auto offset : legacy_color_slots) {
legacy_slots_are_unhandled &= g_hw_ctx_func[offset] == nullptr &&
g_hw_ctx_indirect_func[offset] == nullptr;
}
}
// These offsets are the removed GCN DB_DEPTH_INFO, DB_DEPTH_SIZE, and
// DB_DEPTH_SLICE holes. Keep them numeric so deleting their legacy PM4 names
// does not weaken this dispatch-table regression check.
constexpr std::array<uint32_t, 3> legacy_depth_slots{0x000fu, 0x0016u,
0x0017u};
for (const auto offset : legacy_depth_slots) {
legacy_slots_are_unhandled &= g_hw_ctx_func[offset] == nullptr &&
g_hw_ctx_indirect_func[offset] == nullptr;
}
// Removed GCN PA_SC_NGG_MODE_CNTL.
legacy_slots_are_unhandled &= g_hw_ctx_func[0x0314u] == nullptr &&
g_hw_ctx_indirect_func[0x0314u] == nullptr;
// Removed GCN shader resource/checksum/queue registers. Numeric offsets keep
// this check independent of the deleted legacy names.
constexpr std::array<uint32_t, 17> legacy_shader_slots{
0x000u, 0x001u, 0x030u, 0x0b0u, 0x0bcu, 0x130u,
0x14au, 0x14bu, 0x20eu, 0x20fu, 0x210u, 0x211u, 0x216u, 0x217u,
0x219u, 0x21au, 0x27du,
};
for (const auto offset : legacy_shader_slots) {
legacy_slots_are_unhandled &= g_hw_sh_func[offset] == nullptr &&
g_hw_sh_indirect_func[offset] == nullptr;
}
// Restoring shader state after Toolkit rendering still emits the compiler's
// GS-front allocation metadata at 0xca.
for (uint32_t offset = 0x0cbu; offset <= 0x0ebu; offset++) {
legacy_slots_are_unhandled &= g_hw_sh_func[offset] == nullptr &&
g_hw_sh_indirect_func[offset] == nullptr;
}
std::array<uint32_t, 3> index_size_packet{
KYTY_PM4(3, Pm4::IT_SET_UCONFIG_REG_INDEX, Pm4::R_ZERO),
0x20000000u | Pm4::VGT_INDEX_TYPE,
0x441u,
};
Pm4Execution index_size_execution;
const bool native_index_size_packet_complete =
packet.Process(index_size_execution, index_size_packet) ==
Pm4ProcessResult::Complete &&
g_hw_uc_func[Pm4::VGT_INDEX_TYPE] != nullptr &&
g_hw_uc_indirect_func[Pm4::VGT_INDEX_TYPE] != nullptr &&
g_hw_uc_func[Pm4::IA_MULTI_VGT_PARAM] != nullptr &&
g_hw_uc_indirect_func[Pm4::IA_MULTI_VGT_PARAM] != nullptr;
const bool native_ps_address_slots_are_handled =
g_hw_sh_func[Pm4::SPI_SHADER_USER_DATA_ADDR_LO_PS] != nullptr &&
g_hw_sh_func[Pm4::SPI_SHADER_USER_DATA_ADDR_HI_PS] != nullptr &&
g_hw_sh_indirect_func[Pm4::SPI_SHADER_USER_DATA_ADDR_LO_PS] != nullptr &&
g_hw_sh_indirect_func[Pm4::SPI_SHADER_USER_DATA_ADDR_HI_PS] != nullptr;
const bool native_pace_slots_are_handled =
g_hw_sh_indirect_func[Pm4::SPI_SHADER_PACE_ID_PS] != nullptr &&
g_hw_sh_indirect_func[Pm4::SPI_SHADER_PACE_ID_GS] != nullptr &&
g_hw_sh_indirect_func[Pm4::COMPUTE_PACE_ID] != nullptr;
Require("Pm4NativeTargetGeometry", "legacy register holes",
legacy_slots_are_unhandled && native_pace_slots_are_handled &&
native_ps_address_slots_are_handled && native_index_size_packet_complete,
"a removed GCN register has a handler or a native packet is not handled");
std::printf("[host] %-32s ok\n", "Pm4NativeTargetGeometry");
}
void CheckPm4IndirectShaderRegisters(RenderContext &renderer) {
GraphicsInitJmpTables();
CommandProcessor processor(renderer, 0);
std::array<uint32_t, 24> registers{
Pm4::SPI_SHADER_PGM_RSRC4_GS, 0x0badc0deu,
Pm4::SPI_SHADER_PGM_CHKSUM_HS, 0x12345678u,
Pm4::SPI_SHADER_PGM_RSRC4_HS, 0x87654321u,
Pm4::SPI_SHADER_PGM_LO_ES, 0x050025e5u,
Pm4::SPI_SHADER_PGM_HI_ES, 0u,
Pm4::SPI_SHADER_PGM_RSRC1_GS, 0x60000002u,
Pm4::SPI_SHADER_PGM_RSRC2_GS, 0x00030008u,
0x000000cau, 0x03000002u,
Pm4::SPI_SHADER_PGM_LO_PS, 0x05000104u,
Pm4::SPI_SHADER_PGM_HI_PS, 0u,
Pm4::SPI_SHADER_PGM_RSRC1_PS, 0x022c018du,
Pm4::SPI_SHADER_PGM_RSRC2_PS, 0x0000003cu,
};
const auto address = reinterpret_cast<uint64_t>(registers.data());
std::array<uint32_t, 5> command{
KYTY_PM4(5, Pm4::IT_SET_SH_REG_INDIRECT, Pm4::R_ZERO),
static_cast<uint32_t>(address), static_cast<uint32_t>(address >> 32u),
0x80000000u, static_cast<uint32_t>(registers.size() / 2u),
};
Pm4Execution execution;
const bool handlers_present =
g_hw_sh_indirect_func[Pm4::SPI_SHADER_PGM_RSRC4_GS] != nullptr &&
g_hw_sh_indirect_func[Pm4::SPI_SHADER_PGM_CHKSUM_HS] != nullptr &&
g_hw_sh_indirect_func[Pm4::SPI_SHADER_PGM_RSRC4_HS] != nullptr;
Require("Pm4IndirectShaderRegisters", "indirect shader register stream",
handlers_present &&
processor.Process(execution, command) == Pm4ProcessResult::Complete,
"shader registers were rejected by SET_SH_REG_INDIRECT");
const auto &vs = processor.GetShCtx().GetVs();
const auto &ps = processor.GetShCtx().GetPs().ps_regs;
Require("Pm4IndirectShaderRegisters", "native state around front metadata",
vs.es_regs.data_addr == 0x50025e500ull &&
vs.gs_regs.rsrc1.vgprs == 2u &&
vs.gs_regs.rsrc1.gs_vgpr_component_count == 3u &&
vs.gs_regs.rsrc2.es_vgpr_component_count == 3u &&
vs.gs_regs.rsrc2.user_sgpr == 4u &&
ps.data_addr == 0x500010400ull && ps.rsrc1.vgprs == 13u &&
ps.rsrc2.user_sgpr == 30u,
"GS-front metadata changed native GS state or prevented later PS writes");
std::printf("[host] %-32s ok\n", "Pm4IndirectShaderRegisters");
}
void CheckPm4IndirectUconfigRegisters(RenderContext &renderer) {
GraphicsInitJmpTables();
CommandProcessor processor(renderer, 0);
std::array<uint32_t, 2> registers{
Pm4::UC_PARAMETER_OVERSUBSCRIPTION, 0x12345678u,
};
const auto address = reinterpret_cast<uint64_t>(registers.data());
std::array<uint32_t, 5> command{
KYTY_PM4(5, Pm4::IT_SET_UCONFIG_REG_INDIRECT, Pm4::R_ZERO),
static_cast<uint32_t>(address), static_cast<uint32_t>(address >> 32u),
0x80000000u, 1u,
};
Pm4Execution execution;
const bool handler_present =
g_hw_uc_indirect_func[Pm4::UC_PARAMETER_OVERSUBSCRIPTION] != nullptr;
Require("Pm4IndirectUconfigRegisters", "indirect UCONFIG register stream",
handler_present &&
processor.Process(execution, command) == Pm4ProcessResult::Complete,
"UCONFIG register was rejected by SET_UCONFIG_REG_INDIRECT");
std::printf("[host] %-32s ok\n", "Pm4IndirectUconfigRegisters");
}
void CheckPm4DirectShaderRegisterFallback(RenderContext &renderer) {
GraphicsInitJmpTables();
CommandProcessor processor(renderer, 0);
constexpr uint64_t shader_address = 0x0000123456789a00ull;
std::array<uint32_t, 4> packet{
KYTY_PM4(4, Pm4::IT_SET_SH_REG, Pm4::R_ZERO),
Pm4::SPI_SHADER_PGM_LO_ES,
static_cast<uint32_t>(shader_address >> 8u),
static_cast<uint32_t>(shader_address >> 40u),
};
Pm4Execution execution;
Require("Pm4DirectShaderRegisterFallback", "ES program base",
processor.Process(execution, packet) == Pm4ProcessResult::Complete &&
processor.GetShCtx().GetVs().es_regs.data_addr == shader_address,
"direct SET_SH_REG did not route its ES base pair through the "
"register handlers");
std::printf("[host] %-32s ok\n", "Pm4DirectShaderRegisterFallback");
}
void CheckPm4GuardBandRegisterRanges(RenderContext &renderer) {
GraphicsInitJmpTables();
CommandProcessor processor(renderer, 0);
processor.GetCtx().SetGuardBands(2.0f, 3.0f, 4.0f, 5.0f);
std::array<uint32_t, 3> single{
KYTY_PM4(3, Pm4::IT_SET_CONTEXT_REG, Pm4::R_ZERO),
Pm4::PA_CL_GB_VERT_CLIP_ADJ,
std::bit_cast<uint32_t>(6.0f),
};
Pm4Execution single_execution;
const auto single_result = processor.Process(single_execution, single);
const auto &single_viewport = processor.GetCtx().GetScreenViewport();
Require("Pm4GuardBandRegisterRanges", "single register",
single_result == Pm4ProcessResult::Complete &&
single_viewport.guard_band_horz_clip == 2.0f &&
single_viewport.guard_band_vert_clip == 6.0f &&
single_viewport.guard_band_horz_discard == 4.0f &&
single_viewport.guard_band_vert_discard == 5.0f,
"single guard-band write changed the wrong context state");
std::array<uint32_t, 6> range{
KYTY_PM4(6, Pm4::IT_SET_CONTEXT_REG, Pm4::R_ZERO),
Pm4::PA_CL_GB_VERT_CLIP_ADJ,
std::bit_cast<uint32_t>(7.0f),
std::bit_cast<uint32_t>(8.0f),
std::bit_cast<uint32_t>(9.0f),
std::bit_cast<uint32_t>(10.0f),
};
Pm4Execution range_execution;
const auto range_result = processor.Process(range_execution, range);
const auto &range_viewport = processor.GetCtx().GetScreenViewport();
Require("Pm4GuardBandRegisterRanges", "full range",
range_result == Pm4ProcessResult::Complete &&
range_viewport.guard_band_horz_clip == 9.0f &&
range_viewport.guard_band_vert_clip == 7.0f &&
range_viewport.guard_band_horz_discard == 10.0f &&
range_viewport.guard_band_vert_discard == 8.0f,
"full guard-band range did not preserve hardware register order");
std::printf("[host] %-32s ok\n", "Pm4GuardBandRegisterRanges");
}
void CheckPm4BlendColorRegisterRanges(RenderContext &renderer) {
GraphicsInitJmpTables();
CommandProcessor processor(renderer, 0);
processor.GetCtx().SetBlendColor({1.0f, 2.0f, 3.0f, 4.0f});
constexpr std::array<std::pair<uint32_t, float>, 4> singleton_values{{
{Pm4::CB_BLEND_RED, 5.0f},
{Pm4::CB_BLEND_BLUE, 7.0f},
{Pm4::CB_BLEND_GREEN, 6.0f},
{Pm4::CB_BLEND_ALPHA, 8.0f},
}};
bool singletons_complete = true;
for (const auto &[offset, value] : singleton_values) {
std::array<uint32_t, 3> packet{
KYTY_PM4(3, Pm4::IT_SET_CONTEXT_REG, Pm4::R_ZERO),
offset,
std::bit_cast<uint32_t>(value),
};
Pm4Execution execution;
singletons_complete &=
processor.Process(execution, packet) == Pm4ProcessResult::Complete;
}
const auto &singletons = processor.GetCtx().GetBlendColor();
Require("Pm4BlendColorRegisterRanges", "single registers",
singletons_complete && singletons.red == 5.0f &&
singletons.green == 6.0f && singletons.blue == 7.0f &&
singletons.alpha == 8.0f,
"single blend-color writes did not follow native PS5 register order");
std::array<uint32_t, 4> partial{
KYTY_PM4(4, Pm4::IT_SET_CONTEXT_REG, Pm4::R_ZERO),
Pm4::CB_BLEND_BLUE,
std::bit_cast<uint32_t>(9.0f),
std::bit_cast<uint32_t>(10.0f),
};
Pm4Execution partial_execution;
const auto partial_result = processor.Process(partial_execution, partial);
const auto &partial_color = processor.GetCtx().GetBlendColor();
Require("Pm4BlendColorRegisterRanges", "partial range",
partial_result == Pm4ProcessResult::Complete &&
partial_color.red == 5.0f && partial_color.green == 10.0f &&
partial_color.blue == 9.0f && partial_color.alpha == 8.0f,
"partial blend-color range changed the wrong channels");
std::array<uint32_t, 6> full{
KYTY_PM4(6, Pm4::IT_SET_CONTEXT_REG, Pm4::R_ZERO),
Pm4::CB_BLEND_RED,
std::bit_cast<uint32_t>(11.0f),
std::bit_cast<uint32_t>(13.0f),
std::bit_cast<uint32_t>(12.0f),
std::bit_cast<uint32_t>(14.0f),
};
Pm4Execution full_execution;
const auto full_result = processor.Process(full_execution, full);
const auto &full_color = processor.GetCtx().GetBlendColor();
Require("Pm4BlendColorRegisterRanges", "full range",
full_result == Pm4ProcessResult::Complete &&
full_color.red == 11.0f && full_color.green == 12.0f &&
full_color.blue == 13.0f && full_color.alpha == 14.0f,
"full blend-color range did not preserve native PS5 register order");
std::printf("[host] %-32s ok\n", "Pm4BlendColorRegisterRanges");
}
void CheckPm4PolygonOffsetRegisters(RenderContext &renderer) {
GraphicsInitJmpTables();
CommandProcessor processor(renderer, 0);
const auto &defaults = processor.GetCtx().GetPolyOffset();
Require("Pm4PolygonOffset", "format defaults",
defaults.neg_num_db_bits == -23 && defaults.db_is_float_fmt,
"polygon offset Z format did not default to D32F");
constexpr std::array<uint32_t, 6> payload{0x000000f0u,
std::bit_cast<uint32_t>(2.5f),
std::bit_cast<uint32_t>(16.0f),
std::bit_cast<uint32_t>(-3.25f),
std::bit_cast<uint32_t>(32.0f),
std::bit_cast<uint32_t>(4.5f)};
const auto consumed = HwCtxSetPolyOffsetRegisters(
processor, KYTY_PM4(8, Pm4::IT_SET_CONTEXT_REG, Pm4::R_ZERO),
Pm4::PA_SU_POLY_OFFSET_DB_FMT_CNTL, payload.data(), 0);
const auto &direct = processor.GetCtx().GetPolyOffset();
Require("Pm4PolygonOffset", "direct registers",
consumed == payload.size() && direct.neg_num_db_bits == -16 &&
!direct.db_is_float_fmt && direct.clamp == 2.5f &&
direct.front_scale == 16.0f && direct.front_offset == -3.25f &&
direct.back_scale == 32.0f && direct.back_offset == 4.5f,
"direct polygon offset register packet was decoded incorrectly");
g_hw_ctx_indirect_func[Pm4::PA_SU_POLY_OFFSET_DB_FMT_CNTL](
processor, Pm4::PA_SU_POLY_OFFSET_DB_FMT_CNTL, 0x000001e9u);
g_hw_ctx_indirect_func[Pm4::PA_SU_POLY_OFFSET_FRONT_SCALE](
processor, Pm4::PA_SU_POLY_OFFSET_FRONT_SCALE,
std::bit_cast<uint32_t>(24.0f));
const auto &indirect = processor.GetCtx().GetPolyOffset();
Require("Pm4PolygonOffset", "indirect registers",
indirect.neg_num_db_bits == -23 && indirect.db_is_float_fmt &&
indirect.front_scale == 24.0f && indirect.front_offset == -3.25f,
"indirect polygon offset register write was decoded incorrectly");
std::printf("[host] %-32s ok\n", "Pm4PolygonOffset");
}
void CheckPm4DepthControlHighBits(RenderContext &renderer) {
GraphicsInitJmpTables();
CommandProcessor processor(renderer, 0);
std::array<uint32_t, 3> direct{
KYTY_PM4(3, Pm4::IT_SET_CONTEXT_REG, Pm4::R_ZERO),
Pm4::DB_DEPTH_CONTROL,
0x80000072u,
};
Pm4Execution direct_execution;
const bool direct_complete =
processor.Process(direct_execution, direct) == Pm4ProcessResult::Complete;
const auto &direct_control = processor.GetCtx().GetDepthControl();
const bool direct_decoded =
!direct_control.stencil_enable && direct_control.z_enable &&
!direct_control.z_write_enable && !direct_control.depth_bounds_enable &&
direct_control.zfunc == 7u && !direct_control.backface_enable &&
direct_control.stencilfunc == 0u && direct_control.stencilfunc_bf == 0u;
std::array<uint32_t, 2> registers{
Pm4::DB_DEPTH_CONTROL,
0x4070078du,
};
const auto address = reinterpret_cast<uint64_t>(registers.data());
std::array<uint32_t, 5> indirect{
KYTY_PM4(5, Pm4::IT_SET_CONTEXT_REG_INDIRECT, Pm4::R_ZERO),
static_cast<uint32_t>(address), static_cast<uint32_t>(address >> 32u),
0x80000000u, 1u,
};
Pm4Execution indirect_execution;
const bool indirect_complete =
processor.Process(indirect_execution, indirect) ==
Pm4ProcessResult::Complete;
const auto &indirect_control = processor.GetCtx().GetDepthControl();
const bool indirect_decoded =
indirect_control.stencil_enable && !indirect_control.z_enable &&
indirect_control.z_write_enable && indirect_control.depth_bounds_enable &&
indirect_control.zfunc == 0u && indirect_control.backface_enable &&
indirect_control.stencilfunc == 7u &&
indirect_control.stencilfunc_bf == 7u;
Require("Pm4DepthControlHighBits", "preserved high bits",
direct_complete && direct_decoded && indirect_complete &&
indirect_decoded,
"DB_DEPTH_CONTROL high bits rejected or changed decoded PS5 state");
std::printf("[host] %-32s ok\n", "Pm4DepthControlHighBits");
}
void CheckPm4DepthRenderOverride(RenderContext &renderer) {
constexpr const char *name = "Pm4DepthRenderOverride";
GraphicsInitJmpTables();
CommandProcessor processor(renderer, 0);
struct Case {
uint32_t value;
HW::DepthRenderOverride expected;
};
constexpr std::array cases{Case{0x28000000u, {true, true, false, false}},
Case{0x50000000u, {false, false, true, true}},
Case{0x18000000u, {false, true, false, true}},
Case{0x60000000u, {true, false, true, false}},
Case{0xf8000000u, {true, true, true, true}},
Case{0x82001a0fu, {false, false, false, false}}};
for (const bool indirect : {false, true}) {
for (const auto &test : cases) {
std::array<uint32_t, 2> registers{Pm4::DB_RENDER_OVERRIDE, test.value};
const auto address = reinterpret_cast<uint64_t>(registers.data());
const std::array<uint32_t, 3> direct_packet{
KYTY_PM4(3, Pm4::IT_SET_CONTEXT_REG, Pm4::R_ZERO),
Pm4::DB_RENDER_OVERRIDE, test.value};
const std::array<uint32_t, 5> indirect_packet{
KYTY_PM4(5, Pm4::IT_SET_CONTEXT_REG_INDIRECT, Pm4::R_ZERO),
static_cast<uint32_t>(address), static_cast<uint32_t>(address >> 32u),
0x80000000u, 1u};
Pm4Execution execution;
const auto result = processor.Process(
execution, indirect ? std::span<const uint32_t>{indirect_packet}
: std::span<const uint32_t>{direct_packet});
const auto &actual = processor.GetCtx().GetDepthRenderOverride();
Require(
name, indirect ? "indirect register" : "direct register",
result == Pm4ProcessResult::Complete &&
actual.force_z_valid == test.expected.force_z_valid &&
actual.force_z_dirty == test.expected.force_z_dirty &&
actual.force_stencil_valid == test.expected.force_stencil_valid &&
actual.force_stencil_dirty == test.expected.force_stencil_dirty,
"DB_RENDER_OVERRIDE lost, exchanged, or retained force flags");
}
}
std::printf("[host] %-32s ok\n", name);
}
void CheckShaderFusion() {
struct Case {
bool gs;
uint32_t front1, front2, back1, back2, fused1, fused2, older2;
};
// Expected register results for both shader-fusion variants.
constexpr Case cases[]{
{true, 0x600c0007, 0x00030010, 0x600c0007, 0x008b0000,
0x600c0007, 0x008b0010, 0x008b0010}, // Captured volume ES + GS.
{true, 0x20ac0003, 0xf805004a, 0x40ec0007, 0x028b003e,
0x40ec0007, 0x1a8f000a, 0xfa8f000a},
{false, 0x20ac0003, 0xf805004a, 0x10ec0007, 0x028b003e,
0x20ec0007, 0x1a8b000a, 0xfa8b000a},
};
ShaderUserData user_data{};
ShaderSpecialRegs specials{};
for (const auto &c : cases) {
const auto checksum = c.gs ? Pm4::SPI_SHADER_PGM_CHKSUM_GS : Pm4::SPI_SHADER_PGM_CHKSUM_HS;
const auto rsrc1 = c.gs ? Pm4::SPI_SHADER_PGM_RSRC1_GS : Pm4::SPI_SHADER_PGM_RSRC1_HS;
const auto rsrc2 = c.gs ? Pm4::SPI_SHADER_PGM_RSRC2_GS : Pm4::SPI_SHADER_PGM_RSRC2_HS;
const auto lo = c.gs ? Pm4::SPI_SHADER_PGM_LO_ES : Pm4::SPI_SHADER_PGM_LO_LS;
std::array<ShaderRegister, 6> front_regs{{
{checksum, 0x1234}, {checksum, 0x5678}, {rsrc1, c.front1},
{rsrc2, c.front2}, {lo, 0}, {lo + 1u, 0}}};
const std::array<ShaderRegister, 6> back_regs{{
{checksum, 0}, {checksum, 0}, {rsrc1, c.back1},
{rsrc2, c.back2}, {lo, 0}, {lo + 1u, 0xabcdef00}}};
Shader front{};
front.user_data = &user_data;
front.code = reinterpret_cast<const void *>(0x12580e91c00ull);
front.sh_registers = front_regs.data();
front.specials = &specials;
front.type = static_cast<uint8_t>(c.gs ? Prospero::ShaderBinaryType::kGsFront
: Prospero::ShaderBinaryType::kHsFront);
front.num_sh_registers = front_regs.size();
for (const bool older : {false, true}) {
for (const bool in_place : {false, true}) {
auto registers = back_regs;
std::array<ShaderRegister, 6> scratch{};
Shader back = front;
back.sh_registers = registers.data();
back.type = static_cast<uint8_t>(c.gs ? Prospero::ShaderBinaryType::kGsBack
: Prospero::ShaderBinaryType::kHsBack);
Shader fused{};
const auto fuse = older ? Gen5::AgcFuseShaderHalves : Gen5::AgcFuseShaderHalves_0200;
const auto result = fuse(&fused, &front, &back, in_place ? nullptr : scratch.data());
const auto *regs = fused.sh_registers;
Require("ShaderFusion", "native fused registers",
result == 0 && regs == (in_place ? registers.data() : scratch.data()) &&
regs[0].value == 0x1234 && regs[1].value == 0x5678 &&
regs[2].value == c.fused1 && regs[3].value == (older ? c.older2 : c.fused2) &&
regs[4].value == 0x2580e91c && regs[5].value == 0xabcdef01 &&
fused.type == static_cast<uint8_t>(c.gs ? Prospero::ShaderBinaryType::kGs
: Prospero::ShaderBinaryType::kHs) &&
fused.user_data == (older ? &user_data : nullptr) &&
(in_place || std::memcmp(registers.data(), back_regs.data(), sizeof(registers)) == 0),
"shader fusion lost front resources or changed the export's allocation/metadata behavior");
}
}
}
std::printf("[host] %-32s ok\n", "ShaderFusion");
}
void CheckAgcSystemTable(RenderContext &renderer) {
constexpr const char *name = "AgcSystemTable";
constexpr uint64_t driver_base = 0x0fe0000000ull;
constexpr uint64_t table_address = driver_base + 0x40000;
constexpr std::array<uint32_t, 16> native_validator{
0xbeeb03ff, 0x00000021, 0xb96a1818, 0xbf06106a,
0xbf800000, 0x8590807e, 0xb96a0a18, 0xbf066a80,
0xbf800000, 0x85ea807e, 0x88ea106a, 0xbf870003,
0xbefc03ff, 0x8a6ca000, 0xbf920009, 0xbefd210e};
LibKernel::Memory::InstallGpuResources(&renderer);
Require(name, "driver module startup", Gen5Driver::Initialize() == 0,
"driver module startup failed before guest initialization");
const auto direct_memory_size = LibKernel::Memory::KernelGetDirectMemorySize();
int64_t available_offset = -1;
size_t available_size = 0;
Require(name, "remaining direct pool",
LibKernel::Memory::KernelAvailableDirectMemorySize(
0, direct_memory_size, 0, &available_offset, &available_size) == 0 &&
available_size == direct_memory_size - 0x200000,
"driver startup did not reserve its backing in the reported direct pool");
int64_t guest_allocation = -1;
Require(name, "guest consumes remaining direct pool",
LibKernel::Memory::KernelAllocateDirectMemory(
0, direct_memory_size, available_size, 0x4000, 0, &guest_allocation) == 0,
"guest could not allocate all remaining direct memory");
uint32_t state = 0xabcdef01u;
Require(name, "driver startup", Gen5::AgcInit(&state, 13) == 0 && state == 0xabcdef01u,
"Agc startup required free guest memory or changed its ignored public state pointer");
LibKernel::Memory::VirtualQueryInfo mapping{};
Require(name, "shared driver backing",
LibKernel::Memory::KernelVirtualQuery(reinterpret_cast<void *>(driver_base), 0,
&mapping, sizeof(mapping)) == 0 &&
mapping.start == driver_base && mapping.end == driver_base + 0x200000 &&
mapping.memory_type == 12 && mapping.protection == 0x33 && mapping.is_direct &&
std::strcmp(mapping.name, "SceAgcDriver") == 0 &&
renderer.IsMapped(driver_base, 0x200000),
"driver startup bypassed shared DMEM metadata or GPU range registration");
ShaderBufferResource descriptor{};
std::array<uint64_t, 16> functions{};
std::array<uint32_t, 16> code{};
Require(name, "ordered-count function backing",
LibKernel::Memory::TryReadBacking(table_address, &descriptor, sizeof(descriptor)) &&
descriptor.Base48() == table_address + 0x60 && descriptor.Stride() == 16 &&
descriptor.NumRecords() == 8 && descriptor.fields[3] == 0x5204 &&
LibKernel::Memory::TryReadBacking(descriptor.Base48(), functions.data(),
sizeof(functions)) &&
functions[4] != 0 && (functions[4] & 0xffu) == 0 && functions[5] == 0 &&
LibKernel::Memory::TryReadBacking(functions[4], code.data(), sizeof(code)) &&
code == native_validator,
"system descriptor did not reach the queue validator and its user data");
auto *instrumentation = reinterpret_cast<uint64_t *>(table_address + 0x18);
Require(name, "default instrumentation", *instrumentation == 0,
"unused vertex validation was enabled at startup");
*instrumentation = 0x12340000u;
Require(name, "repeat startup", Gen5Driver::Initialize() == 0 &&
Gen5::AgcInit(nullptr, 13) == 0 &&
*instrumentation == 0x12340000u,
"repeat Agc initialization reset existing system state");
*instrumentation = 0;
Require(name, "release guest allocation",
LibKernel::Memory::KernelCheckedReleaseDirectMemory(guest_allocation, available_size) == 0,
"guest allocation release failed");
LibKernel::Memory::InstallGpuResources(nullptr);
std::printf("[host] %-32s ok\n", name);
}
struct CommandBufferLayout {
using Callback = KYTY_SYSV_ABI bool (*)(Gen5::CommandBuffer *, uint32_t,
void *);
uint32_t *bottom;
uint32_t *top;
uint32_t *cursor_up;
uint32_t *cursor_down;
Callback callback;
void *user_data;
uint32_t reserved_dw;
};
static_assert(offsetof(CommandBufferLayout, cursor_up) == 0x10);
static_assert(offsetof(CommandBufferLayout, reserved_dw) == 0x30);
void CheckPm4AcquireMemPackets(RenderContext &renderer) {
constexpr const char *name = "Pm4AcquireMemPackets";
GraphicsInitJmpTables();
CommandProcessor processor(renderer, 0);
// Byte ranges cover every touched 256-byte block, including partial first and last blocks.
struct Case {
const char *label;
bool async;
uint8_t engine;
uint32_t cb_db, gcr;
uint64_t base, size;
uint32_t poll;
std::array<uint32_t, 8> expected;
};
const Case cases[]{
{"captured byte range", false, 1, 0, 0x9000, 0x2036000308ull, 0x20, 400,
{0xc0065800, 0x80000000, 1, 0, 0x20360003, 0, 25, 0x9000}},
{"crossing high address", false, 0, 0x7fc0, 0x20, 0xab12345678f0ull, 0x40, 400,
{0xc0065800, 0x7fc0, 2, 0, 0x12345678, 0xab, 25, 0x20}},
{"full range", false, 0, 0x7fc0, 0x20, 0, UINT64_MAX, 400,
{0xc0065800, 0x7fc0, UINT32_MAX, 0xff, 0, 0, 25, 0x20}},
{"wide range and saturated poll", false, 1, 0, 0x380, 0, 0x10000000000ull, UINT32_MAX,
{0xc0065800, 0x80000000, 0, 1, 0, 0, 0xffff, 0x380}},
{"async byte range", true, 0, 0, 0x9000, 0x2036000308ull, 0x20, 400,
{0xc0065800, 0, 1, 0, 0x20360003, 0, 25, 0x9000}},
};
Require(name, "packet sizes", Gen5::AgcDcbAcquireMemGetSize() == 32 &&
Gen5::AgcAcbAcquireMemGetSize() == 32,
"native single ACQUIRE_MEM packet size changed");
for (const auto &test : cases) {
std::array<uint32_t, 8> packet{};
CommandBufferLayout layout{packet.data(), packet.data() + packet.size(),
packet.data(), packet.data() + packet.size(),
nullptr, nullptr, 0};
auto *buffer = reinterpret_cast<Gen5::CommandBuffer *>(&layout);
const auto *base = reinterpret_cast<const volatile void *>(test.base);
auto *emitted = test.async
? Gen5::AgcAcbAcquireMem(buffer, test.gcr, base, test.size, test.poll)
: Gen5::AgcDcbAcquireMem(buffer, test.engine, test.cb_db, test.gcr,
base, test.size, test.poll);
Require(name, test.label, emitted == packet.data() &&
layout.cursor_up == packet.data() + packet.size() && packet == test.expected,
"ACQUIRE_MEM differs from the pinned native packet");
const auto tick = renderer.GetCommandScheduler().CurrentTick();
Pm4Execution execution;
Require(name, test.label,
processor.Process(execution, packet) == Pm4ProcessResult::Complete &&
packet == test.expected && renderer.GetCommandScheduler().CurrentTick() == tick,
"native ACQUIRE_MEM was not consumed without mutation or queue submission");
}
const std::array<uint32_t, 7> ps4_packet{0xc0055800, 0x00400000, 1, 0, 0, 0, 10};
Pm4Execution execution;
Require(name, "native PS4 packet",
processor.Process(execution, ps4_packet) == Pm4ProcessResult::Complete,
"seven-DWORD ACQUIRE_MEM was not consumed");
std::printf("[host] %-32s ok\n", name);
}
void CheckPm4WaitPackets(RenderContext &renderer) {
GraphicsInitJmpTables();
CommandProcessor processor(renderer, 0);
constexpr auto packet_mismatch = static_cast<int>(0x8a6c000cu);
const auto execute = [&](uint32_t *packet, uint32_t size_dw) {
Pm4Execution execution;
return processor.Process(execution, {packet, size_dw}) ==
Pm4ProcessResult::Complete;
};
alignas(8) uint32_t label32 = 0x11223344u;
std::array<uint32_t, 16> packet32{};
CommandBufferLayout dcb32{packet32.data(),
packet32.data() + packet32.size(),
packet32.data(),
packet32.data() + packet32.size(),
nullptr,
nullptr,
0};
const auto address32 = reinterpret_cast<uint64_t>(&label32);
auto *emitted32 =
Gen5::AgcDcbWaitRegMem(reinterpret_cast<Gen5::CommandBuffer *>(&dcb32), 0,
3, 0, 0, &label32, label32, UINT32_MAX, 400);
Require("Pm4WaitPackets", "native 32-bit packet",
Gen5::AgcDcbWaitOnAddressGetSize(0) == 56 &&
emitted32 == packet32.data() &&
dcb32.cursor_up == packet32.data() + 14 &&
packet32[0] == 0xc0027904u && packet32[1] == 0x342u &&
packet32[2] ==
(0xc8010000u |
(static_cast<uint32_t>(address32 >> 32u) & 0xffffu)) &&
packet32[3] == static_cast<uint32_t>(address32) &&
packet32[4] == 0xc0053c00u && packet32[5] == 0x13u &&
packet32[6] == (static_cast<uint32_t>(address32) & ~0x3u) &&
packet32[7] ==
(static_cast<uint32_t>(address32 >> 32u) & 0x3ffffu) &&
packet32[8] == label32 && packet32[9] == UINT32_MAX &&
packet32[10] == 25u && packet32[11] == 0xc0017904u &&
packet32[12] == 0x342u && packet32[13] == 0xc8000000u &&
execute(packet32.data(), 14),
"32-bit wait packet differs from the expected stream");
alignas(8) uint32_t patched_label32 = 0xaabbccddu;
const auto patched_address32 = reinterpret_cast<uint64_t>(&patched_label32);
packet32[6] |= 0x2u;
packet32[7] |= 0xa5a40000u;
Require(
"Pm4WaitPackets", "native 32-bit patch",
Gen5::AgcWaitRegMemPatchAddress(packet32.data(), &patched_label32) == 0 &&
Gen5::AgcWaitRegMemPatchReference(packet32.data(), patched_label32) ==
0 &&
packet32[2] ==
(0xc8010000u |
(static_cast<uint32_t>(patched_address32 >> 32u) & 0xffffu)) &&
packet32[3] == static_cast<uint32_t>(patched_address32) &&
packet32[6] ==
((static_cast<uint32_t>(patched_address32) & ~0x3u) | 0x2u) &&
packet32[7] ==
((static_cast<uint32_t>(patched_address32 >> 32u) & 0x3ffffu) |
0xa5a40000u) &&
packet32[8] == patched_label32 && execute(packet32.data(), 14),
"32-bit native wait patch did not update metadata and hardware packet");
alignas(8) uint64_t label64 = 0x1122334455667788ull;
std::array<uint32_t, 16> packet64{};
CommandBufferLayout dcb64{packet64.data(),
packet64.data() + packet64.size(),
packet64.data(),
packet64.data() + packet64.size(),
nullptr,
nullptr,
0};
const auto address64 = reinterpret_cast<uint64_t>(&label64);
auto *emitted64 =
Gen5::AgcDcbWaitRegMem(reinterpret_cast<Gen5::CommandBuffer *>(&dcb64), 1,
3, 0, 0, &label64, label64, UINT64_MAX, 400);
Require("Pm4WaitPackets", "native 64-bit packet",
Gen5::AgcDcbWaitOnAddressGetSize(1) == 64 &&
Gen5::AgcDcbWaitOnAddressGetSize(2) == 0 &&
emitted64 == packet64.data() &&
dcb64.cursor_up == packet64.data() + packet64.size() &&
packet64[0] == 0xc0027904u && packet64[1] == 0x342u &&
packet64[2] ==
(0xc8020000u |
(static_cast<uint32_t>(address64 >> 32u) & 0xffffu)) &&
packet64[3] == static_cast<uint32_t>(address64) &&
packet64[4] == 0xc0079300u && packet64[5] == 0x13u &&
packet64[6] == (static_cast<uint32_t>(address64) & ~0x7u) &&
packet64[7] ==
(static_cast<uint32_t>(address64 >> 32u) & 0x3ffffu) &&
packet64[8] == static_cast<uint32_t>(label64) &&
packet64[9] == static_cast<uint32_t>(label64 >> 32u) &&
packet64[10] == UINT32_MAX && packet64[11] == UINT32_MAX &&
packet64[12] == 25u && packet64[13] == 0xc0017904u &&
packet64[14] == 0x342u && packet64[15] == 0xc8000000u &&
execute(packet64.data(), packet64.size()),
"64-bit wait packet differs from the expected stream");
alignas(8) uint64_t patched_label64 = 0x11223344aabbccddull;
const auto patched_address64 = reinterpret_cast<uint64_t>(&patched_label64);
packet64[6] |= 0x5u;
packet64[7] |= 0x5a580000u;
Require(
"Pm4WaitPackets", "native 64-bit patch",
Gen5::AgcWaitRegMemPatchAddress(packet64.data(), &patched_label64) == 0 &&
Gen5::AgcWaitRegMemPatchReference(packet64.data(),
0xdeadbeefaabbccddull) == 0 &&
packet64[2] ==
(0xc8020000u |
(static_cast<uint32_t>(patched_address64 >> 32u) & 0xffffu)) &&
packet64[3] == static_cast<uint32_t>(patched_address64) &&
packet64[6] ==
((static_cast<uint32_t>(patched_address64) & ~0x7u) | 0x5u) &&
packet64[7] ==
((static_cast<uint32_t>(patched_address64 >> 32u) & 0x3ffffu) |
0x5a580000u) &&
packet64[8] == 0xaabbccddu && packet64[9] == 0x11223344u &&
execute(packet64.data(), packet64.size()),
"64-bit native wait patch did not preserve the native high reference "
"DWORD");
std::array<uint32_t, 16> invalid{};
const uint32_t short_payload = 0x342u;
Require(
"Pm4WaitPackets", "invalid packet",
Gen5::AgcWaitRegMemPatchAddress(invalid.data(), &label32) ==
packet_mismatch &&
Gen5::AgcWaitRegMemPatchReference(invalid.data(), label32) ==
packet_mismatch &&
!Gen5::AgcIsInternalDataPacket(
KYTY_PM4(2, Pm4::IT_SET_UCONFIG_REG, 1), &short_payload),
"wait patch accepted a packet without the metadata prefix");
std::printf("[host] %-32s ok\n", "Pm4WaitPackets");
}
void CheckPm4DrawIndirectMultiPacket(RenderContext &renderer) {
GraphicsInitJmpTables();
CommandProcessor processor(renderer, 0);
const auto execute = [&](uint32_t *packet, uint32_t size_dw) {
Pm4Execution execution;
return processor.Process(execution, {packet, size_dw}) ==
Pm4ProcessResult::Complete;
};
constexpr uint64_t modifier = 0x0000000069380b1dull;
constexpr auto packet_mismatch = static_cast<int>(0x8a6c000cu);
const auto *count_address = reinterpret_cast<const volatile void *>(
static_cast<uintptr_t>(0x123456789abcdef3ull));
constexpr std::array<uint32_t, 16> expected{
0xc0017904u, 0x00000342u, 0xc6000008u, 0xc0082c00u,
0x11223344u, 0x00000111u, 0x00000113u, 0xc8000115u,
0x55667788u, 0x9abcdef0u, 0x12345678u, 0xaabbccddu,
0x00000022u, 0xc0017904u, 0x00000342u, 0xc6000000u};
std::array<uint32_t, 16> packet{};
CommandBufferLayout dcb{packet.data(),
packet.data() + packet.size(),
packet.data(),
packet.data() + packet.size(),
nullptr,
nullptr,
0};
auto *emitted = Gen5::AgcDcbDrawIndirectMulti(
reinterpret_cast<Gen5::CommandBuffer *>(&dcb), 0x11223344u, 1u,
0x55667788u, count_address, 0xaabbccddu, modifier);
const uint32_t invalid_payload[]{0x342u, 0xc6000010u};
Require("Pm4DrawIndirectMulti", "native packet",
emitted == packet.data() &&
dcb.cursor_up == packet.data() + packet.size() &&
packet == expected && execute(packet.data(), 3u) &&
execute(packet.data() + 13u, 3u) &&
Gen5::AgcWaitRegMemPatchAddress(packet.data(), count_address) ==
packet_mismatch &&
Gen5::AgcWaitRegMemPatchReference(packet.data(), 0x12345678u) ==
packet_mismatch &&
!Gen5::AgcIsInternalDataPacket(0xc0017904u, invalid_payload),
"indirect multi-draw packet differs from the expected stream");
constexpr uint32_t data_offset = 0x5c08;
constexpr uint32_t stride = 32;
std::array<uint32_t, data_offset / 4 + stride / 4 + 5> arguments{};
const std::array<uint32_t, 5> first{0, 1, 3, 0xfffffffeu, 9};
const std::array<uint32_t, 5> second{0, 2, 5, 7, 11};
std::copy(first.begin(), first.end(), arguments.begin() + data_offset / 4);
std::copy(second.begin(), second.end(), arguments.begin() + (data_offset + stride) / 4);
std::array<uint32_t, 8> indices{};
processor.BufferInit();
processor.SetDrawIndirectArgsBaseAddress(reinterpret_cast<uint64_t>(arguments.data()));
processor.SetIndexBaseAddress(reinterpret_cast<uint64_t>(indices.data()));
processor.SetIndexType(0);
std::array<uint32_t, 5> captured{0xc0032500u, data_offset, 0x94, 0x95, 0};
Require("Pm4DrawIndirectMulti", "captured indexed single execution",
execute(captured.data(), captured.size()),
"indexed indirect draw rejected native source-select zero");
// Zero primitive counts still traverse CP argument decoding and renderer entry,
// while avoiding unrelated shader and attachment requirements.
for (const bool indexed : {false, true}) {
for (const bool multi : {false, true}) {
for (const uint64_t draw_modifier : {0ull, 0x100ull, 1ull << 32u, (1ull << 32u) | 0x100ull}) {
packet.fill(0);
dcb.cursor_up = packet.data();
auto *buffer = reinterpret_cast<Gen5::CommandBuffer *>(&dcb);
alignas(4) uint32_t draw_count = 3;
if (multi) {
emitted = indexed ? Gen5::AgcDcbDrawIndexIndirectMulti(buffer, data_offset, 1, 2,
&draw_count, stride, draw_modifier)
: Gen5::AgcDcbDrawIndirectMulti(buffer, data_offset, 1, 2,
&draw_count, stride, draw_modifier);
} else {
emitted = indexed ? Gen5::AgcDcbDrawIndexIndirect(buffer, data_offset, draw_modifier)
: Gen5::AgcDcbDrawIndirect(buffer, data_offset, draw_modifier);
}
const uint32_t begin = multi && !indexed ? 3 : 0;
const uint32_t words = multi ? (indexed ? 10 : 16) : 5;
const uint32_t opcode = multi ? (indexed ? 0xc0083800u : 0xc0082c00u)
: (indexed ? 0xc0032500u : 0xc0032400u);
const uint32_t initiator = (indexed ? 0 : 2) |
(draw_modifier == 0x100 ? 0x20 : 0);
Require("Pm4DrawIndirectMulti", "indexed/auto initiator emission",
emitted == packet.data() && dcb.cursor_up == packet.data() + words &&
packet[begin] == opcode && packet[begin + 1] == data_offset &&
packet[begin + (multi ? 9 : 4)] == initiator,
"indirect opcode, source select, or modifier bits differ from the native packet");
Require("Pm4DrawIndirectMulti", "single/multi argument execution",
execute(packet.data(), words),
"indirect draw failed shared argument decoding or padded/count-limited traversal");
if (multi) {
draw_count = 0;
Require("Pm4DrawIndirectMulti", "zero indirect count execution",
execute(packet.data(), words), "zero indirect draw count did not complete");
}
}
}
}
renderer.GetCommandScheduler().Finish();
processor.BufferInit();
auto &shaders = processor.GetShCtx();
const auto &gs = shaders.GetVs().gs_user_sgpr.value;
const auto &hs = shaders.GetVs().hs_user_sgpr.value;
const auto check_offsets = [&](auto &commands, const auto &sgprs,
std::initializer_list<uint32_t> expected_offsets,
const char *label) {
Require("Pm4DrawIndirectMulti", label,
execute(commands.data(), commands.size()) &&
std::equal(expected_offsets.begin(), expected_offsets.end(), sgprs + 9),
"indirect draw did not write the expected user SGPR offsets");
};
// These locations match the captured NGG prolog: user SGPR9/10 become s17/s18.
shaders.SetGsUserSgpr(9, 17821, HW::UserSgprType::Unknown);
shaders.SetGsUserSgpr(10, 0x898f3f24u, HW::UserSgprType::Unknown);
arguments = {0, 1};
std::array<uint32_t, 5> indexed_packet{
KYTY_PM4(5, Pm4::IT_DRAW_INDEX_INDIRECT, 0), 0,
Pm4::SPI_SHADER_USER_DATA_GS_0 + 9,
Pm4::SPI_SHADER_USER_DATA_GS_0 + 10, 0};
check_offsets(indexed_packet, gs, {0, 0}, "clear stale offsets");
arguments = {0, 1, 5, static_cast<uint32_t>(-17), 3};
indexed_packet[2] |= (Pm4::SPI_SHADER_USER_DATA_GS_0 + 11) << 16u;
// Native sceAgcDcbDrawIndexIndirect (0x5a00) emits first-index enable in word3 bit28.
indexed_packet[3] |= 1u << 28u;
check_offsets(indexed_packet, gs, {static_cast<uint32_t>(-17), 3, 5}, "indexed offsets");
indexed_packet[3] ^= (1u << 28u) | (1u << 27u);
arguments[2] = 6;
check_offsets(indexed_packet, gs, {static_cast<uint32_t>(-17), 3, 5},
"index destination disabled");
arguments = {0, 1, 31, 5};
std::array<uint32_t, 5> auto_packet{
KYTY_PM4(5, Pm4::IT_DRAW_INDIRECT, 0), 0,
Pm4::SPI_SHADER_USER_DATA_HS_0 + 9,
Pm4::SPI_SHADER_USER_DATA_HS_0 + 10, 2};
check_offsets(auto_packet, hs, {31, 5}, "non-indexed offsets");
Require("Pm4DrawIndirectMulti", "shader stage", gs[9] == static_cast<uint32_t>(-17),
"non-indexed draw patched the wrong shader stage");
arguments = {0, 1, 6, 41, 7, 0, 1, 8, 43, 9, 0, 1, 10, 45, 11};
uint32_t count = 2;
const auto multi_count_address = reinterpret_cast<uint64_t>(&count);
// Native sceAgcDcbDrawIndexIndirectMulti (0x5800) puts this enable in word4 bit28.
std::array<uint32_t, 10> multi_packet{
KYTY_PM4(10, Pm4::IT_DRAW_INDEX_INDIRECT_MULTI, 0), 0,
indexed_packet[2], Pm4::SPI_SHADER_USER_DATA_GS_0 + 10,
(1u << 30u) | (1u << 28u) | Pm4::SH_NOP, 3,
static_cast<uint32_t>(multi_count_address),
static_cast<uint32_t>(multi_count_address >> 32u), 20, 0};
check_offsets(multi_packet, gs, {43, 9, 8}, "counted multi-draw offsets");
count = 0;
check_offsets(multi_packet, gs, {43, 9, 8}, "empty multi-draw");
count = 3;
multi_packet[5] = 1;
multi_packet[3] |= 1u << 28u;
multi_packet[4] &= ~(1u << 28u);
check_offsets(multi_packet, gs, {41, 7, 8}, "multi-draw enable word and count clamp");
arguments = {0, 1, 51, 13, 0, 1, 53, 15};
std::array<uint32_t, 10> auto_multi_packet{
KYTY_PM4(10, Pm4::IT_DRAW_INDIRECT_MULTI, 0), 0,
auto_packet[2], auto_packet[3], Pm4::SH_NOP, 2, 0, 0, 16, 2};
check_offsets(auto_multi_packet, hs, {53, 15}, "non-indexed multi-draw offsets");
arguments = {0, 1, 99, 101};
auto_packet[2] = Pm4::SH_NOP;
check_offsets(auto_packet, hs, {53, 101}, "independent destinations");
auto_packet[3] = Pm4::SH_NOP;
arguments[3] = 103;
check_offsets(auto_packet, hs, {53, 101}, "disabled destinations");
processor.BufferFlush();
std::printf("[host] %-32s ok\n", "Pm4DrawIndirectMulti");
}
void CheckPm4ContextStateOperations(RenderContext &renderer) {
GraphicsInitJmpTables();
CommandProcessor processor(renderer, 0);
constexpr auto primitive_type = static_cast<Prospero::PrimitiveType>(0x35u);
HW::RenderControl original_control{};
original_control.depth_clear_enable = true;
original_control.stencil_clear_enable = true;
processor.GetCtx().SetRenderControl(original_control);
processor.GetCtx().SetRenderTargetMask(0x1234abcd);
processor.GetUcfg().SetPrimitiveType(primitive_type);
g_hw_sh_indirect_func[Pm4::SPI_SHADER_PGM_LO_PS](
processor, Pm4::SPI_SHADER_PGM_LO_PS, 0x00123456);
const auto shader_address = processor.GetShCtx().GetPs().ps_regs.data_addr;
const auto invoke = [&](ContextStateOperation operation) {
std::array<uint32_t, 32> packet{};
std::array<uint32_t, 6> segment_sizes{};
CommandBufferLayout dcb{packet.data(),
packet.data() + packet.size(),
packet.data(),
packet.data() + packet.size(),
nullptr,
nullptr,
0};
const auto operation_value = static_cast<uint32_t>(operation);
size_t segment_count = 0;
switch (operation) {
case ContextStateOperation::Clear:
segment_sizes = {5};
segment_count = 1;
break;
case ContextStateOperation::Push:
segment_sizes = {5, 8, 9, 3, 2};
segment_count = 5;
break;
case ContextStateOperation::Pop:
segment_sizes = {3, 5, 8, 9, 2};
segment_count = 5;
break;
case ContextStateOperation::PushClear:
segment_sizes = {5, 8, 9, 3, 2, 5};
segment_count = 6;
break;
}
const auto size_dw = static_cast<size_t>(
Gen5::AgcDcbContextStateOpGetSize(operation_value) / 4);
auto *emitted = Gen5::AgcDcbContextStateOp(
reinterpret_cast<Gen5::CommandBuffer *>(&dcb), operation_value);
bool packet_matches = true;
size_t segment_offset = 0;
for (size_t i = 0; i < segment_count; i++) {
const auto segment_size = segment_sizes[i];
const auto selector = (i == 0 ? Pm4::R_CONTEXT_STATE : Pm4::R_ZERO);
packet_matches &= packet[segment_offset] ==
KYTY_PM4(segment_size, Pm4::IT_NOP, selector);
for (size_t j = 1; j < segment_size; j++) {
packet_matches &= packet[segment_offset + j] ==
(i == 0 && j == 1 ? operation_value : 0u);
}
segment_offset += segment_size;
}
Require("Pm4ContextState", "HLE packet",
emitted == packet.data() &&
dcb.cursor_up == packet.data() + size_dw &&
segment_offset == size_dw && packet_matches,
"context-state HLE packet does not match its real allocation size");
Pm4Execution execution;
return processor.Process(execution, {packet.data(), size_dw});
};
Require("Pm4ContextState", "push-clear",
invoke(ContextStateOperation::PushClear) ==
Pm4ProcessResult::Complete &&
!processor.GetCtx().GetRenderControl().depth_clear_enable &&
!processor.GetCtx().GetRenderControl().stencil_clear_enable &&
processor.GetCtx().GetRenderTargetMask() ==
HW::Context{}.GetRenderTargetMask() &&
processor.GetUcfg().GetPrimType() == primitive_type &&
processor.GetShCtx().GetPs().ps_regs.data_addr == shader_address,
"push-clear did not save and clear only Cx state");
processor.GetCtx().SetRenderTargetMask(0x55aa55aa);
Require("Pm4ContextState", "pop",
invoke(ContextStateOperation::Pop) == Pm4ProcessResult::Complete &&
processor.GetCtx().GetRenderControl().depth_clear_enable &&
processor.GetCtx().GetRenderControl().stencil_clear_enable &&
processor.GetCtx().GetRenderTargetMask() == 0x1234abcd,
"pop did not restore the complete saved Cx state");
Require("Pm4ContextState", "push",
invoke(ContextStateOperation::Push) == Pm4ProcessResult::Complete &&
processor.GetCtx().GetRenderTargetMask() == 0x1234abcd,
"push changed live Cx state");
processor.GetCtx().SetRenderTargetMask(0x01020304);
Require("Pm4ContextState", "push-pop restore",
invoke(ContextStateOperation::Pop) == Pm4ProcessResult::Complete &&
processor.GetCtx().GetRenderTargetMask() == 0x1234abcd,
"ordinary push/pop did not restore Cx state");
Require("Pm4ContextState", "clear",
invoke(ContextStateOperation::Clear) == Pm4ProcessResult::Complete &&
!processor.GetCtx().GetRenderControl().depth_clear_enable &&
processor.GetCtx().GetRenderTargetMask() ==
HW::Context{}.GetRenderTargetMask() &&
processor.GetUcfg().GetPrimType() == primitive_type &&
processor.GetShCtx().GetPs().ps_regs.data_addr == shader_address,
"clear reset state outside the Cx register domain");
processor.GetCtx().SetRenderTargetMask(0x0badc0de);
Require("Pm4ContextState", "push before clear-state packet",
invoke(ContextStateOperation::Push) == Pm4ProcessResult::Complete,
"push before CLEAR_STATE failed");
processor.GetCtx().SetRenderTargetMask(0xfeedface);
std::array<uint32_t, 2> clear_state_packet{0xc0001200, 0};
Pm4Execution clear_state_execution;
Require("Pm4ContextState", "clear-state packet",
processor.Process(clear_state_execution, clear_state_packet) ==
Pm4ProcessResult::Complete &&
processor.GetCtx().GetRenderTargetMask() ==
HW::Context{}.GetRenderTargetMask(),
"CLEAR_STATE did not clear the current Cx state");
Require("Pm4ContextState", "pop after clear-state packet",
invoke(ContextStateOperation::Pop) == Pm4ProcessResult::Complete &&
processor.GetCtx().GetRenderTargetMask() == 0x0badc0de,
"CLEAR_STATE discarded the pushed Cx state");
// CLEAR_STATE restores the viewport, scissor, and guard-band register defaults.
processor.GetCtx().SetScreenScissor(1, 2, 3, 4);
processor.GetCtx().SetGuardBands(2.0f, 3.0f, 4.0f, 5.0f);
Pm4Execution scissor_reset;
Require("Pm4ContextState", "scissor reset packet",
processor.Process(scissor_reset, clear_state_packet) == Pm4ProcessResult::Complete,
"CLEAR_STATE could not reset the rasterization state");
const auto &viewport = processor.GetCtx().GetScreenViewport();
const auto defaults = calc_final_scissor(viewport, {}, {20000, 20000}, 0);
Require("Pm4ContextState", "Rasterization defaults",
defaults.left == 0 && defaults.top == 0 && defaults.right == 16384 &&
defaults.bottom == 16384 && viewport.guard_band_horz_clip == 1.0f &&
viewport.guard_band_vert_clip == 1.0f &&
viewport.guard_band_horz_discard == 1.0f &&
viewport.guard_band_vert_discard == 1.0f &&
std::ranges::all_of(viewport.viewports, [](const auto &slot) {
return slot.xscale == 1.0f && slot.yscale == 1.0f && slot.zscale == 1.0f &&
slot.viewport_scissor_right == 16384 &&
slot.viewport_scissor_bottom == 16384 &&
!slot.viewport_scissor_window_offset_enable;
}),
"CLEAR_STATE did not restore scissor and guard-band defaults");
const std::array scissor_registers{
std::pair{Pm4::PA_SC_SCREEN_SCISSOR_TL, Pm4::PA_SC_SCREEN_SCISSOR_BR},
std::pair{Pm4::PA_SC_WINDOW_SCISSOR_TL, Pm4::PA_SC_WINDOW_SCISSOR_BR},
std::pair{Pm4::PA_SC_GENERIC_SCISSOR_TL, Pm4::PA_SC_GENERIC_SCISSOR_BR},
std::pair{Pm4::PA_SC_VPORT_SCISSOR_0_TL, Pm4::PA_SC_VPORT_SCISSOR_0_BR}};
for (const auto &[tl, br] : scissor_registers) {
processor.GetCtx().Reset();
g_hw_ctx_indirect_func[tl](processor, tl, 0);
g_hw_ctx_indirect_func[br](processor, br, 0);
HW::ScanModeControl scan{};
scan.vport_scissor_enable = tl == Pm4::PA_SC_VPORT_SCISSOR_0_TL;
const auto empty = calc_final_scissor(processor.GetCtx().GetScreenViewport(),
scan, {3840, 2160}, 0);
Require("Pm4ContextState", "explicit empty scissor",
empty.right == empty.left && empty.bottom == empty.top,
"an explicitly empty scissor was expanded or ignored");
}
processor.GetCtx().SetRenderTargetMask(0xabcdef01);
Require("Pm4ContextState", "push before processor reset",
invoke(ContextStateOperation::Push) == Pm4ProcessResult::Complete,
"push before reset failed");
processor.Reset();
processor.GetCtx().SetRenderTargetMask(0x76543210);
Require("Pm4ContextState", "processor reset discards push",
invoke(ContextStateOperation::Push) == Pm4ProcessResult::Complete &&
invoke(ContextStateOperation::Pop) ==
Pm4ProcessResult::Complete &&
processor.GetCtx().GetRenderTargetMask() == 0x76543210,
"processor reset retained an invalid saved Cx state");
Require("Pm4ContextState", "HLE packet size",
Gen5::AgcDcbContextStateOpGetSize(0) == 20 &&
Gen5::AgcDcbContextStateOpGetSize(1) == 108 &&
Gen5::AgcDcbContextStateOpGetSize(2) == 108 &&
Gen5::AgcDcbContextStateOpGetSize(3) == 128 &&
Gen5::AgcDcbContextStateOpGetSize(4) == 0,
"context-state HLE sizes do not match the expected packet layout");
std::printf("[host] %-32s ok\n", "Pm4ContextState");
}
void CheckPm4IndirectControlFlow(RenderContext &renderer) {
constexpr const char *name = "Pm4IndirectControlFlow";
GraphicsInitJmpTables();
CommandProcessor processor(renderer, 0);
uint32_t selected = 0;
uint32_t returned = 0;
const auto address = [](const void *value) {
return reinterpret_cast<uint64_t>(value);
};
const auto write = [&](uint32_t *destination, uint32_t value) {
return std::array<uint32_t, 5>{
KYTY_PM4(5, Pm4::IT_WRITE_DATA, 0), 0,
static_cast<uint32_t>(address(destination)),
static_cast<uint32_t>(address(destination) >> 32u), value};
};
const auto indirect = [&](std::span<const uint32_t> target, bool chain) {
uint32_t control = 0x0f200000u | static_cast<uint32_t>(target.size());
if (chain) {
control |= 1u << 20u;
}
return std::array<uint32_t, 4>{
KYTY_PM4(4, Pm4::IT_INDIRECT_BUFFER, 0),
static_cast<uint32_t>(address(target.data())),
static_cast<uint32_t>(address(target.data()) >> 32u), control};
};
// A one-word NOP must leave the following completion write reachable.
std::array<uint32_t, 7> nop_commands{};
CommandBufferLayout nop_dcb{nop_commands.data(), nop_commands.data() + nop_commands.size(),
nop_commands.data(), nop_commands.data() + nop_commands.size(), nullptr, nullptr, 0};
auto* nop_cb = reinterpret_cast<Gen5::CommandBuffer*>(&nop_dcb);
Require(name, "zero-sized NOP", Gen5::AgcCbNop(nop_cb, 0) == nullptr &&
nop_dcb.cursor_up == nop_commands.data() && nop_commands[0] == 0,
"zero-sized reservation changed the command buffer");
Require(name, "empty command list NOP", Gen5::AgcCbNop(nop_cb, 1) == nop_commands.data() &&
nop_dcb.cursor_up == nop_commands.data() + 1 && nop_commands[0] == 0xffff1000u &&
Gen5::AgcCbNopGetSize(1) == 4 && Gen5::AgcGetPacketSize(nop_commands.data()) == 1,
"native NOP(1) did not reserve one DWORD");
const auto completion = write(&selected, 7);
std::copy(completion.begin(), completion.end(), nop_commands.begin() + 1);
nop_commands.back() = 0xffff1000u;
for (uint32_t predication : {1u, 0u}) {
Gen5::AgcSetRangePredication(nop_commands.data(), nop_commands.data() + nop_commands.size(), predication);
Require(name, "NOP range predication", nop_commands[0] == (0xffff1000u | predication) &&
nop_commands.back() == (0xffff1000u | predication) &&
nop_commands[1] == (completion[0] | predication) && nop_commands[5] == 7,
"range predication skipped a packet or modified a payload");
Pm4Execution execution;
selected = 0;
Require(name, "NOP and completion execution",
processor.Process(execution, nop_commands) == Pm4ProcessResult::Complete && selected == 7,
"one-DWORD NOP prevented the completion write or overread the stream end");
}
Common::File dump;
Require(name, "NOP dump file", dump.CreateInMem(), "could not create PM4 dump");
Pm4::DumpPm4PacketStream(&dump, nop_commands.data(), 0, nop_commands.size());
std::array<uint32_t, 3> payload{0, 0x12345678u, 0x9abcdef0u};
CommandBufferLayout payload_dcb{payload.data(), payload.data() + payload.size(),
payload.data(), payload.data() + payload.size(), nullptr, nullptr, 0};
Require(name, "NOP payload reservation",
Gen5::AgcCbNop(reinterpret_cast<Gen5::CommandBuffer*>(&payload_dcb), 3) == payload.data() &&
payload == std::array<uint32_t, 3>{0xc0011000u, 0x12345678u, 0x9abcdef0u},
"NOP emitter overwrote the caller's reserved payload");
for (const uint32_t value : {0x68750820u, 0x68751777u, 0x68751004u}) {
std::array<uint32_t, 8> commands{KYTY_PM4(3, Pm4::IT_NOP, Pm4::R_ZERO), value, 0x12345678u};
std::copy(completion.begin(), completion.end(), commands.begin() + 3);
processor.SetUserDataMarker(HW::UserSgprType::Unknown);
selected = 0;
Pm4Execution execution;
Require(name, "opaque NOP payload",
processor.Process(execution, commands) == Pm4ProcessResult::Complete && selected == 7 &&
processor.GetUserDataMarker() == HW::UserSgprType::Unknown && commands[1] == value,
"a NOP payload prefix executed a marker or blocked the following packet");
}
const std::array<uint32_t, 2> vsharp_marker{
KYTY_PM4(2, Pm4::IT_NOP, Pm4::R_ZERO), 0x68750004u};
Pm4Execution marker_execution;
Require(name, "exact user-data marker",
processor.Process(marker_execution, vsharp_marker) == Pm4ProcessResult::Complete &&
processor.GetUserDataMarker() == HW::UserSgprType::Vsharp,
"exact marker dispatch lost the existing user-data annotation");
processor.SetUserDataMarker(HW::UserSgprType::Unknown);
alignas(8) uint64_t released = 0;
std::array<uint32_t, 20> disabled_commands{};
const auto disabled_write = write(&returned, 99);
std::copy(disabled_write.begin(), disabled_write.end(), disabled_commands.begin());
disabled_commands[5] = KYTY_PM4(2, Pm4::IT_NUM_INSTANCES, 0);
disabled_commands[6] = 0x68750123u;
CommandBufferLayout release_dcb{disabled_commands.data() + 7, disabled_commands.data() + 15,
disabled_commands.data() + 7, disabled_commands.data() + 15, nullptr, nullptr, 0};
auto* release = Gen5::AgcCbReleaseMem(
reinterpret_cast<Gen5::CommandBuffer*>(&release_dcb), 0x28, 0, 0, 0,
reinterpret_cast<const volatile Gen5::Label*>(&released), 2, 77, 0, 0, 0, 0);
const auto enabled_write = write(&selected, 88);
std::copy(enabled_write.begin(), enabled_write.end(), disabled_commands.begin() + 15);
const auto unpatched_commands = disabled_commands;
Require(name, "SetNop packets",
release == disabled_commands.data() + 7 &&
Gen5::AgcSetNop(disabled_commands.data()) == 0 &&
Gen5::AgcSetNop(disabled_commands.data() + 5) == 0 && Gen5::AgcSetNop(release) == 0 &&
disabled_commands[0] == ((disabled_write[0] & 0xffff00ffu) | (Pm4::IT_NOP << 8u)) &&
disabled_commands[5] == KYTY_PM4(2, Pm4::IT_NOP, Pm4::R_ZERO) &&
disabled_commands[6] == 0x123u &&
release[0] == KYTY_PM4(8, Pm4::IT_NOP, Pm4::R_ZERO) &&
unpatched_commands[7] == KYTY_PM4(8, Pm4::IT_NOP, Pm4::R_RELEASE_MEM) &&
Gen5::AgcGetPacketSize(disabled_commands.data()) == disabled_write.size() &&
Gen5::AgcGetPacketSize(disabled_commands.data() + 5) == 2 &&
Gen5::AgcGetPacketSize(release) == 8 &&
std::equal(disabled_commands.begin() + 1, disabled_commands.begin() + 5,
unpatched_commands.begin() + 1) &&
std::equal(disabled_commands.begin() + 8, disabled_commands.end(),
unpatched_commands.begin() + 8),
"SetNop changed a packet length, kept its operation or marker, or touched its payload");
selected = returned = 0;
Pm4Execution disabled_execution;
Require(name, "SetNop execution",
processor.Process(disabled_execution, disabled_commands) == Pm4ProcessResult::Complete &&
returned == 0 && released == 0 && selected == 88,
"a disabled packet still executed or hid the packet after it");
const auto then_commands = write(&selected, 11);
const auto else_commands = write(&selected, 33);
const auto branch_suffix = write(&selected, 44);
const auto caller_suffix = write(&returned, 22);
alignas(8) uint64_t condition = 0;
std::array<uint32_t, 19> branch{
KYTY_PM4(14, Pm4::IT_INDIRECT_BUFFER, 0), 0,
static_cast<uint32_t>(address(&condition)),
static_cast<uint32_t>(address(&condition) >> 32u), UINT32_MAX,
UINT32_MAX, 1, 0,
static_cast<uint32_t>(address(then_commands.data())),
static_cast<uint32_t>(address(then_commands.data()) >> 32u),
static_cast<uint32_t>(then_commands.size()),
static_cast<uint32_t>(address(else_commands.data())),
static_cast<uint32_t>(address(else_commands.data()) >> 32u),
static_cast<uint32_t>(else_commands.size())};
std::copy(branch_suffix.begin(), branch_suffix.end(), branch.begin() + 14);
std::array<uint32_t, 9> caller{};
const auto call_branch = indirect(branch, false);
std::copy(call_branch.begin(), call_branch.end(), caller.begin());
std::copy(caller_suffix.begin(), caller_suffix.end(), caller.begin() + 4);
for (uint32_t mode : {1u, 2u}) {
branch[1] = mode | (3u << 8u);
for (uint64_t value : {0ull, 1ull}) {
condition = value;
selected = returned = 0;
uint32_t expected = 44;
if (value == 1) {
expected = 11;
} else if (mode == 2) {
expected = 33;
}
Pm4Execution execution;
Require(name, "conditional fetcher replacement",
processor.Process(execution, caller) == Pm4ProcessResult::Complete &&
selected == expected && returned == 22,
"taken branch resumed its discarded suffix or lost the caller's return");
}
}
std::array<uint32_t, 5> cx_packet{};
CommandBufferLayout cx_dcb{cx_packet.data(), cx_packet.data() + cx_packet.size(),
cx_packet.data(), cx_packet.data() + cx_packet.size(),
nullptr, nullptr, 0};
const ShaderRegister cx_register{Pm4::CB_TARGET_MASK, 0x76543210u};
Require(name, "indirect context packet size",
Gen5::AgcDcbSetCxRegistersIndirectGetSize() == sizeof(cx_packet) &&
Gen5::AgcDcbSetCxRegistersIndirect(
reinterpret_cast<Gen5::CommandBuffer *>(&cx_dcb), &cx_register, 1) ==
cx_packet.data() && cx_dcb.cursor_up == cx_dcb.top,
"indirect Cx size did not match the emitted command buffer");
std::array<uint32_t, 14> patched_branch{};
CommandBufferLayout branch_dcb{
patched_branch.data(), patched_branch.data() + patched_branch.size(),
patched_branch.data(), patched_branch.data() + patched_branch.size(),
nullptr, nullptr, 0};
Require(name, "branch packet size",
Gen5::AgcCbBranchGetSize() == sizeof(patched_branch) &&
Gen5::AgcCbBranch(reinterpret_cast<Gen5::CommandBuffer *>(&branch_dcb),
1, 0, &condition, 0, 0, 0, nullptr, 0,
3, else_commands.data(), else_commands.size()) ==
patched_branch.data() && branch_dcb.cursor_up == branch_dcb.top,
"branch size did not match the emitted command buffer");
patched_branch[8] = 3;
patched_branch[10] = 0xc5a00000u;
const auto unpatched = patched_branch;
Require(name, "patched indirect context branch",
Gen5::AgcBranchPatchSetThenTarget(patched_branch.data(), 2,
cx_packet.data(), cx_packet.size()) == 0 &&
(patched_branch[8] & 3u) == 3u &&
patched_branch[10] == 0xe5a00005u &&
std::equal(patched_branch.begin(), patched_branch.begin() + 8,
unpatched.begin()) &&
std::equal(patched_branch.begin() + 11, patched_branch.end(),
unpatched.begin() + 11),
"branch target patch lost reserved fields or changed the condition/else target");
Pm4Execution patched_execution;
Require(name, "patched branch execution",
processor.Process(patched_execution, patched_branch) == Pm4ProcessResult::Complete &&
processor.GetCtx().GetRenderTargetMask() == cx_register.value,
"patched branch did not fetch and apply its indirect context registers");
patched_branch[0] = KYTY_PM4(14, Pm4::IT_NOP, 0);
const auto mismatched = patched_branch;
Require(name, "branch packet mismatch",
Gen5::AgcBranchPatchSetThenTarget(patched_branch.data(), 0,
then_commands.data(), then_commands.size()) ==
static_cast<int>(0x8a6c000cu) && patched_branch == mismatched,
"branch patch accepted or changed a different packet type");
// This stream exceeds native recursion capacity, while chains need one fetcher cursor.
std::vector<std::array<uint32_t, 4>> links(65536);
std::span<const uint32_t> target = then_commands;
for (auto &link : links) {
link = indirect(target, true);
target = link;
}
const auto call_chain = indirect(target, false);
std::copy(call_chain.begin(), call_chain.end(), caller.begin());
selected = returned = 0;
Pm4Execution execution;
Require(name, "long chain and call return",
processor.Process(execution, caller) == Pm4ProcessResult::Complete &&
selected == 11 && returned == 22,
"long chain did not complete and return to the original caller");
std::printf("[host] %-32s ok\n", name);
}
void CheckPm4WaitResume(RenderContext &renderer) {
GraphicsInitJmpTables();
CommandProcessor processor(renderer, 0);
uint32_t label = 0;
uint32_t prefix = 0;
uint32_t child_observation = UINT32_MAX;
uint32_t suffix = 0;
const auto address = [](const void *value) {
return reinterpret_cast<uint64_t>(value);
};
std::array<uint32_t, 12> child{};
child[0] = KYTY_PM4(5, Pm4::IT_WRITE_DATA, 0);
child[1] = 0;
child[2] = static_cast<uint32_t>(address(&child_observation));
child[3] = static_cast<uint32_t>(address(&child_observation) >> 32u);
child[4] = 0;
child[5] = KYTY_PM4(7, Pm4::IT_WAIT_REG_MEM, 0);
child[6] = 0x10u | 3u;
child[7] = static_cast<uint32_t>(address(&label));
child[8] = static_cast<uint32_t>(address(&label) >> 32u);
child[9] = 1;
child[10] = UINT32_MAX;
std::array<uint32_t, 4> nested{};
nested[0] = KYTY_PM4(4, Pm4::IT_INDIRECT_BUFFER, 0);
nested[1] = static_cast<uint32_t>(address(child.data()));
nested[2] = static_cast<uint32_t>(address(child.data()) >> 32u);
nested[3] = 0x0f300000u | static_cast<uint32_t>(child.size());
std::array<uint32_t, 14> commands{};
commands[0] = KYTY_PM4(5, Pm4::IT_WRITE_DATA, 0);
commands[1] = 0;
commands[2] = static_cast<uint32_t>(address(&prefix));
commands[3] = static_cast<uint32_t>(address(&prefix) >> 32u);
commands[4] = 11;
commands[5] = KYTY_PM4(4, Pm4::IT_INDIRECT_BUFFER, 0);
commands[6] = static_cast<uint32_t>(address(nested.data()));
commands[7] = static_cast<uint32_t>(address(nested.data()) >> 32u);
commands[8] = 0x0f200000u | static_cast<uint32_t>(nested.size());
commands[9] = KYTY_PM4(5, Pm4::IT_WRITE_DATA, 0);
commands[10] = 0;
commands[11] = static_cast<uint32_t>(address(&suffix));
commands[12] = static_cast<uint32_t>(address(&suffix) >> 32u);
commands[13] = 22;
Pm4Execution execution;
Require("Pm4WaitResume", "suspend",
processor.Process(execution, commands) == Pm4ProcessResult::Blocked &&
prefix == 11 && child_observation == 0 && suffix == 0,
"blocked indirect wait did not preserve its command position");
Require("Pm4WaitResume", "still blocked",
processor.Process(execution, commands) == Pm4ProcessResult::Blocked &&
!execution.MadeProgress() && suffix == 0,
"blocked wait advanced or replayed its caller");
label = 1;
child[4] = 1;
Require(
"Pm4WaitResume", "resume",
processor.Process(execution, commands) == Pm4ProcessResult::Complete &&
prefix == 11 && child_observation == 0 && suffix == 22,
"resumed indirect wait replayed a child or did not finish its parent");
std::printf("[host] %-32s ok\n", "Pm4WaitResume");
}
void CheckPm4RewindResume(RenderContext &renderer) {
constexpr const char *name = "Pm4RewindResume";
Loader::SymbolDatabase symbols;
Libs::InitAgcDriver_1(&symbols);
const auto *emit_symbol = symbols.FindByNid("DwICrVxerkY", Loader::SymbolType::Func);
const auto *patch_symbol = symbols.FindByNid("eWaWyFegzgQ", Loader::SymbolType::Func);
Require(name, "async exports",
emit_symbol != nullptr && patch_symbol != nullptr &&
emit_symbol->vaddr == reinterpret_cast<uint64_t>(&Gen5::AgcAcbRewind) &&
patch_symbol->vaddr ==
reinterpret_cast<uint64_t>(&Gen5::AgcRewindPatchSetRewindState),
"async rewind imports did not resolve to the emitter and state patch");
const auto async_patch = reinterpret_cast<decltype(&Gen5::AgcRewindPatchSetRewindState)>(
patch_symbol->vaddr);
constexpr uint32_t sentinel = 0x12345678u;
std::array<uint32_t, 4> packet{sentinel, sentinel, sentinel, sentinel};
CommandBufferLayout acb{packet.data(), packet.data() + packet.size(),
packet.data() + 1, packet.data() + 3, nullptr, nullptr, 0};
auto *buffer = reinterpret_cast<Gen5::CommandBuffer *>(&acb);
for (const uint8_t state : {0, 1}) {
for (const uint8_t offload : {0, 1}) {
acb.cursor_up = packet.data() + 1;
Require(name, "async packet",
Gen5::AgcAcbRewind(buffer, state, offload) == packet.data() + 1 &&
acb.cursor_up == packet.data() + 3 &&
packet[0] == sentinel && packet[3] == sentinel &&
packet[1] == 0xc0005900u &&
packet[2] == ((uint32_t{state} << 31u) | (uint32_t{offload} << 24u)),
"async rewind packet or two-DWORD allocation differs from the native command");
}
}
const auto before = packet;
acb.cursor_up = packet.data() + 1;
acb.reserved_dw = 1;
Require(name, "bounded allocation",
Gen5::AgcAcbRewind(buffer, 1, 1) == nullptr &&
Gen5::AgcAcbRewind(nullptr, 1, 1) == nullptr &&
acb.cursor_up == packet.data() + 1 && packet == before,
"failed rewind allocation changed the cursor or command storage");
// Native patchers inspect only the opcode, preserving the header and low 31 bits.
std::array<uint32_t, 2> command{0xa5ff591fu, 0x0155aa55u};
Require(name, "patch preserves fields",
async_patch(command.data(), 1) == 0 && command[0] == 0xa5ff591fu &&
command[1] == 0x8155aa55u && async_patch(command.data(), 0) == 0 &&
command[0] == 0xa5ff591fu && command[1] == 0x0155aa55u,
"rewind patch changed the offload bit, header, or unrelated payload bits");
command[0] = 0xc0001000u;
const auto invalid = command;
Require(name, "wrong opcode",
async_patch(command.data(), 1) == static_cast<int>(0x8a6c000cu) &&
command == invalid,
"rewind patch accepted or modified a packet with the wrong opcode");
GraphicsInitJmpTables();
CommandProcessor processor(renderer, 0);
for (const bool async : {false, true}) {
for (const uint8_t offload : {0, 1}) {
if (!async && offload != 0) {
continue;
}
uint32_t suffix = 0;
const auto address = reinterpret_cast<uint64_t>(&suffix);
std::array<uint32_t, 7> commands{
0, 0, KYTY_PM4(5, Pm4::IT_WRITE_DATA, 0), 0,
static_cast<uint32_t>(address), static_cast<uint32_t>(address >> 32u), 33};
CommandBufferLayout cb{commands.data(), commands.data() + 2,
commands.data(), commands.data() + 2, nullptr, nullptr, 0};
auto *command_buffer = reinterpret_cast<Gen5::CommandBuffer *>(&cb);
const auto *emitted = async ? Gen5::AgcAcbRewind(command_buffer, 0, offload)
: Gen5::AgcDcbRewind(command_buffer, 0);
const auto patch = async ? async_patch : &Gen5::AgcRewindPatchSetRewindState;
Pm4Execution execution;
Require(name, "pending",
emitted == commands.data() &&
processor.Process(execution, commands) == Pm4ProcessResult::Blocked &&
suffix == 0,
"pending rewind did not preserve its command position");
Require(name, "still pending",
processor.Process(execution, commands) == Pm4ProcessResult::Blocked &&
!execution.MadeProgress() && suffix == 0,
"pending rewind advanced before its state was patched");
Require(name, "patch and resume",
patch(commands.data(), 1) == 0 &&
commands[1] == (0x80000000u | (uint32_t{offload} << 24u)) &&
processor.Process(execution, commands) == Pm4ProcessResult::Complete &&
suffix == 33,
"patched rewind lost the offload bit or failed to resume");
}
}
std::printf("[host] %-32s ok\n", name);
}
void CheckPm4CeCompletion(RenderContext &renderer) {
GraphicsInitJmpTables();
CommandProcessor processor(renderer, 0);
uint32_t suffix = 0;
const auto address = reinterpret_cast<uint64_t>(&suffix);
std::array<uint32_t, 9> commands{};
commands[0] = 0xc0008600u;
commands[1] = 1;
commands[2] = 0xc0008500u;
commands[3] = 0;
commands[4] = KYTY_PM4(5, Pm4::IT_WRITE_DATA, 0);
commands[5] = 0;
commands[6] = static_cast<uint32_t>(address);
commands[7] = static_cast<uint32_t>(address >> 32u);
commands[8] = 33;
processor.ResetDeCe();
Pm4Execution execution;
Require("Pm4CeCompletion", "wait",
processor.Process(execution, commands) == Pm4ProcessResult::Blocked &&
suffix == 0,
"DE did not wait for an active CE stream");
std::array<uint32_t, 2> ce_increment{0xc0008400u, 1};
Pm4Execution ce_execution;
Require("Pm4CeCompletion", "CE increment",
processor.Process(ce_execution, ce_increment) ==
Pm4ProcessResult::Complete,
"CE counter increment did not complete");
Require(
"Pm4CeCompletion", "counter gate",
processor.Process(execution, commands) == Pm4ProcessResult::Complete &&
suffix == 33,
"DE did not resume after CE advanced or failed to increment its counter");
suffix = 0;
Pm4Execution balanced_execution;
Require("Pm4CeCompletion", "balanced wait",
processor.Process(balanced_execution, commands) ==
Pm4ProcessResult::Blocked &&
suffix == 0,
"balanced CE/DE counters did not gate the next DE packet");
processor.SetCeComplete(true);
Require("Pm4CeCompletion", "stream complete",
processor.Process(balanced_execution, commands) ==
Pm4ProcessResult::Complete &&
suffix == 33,
"DE remained blocked after the CE stream completed");
std::printf("[host] %-32s ok\n", "Pm4CeCompletion");
}
} // namespace
} // namespace Libs::Graphics
int main(int argc, char **argv) {
using namespace Libs::Graphics;
std::setvbuf(stdout, nullptr, _IONBF, 0);
EnsureConfigInitialized();
CheckLeastRecentlyUsedCacheOrdering();
if (argc == 2 && std::strcmp(argv[1], "--thread-dimensions-only") == 0) {
VulkanHarness vulkan;
CheckComputeThreadDimensions(vulkan);
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--lds-limit-only") == 0) {
VulkanHarness vulkan;
CheckComputeLdsLimit(vulkan);
for (u32 wave_size : {32u, 64u})
for (bool gds : {false, true})
RunCase(&vulkan, DsReadWriteFullRegionAddress(wave_size, gds));
RunCase(&vulkan, DsWideLdsPartialBounds());
RunCase(&vulkan, DsAtomic64Bounds(false));
RunCase(&vulkan, DsAtomic64Bounds(true));
for (auto test : {DsWideLdsPartialBounds(), DsAtomic64Bounds(false), DsAtomic64Bounds(true), DsAtomicReturn64(),
DsCompareStoreReturn32()}) {
test.compute_info.lds_storage = true;
std::erase(test.required_spirv, "WorkgroupMemoryExplicitLayoutKHR");
test.required_spirv.push_back("StorageBuffer");
RunCase(&vulkan, test);
}
vulkan.CheckNativeIndirectDispatch();
return 0;
}
if (argc == 2 && (std::strcmp(argv[1], "--integer64-compare-only") == 0 ||
std::strcmp(argv[1], "--cmp-i64-only") == 0)) {
VulkanHarness vulkan;
RunCase(&vulkan, VectorCompareInteger64Edges());
RunCase(&vulkan, VectorCompareExecWaveMasks(32));
RunCase(&vulkan, VectorCompareExecWaveMasks(64));
RunCase(&vulkan, VectorCompare64WaveMasks(32, true));
RunCase(&vulkan, VectorCompare64WaveMasks(64, true));
RunCase(&vulkan, VectorVop3CompareEqI64OnGpu());
RunCase(&vulkan, VectorVop3CompareEqU64OnGpu());
RunCase(&vulkan, VectorVop3CompareGtU64OnGpu());
RunCase(&vulkan, VectorVopcCompareLtU64OnGpu());
RunCase(&vulkan, VectorVop3CompareNeU64OnGpu());
RunCase(&vulkan, VectorVopcCmpxNeU64CapturedExecMask());
RunCase(&vulkan, VectorVop3CmpxNeI64CapturedExecMask());
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--new-opcodes-only") == 0) {
VulkanHarness vulkan;
RunCase(&vulkan, VectorCompareInteger64Edges());
RunCase(&vulkan, VectorCompare64WaveMasks(32, true));
RunCase(&vulkan, VectorCompare64WaveMasks(64, true));
RunCase(&vulkan, VectorCompareExecWaveMasks(32));
RunCase(&vulkan, VectorCompareExecWaveMasks(64));
RunCase(&vulkan, VectorCompareF64Edges());
RunCase(&vulkan, VectorF64ModesModifiersAndExec());
RunCase(&vulkan, VectorCompareF16Ops());
RunCase(&vulkan, ScalarBrevB64OperandsAndMasks(32));
RunCase(&vulkan, ScalarBrevB64OperandsAndMasks(64));
for (bool add : {false, true}) {
RunCase(&vulkan, DsAtomic64Bounds(add));
for (u32 wave_size : {32, 64}) {
RunCase(&vulkan, DsAtomic64Contention(add, wave_size));
}
}
RunCase(&vulkan, DsAtomicReturn64());
RunCase(&vulkan, BufferAtomicAndX2GlcAndExec());
RunCase(&vulkan, BufferAtomicAddX2GlcAndExec());
RunCase(&vulkan, ImageAtomicSignedMinMax<false>());
RunCase(&vulkan, ImageAtomicSignedMinMax<true>());
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--s-memrealtime-only") == 0) {
VulkanHarness vulkan;
RunCase(&vulkan, ScalarMemRealtimeCapturedPlaceholder());
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--packed-integer-neg-only") == 0) {
VulkanHarness vulkan;
RunCase(&vulkan, Vop3pIntegerNegationCapturedAndSelectedHalves());
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--dpp-only") == 0) {
VulkanHarness vulkan;
RunGraphicsCase(&vulkan, GraphicsSparseWaveMinimum());
RunCase(&vulkan, ComputeWave64Minimum(false));
RunCase(&vulkan, ComputeWave64Minimum(true));
RunCase(&vulkan, VectorDpp8Captured(false));
for (const u32 wave_size : {32u, 64u})
for (const bool fi : {false, true})
RunCase(&vulkan, VectorDpp8Captured(true, wave_size, fi));
RunCase(&vulkan, VectorDppQuadPermuteReverse());
RunCase(&vulkan, VectorDppRowXmask());
RunCase(&vulkan, VectorDppRowShare(32));
RunCase(&vulkan, VectorDppRowShare(64));
RunCase(&vulkan, VectorDppBankMaskPreservesDestination());
RunCase(&vulkan, VectorDppBoundsControlZeroPreservesDestination());
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--cmpx-o-f32-only") == 0) {
VulkanHarness vulkan;
RunCase(&vulkan, VectorVopcCmpxOrderedCapturedExecMask());
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--fp32-compare-only") == 0) {
VulkanHarness vulkan;
RunCase(&vulkan, VectorCompareF32ExposureGuard());
for (const auto mode : {0xc0u, 0xe0u}) {
RunCase(&vulkan, VectorCompareF32DenormalModes(mode));
}
RunCase(&vulkan, VectorCompareClassF32());
RunCase(&vulkan, VectorCompareClassF32ImmediateAndDynamicMasks());
RunCase(&vulkan, VectorCompareF16Ops());
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--int16-alu-only") == 0) {
VulkanHarness vulkan;
RunCase(&vulkan, VectorCompareI16U16Ops());
RunCase(&vulkan, Vop3Int16Ops());
RunCase(&vulkan, Vop3pInt16Ops());
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--scalar-zero-bit-only") == 0) {
VulkanHarness vulkan;
RunCase(&vulkan, ScalarZeroBitOps());
RunCase(&vulkan, ScalarBitfieldPack());
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--buffer-format-store-only") == 0) {
VulkanHarness vulkan;
RunCase(&vulkan, BufferStoreFormatXyzwPackedUnormSkinningVectors());
RunCase(&vulkan, BufferStoreFormatXZeroFillsPackedRecord());
RunCase(&vulkan, BufferStoreFormatXyzwFloat16ConvertsComponents());
RunCase(&vulkan, BufferStoreFormatXyzwSnorm16CapturedSkinningVectors());
RunCase(&vulkan, BufferStoreFormatXSnorm16ClampsRoundsAndPreservesHalfwords());
for (const auto &test : BufferLoadFormatD16XCases()) {
RunCase(&vulkan, test);
}
for (const auto &test : BufferLoadFormatD16XyzwCases()) {
RunCase(&vulkan, test);
}
for (const auto &test : BufferStoreFormatD16Cases()) {
RunCase(&vulkan, test);
}
for (const auto &test : FormattedStoreConversionCases()) {
RunCase(&vulkan, test);
}
RunCase(&vulkan, BufferStoreFormatXResource16UintWritesHalfword());
RunCase(&vulkan, BufferStoreFormatXResource16UintPreservesAdjacentLanes());
RunCase(&vulkan, BufferStoreFormatXyResource88UintWritesBytes());
RunCase(&vulkan, BufferStoreFormatXyzwDropsPartialRecord());
RunCase(&vulkan, BufferStoreFormatXRejectsPartialRecord());
RunCase(&vulkan, BufferStoreFormatXyRejectsPartialRecord());
RunCase(&vulkan, BufferFormatStoreVariants());
RunCase(&vulkan, TBufferStoreFormatXSintWritesSubDword());
for (bool masked : {false, true}) {
RunCase(&vulkan, BufferStoreD16Hi(false, masked, 64));
for (u32 wave_size : {32u, 64u})
RunCase(&vulkan, BufferStoreD16Hi(true, masked, wave_size));
}
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--buffer-float16-store-only") == 0) {
VulkanHarness vulkan;
RunCase(&vulkan, BufferStoreFormatXyzwFloat16ConvertsComponents());
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--s-ashr-i64-only") == 0) {
VulkanHarness vulkan;
RunCase(&vulkan, ScalarAshrI64Edges(false));
RunCase(&vulkan, ScalarAshrI64Edges(true));
RunCase(&vulkan, ScalarAshrI64OperandsAndAliases());
RunCase(&vulkan, ScalarAshrI64Masks(32));
RunCase(&vulkan, ScalarAshrI64Masks(64));
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--tessellation-only") == 0) {
CheckTessellationPrograms();
CheckEmbeddedFetchVertexOffset();
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--cmpx-eq-u16-only") == 0) {
VulkanHarness vulkan;
RunCase(&vulkan, VectorVopcCmpxEqU16SdwaCompactVop3ExecMask());
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--cmpx-ne-u16-only") == 0) {
VulkanHarness vulkan;
RunCase(&vulkan, VectorVopcCmpxNeU16SdwaCompactVop3ExecMask());
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--cmpx-lt-i16-only") == 0) {
VulkanHarness vulkan;
RunCase(&vulkan, VectorVopcCmpxLtI16CapturedSdwaExecMask());
RunCase(&vulkan, VectorVopcCmpxLtI16WaveMasks(32));
RunCase(&vulkan, VectorVopcCmpxLtI16WaveMasks(64));
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--cmpx-lt-u16-only") == 0) {
VulkanHarness vulkan;
RunCase(&vulkan, VectorVopcCmpxLtU16CapturedSdwaExecMask());
RunCase(&vulkan, VectorVopcCmpxGtU16CapturedSdwaExecMask());
RunCase(&vulkan, VectorVopcSdwaCmpxWritesExecMask());
RunCase(&vulkan, VectorVop3CmpxWritesExecMask());
RunCase(&vulkan, VectorVopcSdwaCmpxClassF32CapturedExecMask());
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--class-f16-only") == 0) {
VulkanHarness vulkan;
RunCase(&vulkan, VectorCmpClassF16(32));
RunCase(&vulkan, VectorCmpClassF16(64));
RunCase(&vulkan, VectorCmpxClassF16(32));
RunCase(&vulkan, VectorCmpxClassF16(64));
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--cmpx-class-f16-only") == 0) {
VulkanHarness vulkan;
RunCase(&vulkan, VectorCmpxClassF16(32));
RunCase(&vulkan, VectorCmpxClassF16(64));
RunCase(&vulkan, VectorCompareClassF32());
RunCase(&vulkan, VectorVopcSdwaCmpxClassF32CapturedExecMask());
RunCase(&vulkan, FloatInlineConstantF16UsesNumericValue());
RunCase(&vulkan, VectorCompareF16Ops());
RunCase(&vulkan, VectorMinMaxF16Ops());
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--cmp-eq-f64-only") == 0) {
VulkanHarness vulkan;
RunCase(&vulkan, VectorCompareF64Edges());
RunCase(&vulkan, VectorCompare64WaveMasks(32));
RunCase(&vulkan, VectorCompare64WaveMasks(64));
RunCase(&vulkan, VectorF64CapturedScreenSpaceShadows());
RunCase(&vulkan, VectorFractF64CapturedAndEdges());
RunCase(&vulkan, VectorMinMaxF64CapturedAndEdges());
RunCase(&vulkan, VectorRoundF64EdgesAndModifiers());
RunCase(&vulkan, VectorFractF64CapturedChainRuntimeExec());
RunCase(&vulkan, VectorF64ModesModifiersAndExec());
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--cmp-ngt-f16-only") == 0) {
VulkanHarness vulkan;
RunCase(&vulkan, VectorVopcCmpNgtF16CapturedSdwaAndEdges());
RunCase(&vulkan, VectorVopcCmpxNgtF16CapturedSdwaExecMask());
RunCase(&vulkan, VectorVopcCmpxNleF16CapturedSdwaExecMask());
RunCase(&vulkan, VectorCompareF16Ops());
RunCase(&vulkan, Wave32VccMasksPreserveOtherHalf());
RunCase(&vulkan, VectorVop3FloatCompareNegSourceModifier());
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--cmp-nlt-f16-only") == 0) {
VulkanHarness vulkan;
RunCase(&vulkan, VectorVopcCmpNltF16CapturedSdwaAndEdges());
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--fract-f16-only") == 0) {
VulkanHarness vulkan;
RunCase(&vulkan, VectorFractF16CapturedAndEdges());
RunCase(&vulkan, VectorFractF16Modifiers());
RunCase(&vulkan, NativeAndSdwa16BitDestinationWrites());
RunCase(&vulkan, VectorSpecialF16Ops());
RunCase(&vulkan, VectorCosF16CapturedSdwaAndEdges());
RunCase(&vulkan, VectorSinF16SdwaAndEdges());
RunCase(&vulkan, VectorFloatConversionOps());
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--scaled-texture-only") == 0) {
VulkanHarness vulkan;
RunCase(&vulkan, ImageSamplePacked8<false>());
RunCase(&vulkan, ImageSamplePacked8<true>());
RunCase(&vulkan, ImageSamplePacked8<false, true>());
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--packed-texture-only") == 0) {
VulkanHarness vulkan;
RunCase(&vulkan, ImageLoadSintUsesSignedSampledImage<false>());
RunCase(&vulkan, ImageLoadSintUsesSignedSampledImage<true>());
vulkan.CheckPackedTextureComponents();
CheckSigned16ImageStore(vulkan);
vulkan.CheckBgra16Readback();
RunCase(&vulkan, ImageLoadPackedUintUnpacksAndSwizzles());
RunCase(&vulkan, ImageLoadPackedUnorm<false, false>());
RunCase(&vulkan, ImageLoadPackedUnorm<false, true>());
RunCase(&vulkan, ImageLoadPackedUnorm<true, false>());
RunCase(&vulkan, ImageLoadPackedUnorm<true, true>());
RunCase(&vulkan, ImageSamplePackedUintConvertsSampleAndGather());
RunCase(&vulkan, ImageSamplePackedUnormBilinear<false>());
RunCase(&vulkan, ImageSamplePackedUnormBilinear<true>());
RunCase(&vulkan, ImageStorePackedUintSaturatesChannels());
RunCase(&vulkan, ImageStorePackedUintHonorsSparseDmask());
RunCase(&vulkan, ImageStorePackedUnorm<false>());
RunCase(&vulkan, ImageStorePackedUnorm<true>());
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--system-dialog-only") == 0) {
CheckErrorDialogLifecycle();
CheckSigninDialogLifecycle();
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--fmask-only") == 0) {
VulkanHarness vulkan;
RunCase(&vulkan, ImageLoadFmaskUsesNativeSampleMapping());
RunCase(&vulkan, ImageLoadR32UintUsesIntegerSampledImage());
return 0;
}
#if KYTY_PLATFORM != KYTY_PLATFORM_WINDOWS
if (argc == 2 && std::strcmp(argv[1], "--shader-fatal-only") == 0) {
CheckShaderRecompilerFatalContracts();
return 0;
}
#endif
if (argc == 2 && std::strcmp(argv[1], "--ray-tracing-only") == 0) {
VulkanHarness vulkan;
RunCase(&vulkan, FlatStackApertures(32));
RunCase(&vulkan, FlatStackApertures(64));
RunCase(&vulkan, BvhIntersections(true, true, 1));
RunCase(&vulkan, BvhIntersections(true, true, 0, true));
for (bool barycentrics : {false, true}) {
for (bool sorted : {false, true}) {
RunCase(&vulkan, BvhIntersections(barycentrics, sorted));
}
}
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--scalar-sext-only") == 0) {
VulkanHarness vulkan;
for (const u32 bits : {8u, 16u}) {
RunCase(&vulkan, ScalarSextMasks(bits, 32));
RunCase(&vulkan, ScalarSextMasks(bits, 64));
}
RunCase(&vulkan, ScalarSextConstantsAndAliases());
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--flat-d16-only") == 0) {
VulkanHarness vulkan;
for (const auto opcode : {ShaderOpcode::BUFFER_LOAD_SHORT_D16, ShaderOpcode::BUFFER_LOAD_SHORT_D16_HI})
RunCase(&vulkan, BufferLoadD16Captured(opcode));
for (const u32 wave : {32u, 64u})
RunCase(&vulkan, BufferLoadD16Captured(ShaderOpcode::BUFFER_LOAD_UBYTE_D16, wave));
RunCase(&vulkan, FlatStoreSlcCaptured());
RunCase(&vulkan, FlatLoadDlcCaptured(32));
RunCase(&vulkan, FlatLoadDlcCaptured(64));
RunCase(&vulkan, GlobalLoadShortD16Captured(32));
RunCase(&vulkan, GlobalLoadShortD16Captured(64));
RunCase(&vulkan, FlatLoadShortD16AddressSegments());
RunCase(&vulkan, FlatSubdwordLoadsApplyByteOffset());
RunCase(&vulkan, FlatVirtualAddressRebasesGuestAllocation());
RunCase(&vulkan, GlobalSignedImmediateRebasesBeforeSaddr());
RunCase(&vulkan, FlatSegmentIgnoresSaddrAndMasksOffsetMsb());
RunCase(&vulkan, ScratchIsPrivatePerInvocation());
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--indirect-buffer-only") == 0) {
VulkanHarness vulkan;
RunCase(&vulkan, ScalarBufferFromLoopReadlane(32));
RunCase(&vulkan, ScalarBufferFromLoopReadlane(64));
CheckBoundedFormattedBufferCopy(vulkan);
CheckFiniteBufferSelection(vulkan);
CheckIndirectBufferStore(vulkan);
CheckBoundedRecordBufferAccess(vulkan);
CheckGpuSelectedRecordBufferAccess(vulkan);
CheckGuardedWaterfallBufferRecords(vulkan);
CheckGpuSelectedFormattedRecordStore(vulkan);
CheckRuntimeBufferRecords(vulkan);
CheckSharedIndirectBufferLoads(vulkan);
RunCase(&vulkan, BufferLoadFormatXGpuSelectedDescriptors());
RunCase(&vulkan, BufferLoadFormatXGpuSelectedDescriptors64());
RunCase(&vulkan, BufferLoadFormatXRejectsPartialRecord());
RunCase(&vulkan, BufferLoadFormatXyRejectsPartialRecord());
RunCase(&vulkan, BufferLoadDwordx4SnapshotsOverlappingAddress());
RunCase(&vulkan, BufferLoadDwordx4ZeroesOnlyOutOfBoundsTail());
RunCase(&vulkan, BufferLoadDwordIdxenUsesDescriptorStride());
RunCase(&vulkan, BufferStoreFormatXAddTidUsesLaneIndex());
RunCase(&vulkan, FlatVirtualAddressRebasesGuestAllocation());
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--image-sample-offset-only") == 0) {
VulkanHarness vulkan;
CheckDynamicSampleOffsets(vulkan, 32u);
CheckDynamicSampleOffsets(vulkan, 64u);
RunCase(&vulkan, ImageSampleOffsetsReadFourNeighbors<32>());
RunCase(&vulkan, ImageSampleOffsetsReadFourNeighbors<64>());
RunCase(&vulkan, ImageSampleA16OffsetKeepsTexelOffset32BitOnGpu<32>());
RunCase(&vulkan, ImageSampleA16OffsetKeepsTexelOffset32BitOnGpu<64>());
RunCase(&vulkan, ImageSampleAndGather());
RunCase(nullptr, ImageGatherCompareOpcodes());
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--image-gather-lod-only") == 0) {
VulkanHarness vulkan;
vulkan.CheckComparisonDepthTexture();
RunCase(&vulkan, ImageGatherExplicitLod());
RunCase(&vulkan, ImageGatherLodPerLane<0>());
RunCase(&vulkan, ImageGatherLodPerLane<1>());
RunCase(&vulkan, ImageGatherLodPerLane<2>());
RunCase(&vulkan, ImageGatherLodPerLane<3>());
RunCase(&vulkan, ImageGatherLodPerLane<4>());
RunCase(&vulkan, ImageGatherLodPerLane<5>());
RunCase(&vulkan, ImageGatherLodPerLane<6>());
RunCase(&vulkan, ImageGatherLodPerLane<7>());
RunCase(&vulkan, ImageGatherLodPerLane<8>());
RunCase(&vulkan, ImageStoreSingleMipBounds<0>());
RunCase(&vulkan, ImageStoreSingleMipBounds<1>());
RunCase(&vulkan, ImageStoreSingleMipBounds<2>());
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--wave64-only") == 0) {
CheckWave64WholeWaveResults();
VulkanHarness vulkan;
RunCase(&vulkan, Wave32VccMasksPreserveOtherHalf());
RunCase(&vulkan, DsPermuteWave64UsesIndependentHalves());
RunCase(&vulkan, DsBpermuteWave64UsesIndependentHalves());
RunCase(&vulkan, Wave64CrossHalfLaneAndLds());
RunCase(&vulkan, Wave64RawMasksAndScalarBranch());
RunCase(&vulkan, Wave64PartialMultidimensionalWorkgroup());
RunCase(&vulkan, Wave64AppendConsumeHighHalf());
RunCase(&vulkan, VectorAddcWritesPerLaneCarryOut());
RunCase(&vulkan, VectorVop3BCarryOutWritesSgprMask());
RunCase(&vulkan, VectorVop3BCarryOutUsesEncodedSdst());
RunCase(&vulkan, DispatcherIrreducibleControlFlow());
RunCase(&vulkan, ScalarSaveExecOps());
RunCase(&vulkan, ScalarOrn2SaveexecUsesSourceOrNotExec());
RunCase(&vulkan, ScalarOrn2SaveexecB32(32, 32));
RunCase(&vulkan, ScalarOrn2SaveexecB32(64, 64));
RunCase(&vulkan, ScalarOrn2SaveexecB32(32, 4));
RunCase(&vulkan, ScalarSubvectorLoops(32));
RunCase(&vulkan, ScalarSubvectorLoops(64));
RunCase(&vulkan, ScalarNotB64UpdatesScc());
RunCase(&vulkan, ScalarSelectB64PreservesMaskProvenance(32));
RunCase(&vulkan, ScalarSelectB64PreservesMaskProvenance(64));
RunCase(&vulkan, ScalarConditionalMoveB64());
RunCase(&vulkan, ScalarConditionalMoveB64PreservesMasks());
RunCase(&vulkan, ScalarWqmB64SelectsSccDomain());
RunCase(&vulkan, ScalarWqmB64PreservesPartialMasks());
RunCase(&vulkan, ScalarWqmB32Masks(32));
RunCase(&vulkan, ScalarWqmB32Masks(64));
RunCase(&vulkan, ScalarMaskProvenanceOverlapAndMixedBinary());
RunCase(&vulkan, ScalarB32MaskDataflow(32));
RunCase(&vulkan, ScalarB32MaskDataflow(64));
RunCase(&vulkan, ScratchIsPrivatePerInvocation());
RunCase(&vulkan, Vop1MoveRelDestination());
RunCase(&vulkan, BranchVccnzUsesWaveMask());
RunCase(&vulkan, ScalarBranchRestoresInactiveLanes(32, 32));
RunCase(&vulkan, ScalarBranchRestoresInactiveLanes(64, 64));
RunCase(&vulkan, ScalarBranchRestoresInactiveLanes(32, 4));
RunCase(&vulkan, ScalarBranchRestoresInactiveLanes(64, 4));
RunCase(&vulkan, ScalarBranchRestoresInactiveLanes(32, 32, true));
RunCase(&vulkan, ScalarBranchRestoresInactiveLanes(64, 64, true));
RunCase(&vulkan, BranchVccnzUsesCarryProducedWaveMask());
RunCase(&vulkan, ImageSampleAndGather());
RunCase(&vulkan, ImageGatherExplicitLod());
RunCase(&vulkan, SharedReturnKeepsSelectedValues());
RunCase(&vulkan, SiblingSharedExitKeepsCapturedConditions());
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--front-face-only") == 0) {
CheckPixelFrontFaceModes();
VulkanHarness vulkan;
for (bool all_bits : {false, true}) {
for (bool reversed : {false, true}) {
RunGraphicsCase(&vulkan, GraphicsFrontFaceBits(reversed, all_bits));
}
}
RunGraphicsCase(&vulkan, GraphicsAncillaryLayer(true));
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--position-w-only") == 0) {
VulkanHarness vulkan;
RunGraphicsCase(&vulkan, GraphicsPositionWExport());
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--object-primitive-id-only") == 0) {
VulkanHarness vulkan;
RunGraphicsCase(&vulkan, GraphicsObjectPrimitiveId(false));
RunGraphicsCase(&vulkan, GraphicsObjectPrimitiveId(true));
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--centroid-only") == 0) {
VulkanHarness vulkan;
RunGraphicsCase(&vulkan, GraphicsPackedHalfCentroid());
RunGraphicsCase(&vulkan, GraphicsPerspectiveSample(false));
RunGraphicsCase(&vulkan, GraphicsPerspectiveSample(true));
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--pixel-alias-only") == 0) {
CheckPixelParameterAliases();
CheckRectListShaders();
VulkanHarness vulkan;
RunGraphicsCase(&vulkan, GraphicsPackedHalfCentroid());
RunGraphicsCase(&vulkan, GraphicsPackedHalfInputAlias(false));
RunGraphicsCase(&vulkan, GraphicsPackedHalfInputAlias(true));
RunGraphicsCase(&vulkan, GraphicsSmoothRawInputAlias());
RunGraphicsCase(&vulkan, GraphicsSmoothFlatInputAlias(false));
RunGraphicsCase(&vulkan, GraphicsSmoothFlatInputAlias(true));
RunGraphicsCase(&vulkan, GraphicsSmoothFlatInputAlias(false, true));
RunGraphicsCase(&vulkan, GraphicsSmoothFlatInputAlias(true, true));
RunGraphicsCase(&vulkan, GraphicsFlatInterpolatorExport());
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--clip-control-only") == 0) {
CheckClipControlDepthClipState();
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--reference-clock-only") == 0) {
CheckReferenceClockScale();
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--scheduler-only") == 0) {
VulkanHarness vulkan;
vulkan.CheckSchedulerTimeline();
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--host-image-allocation-only") == 0) {
VulkanHarness vulkan;
vulkan.CheckHostImageAllocation();
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--occlusion-dump-only") == 0) {
VulkanHarness vulkan;
CheckPm4SyntheticOcclusionCounterDump(vulkan.RuntimeRenderer());
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--predication-only") == 0) {
VulkanHarness vulkan;
CheckPm4Predication(vulkan.RuntimeRenderer());
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--descriptor-heap-only") == 0) {
VulkanHarness vulkan;
vulkan.CheckDescriptorHeapLargeSet();
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--push-constant-bank-only") == 0) {
VulkanHarness vulkan;
vulkan.CheckGraphicsPushConstantBank();
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--shader-data-storage-only") == 0) {
CheckResourcePlanHandoff();
VulkanHarness vulkan;
RunCase(&vulkan, BufferOffsetsUsePackedWords(false));
RunCase(&vulkan, BufferOffsetsUsePackedWords(true));
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--buffer-publication-only") == 0) {
VulkanHarness vulkan;
RunCase(&vulkan, BufferWorkgroupPublication(32));
RunCase(&vulkan, BufferWorkgroupPublication(64));
RunCase(&vulkan, BufferWorkgroupPublication(32, true));
RunCase(&vulkan, BufferWorkgroupPublication(32, false, true));
RunCase(&vulkan, BufferWorkgroupPublication(64, false, true));
RunCase(&vulkan, BufferAtomicVariants());
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--buffer-cmpswap-only") == 0) {
VulkanHarness vulkan;
RunCase(&vulkan, BufferAtomicCmpSwapExactRaw());
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--image-cmpswap-only") == 0) {
VulkanHarness vulkan;
RunCase(&vulkan, ImageAtomicCompareSwapGlcAndExec());
RunCase(&vulkan, ImageAtomicSwapReturnsPreviousTexel());
RunCase(&vulkan, ImageAtomicVariants());
RunCase(&vulkan, ImageAtomicVariants<true>());
RunCase(&vulkan, ImageAtomicSignedMinMax<false, true>());
RunCase(&vulkan, ImageAtomicSignedMinMax<true, true>());
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--image-atomic64-only") == 0) {
VulkanHarness vulkan;
for (const auto &atomic : ImageAtomicIntegerCases) {
RunCase(&vulkan, ImageAtomicIntegerGlcAndExec(atomic, false));
RunCase(&vulkan, ImageAtomicIntegerGlcAndExec(atomic, true));
}
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--ds-ordered-count-only") == 0) {
VulkanHarness vulkan;
for (u32 wave_size : {32, 64}) {
RunCase(&vulkan, DsOrderedCountAddressAndExec(wave_size, false));
RunCase(&vulkan, DsOrderedCountAddressAndExec(wave_size, true));
}
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--ds-atomics-only") == 0) {
VulkanHarness vulkan;
for (const auto &test : MakeCases()) {
if (std::string_view(test.name).starts_with("Ds") ||
std::string_view(test.name).starts_with("BufferAtomic")) {
RunCase(&vulkan, test);
}
}
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--mapped-range-only") == 0) {
VulkanHarness vulkan;
vulkan.CheckGpuMappedRangeLifecycle();
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--stream-buffer-only") == 0) {
VulkanHarness vulkan;
vulkan.CheckStreamBufferRing();
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--image-layout-only") == 0) {
CheckStorageTextureLinearUploadLayout();
CheckStorageTextureDepthTileUploadLayout();
CheckStorageTextureVolumeUploadLayout();
CheckStorageTextureVolumeMipRegions();
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--gpu-tiler-only") == 0) {
VulkanHarness vulkan;
vulkan.CheckGpuTilerCpuParity();
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--gpu-command-lane-only") == 0) {
VulkanHarness vulkan;
vulkan.CheckGpuCommandLane();
vulkan.CheckScalarLeafCall();
CheckAgcSystemTable(vulkan.RuntimeRenderer());
CheckPm4IndirectControlFlow(vulkan.RuntimeRenderer());
CheckPm4WaitResume(vulkan.RuntimeRenderer());
CheckPm4RewindResume(vulkan.RuntimeRenderer());
CheckPm4CeCompletion(vulkan.RuntimeRenderer());
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--suspend-point-only") == 0) {
VulkanHarness vulkan;
vulkan.CheckGpuSuspendPoint();
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--native64-only") == 0) {
VulkanHarness vulkan;
RunCase(&vulkan, ScalarDynamic64BitOps());
RunCase(&vulkan, VectorDynamicU64ShiftEdges());
RunCase(&vulkan, ScalarAshrI64Edges(false));
RunCase(&vulkan, ScalarAshrI64Edges(true));
RunCase(&vulkan, VectorCompareInteger64Edges());
RunCase(&vulkan, VectorMadU64U32UnsignedCarryOut());
RunCase(&vulkan, ScalarWqmB64SelectsSccDomain());
RunCase(&vulkan, ScalarWqmB64PreservesPartialMasks());
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--alignbyte-only") == 0) {
VulkanHarness vulkan;
RunCase(&vulkan, VectorAlignByteUsesTwoBitByteOffset());
RunCase(&vulkan, VectorVop3IntegerOps());
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--mad-f32-only") == 0) {
VulkanHarness vulkan;
RunCase(&vulkan, VectorMadF32RoundsProduct());
RunCase(&vulkan, VectorMadF32FlushesDenorms());
RunCase(&vulkan, Vop3FmacF32NegatedSourceAccumulates());
RunCase(&vulkan, VectorFloatArithmeticOps());
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--zero-shift-only") == 0) {
VulkanHarness vulkan;
RunCase(&vulkan, ScalarZeroShiftWithRuntimeCount());
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--sdwa-bitwise-only") == 0) {
VulkanHarness vulkan;
RunCase(&vulkan, Vop2SdwaBitwiseByteDestinations(32));
RunCase(&vulkan, Vop2SdwaBitwiseByteDestinations(64));
RunCase(&vulkan, Vop2SdwaLshlrevByteDestinations(32));
RunCase(&vulkan, Vop2SdwaLshlrevByteDestinations(64));
RunCase(&vulkan, Vop1SdwaNotPartialSourcesAndDestinations());
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--sdwa-mov-only") == 0) {
VulkanHarness vulkan;
RunCase(&vulkan, Vop1SdwaMovByteDestinations());
RunCase(&vulkan, Vop1SdwaMovByteSources(32));
RunCase(&vulkan, Vop1SdwaMovByteSources(64));
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--sdwa-ffbh-only") == 0) {
VulkanHarness vulkan;
RunCase(&vulkan, Vop1SdwaBfrevSourceSelectors());
RunCase(&vulkan, Vop1SdwaFfbhCapturedScalarLowWordSource());
RunCase(&vulkan, VectorFfbhI32NativeAndVop3OnGpu());
RunCase(&vulkan, Vop1SdwaFfblCapturedHighWordSource());
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--cubeid-omod-only") == 0) {
VulkanHarness vulkan;
RunCase(&vulkan, CubeIdCapturedNegationAndOutputScale());
RunCase(&vulkan, VectorFloatSpecialOps());
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--zero-stride-oob-only") == 0) {
VulkanHarness vulkan;
RunCase(&vulkan, TBufferCapturedZeroStrideOob());
RunCase(&vulkan, TBufferCapturedZeroStrideOob(true));
RunCase(&vulkan, TBufferCapturedZeroStrideOob(false, true));
RunCase(&vulkan, BufferZeroStrideOobFormatsAndWidths());
RunCase(&vulkan, TBufferLoadFormatXIdxenUsesDescriptorStride());
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--sdwa-ashr-only") == 0) {
VulkanHarness vulkan;
RunCase(&vulkan, Vop2SdwaAshrrevCapturedWord0SignExtends());
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--sdwa-max-i32-only") == 0) {
VulkanHarness vulkan;
RunCase(&vulkan, Vop2SdwaMaxI32CapturedHighWord(32));
RunCase(&vulkan, Vop2SdwaMaxI32CapturedHighWord(64));
RunCase(&vulkan, VectorIntegerOps());
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--sdwa-addc-only") == 0) {
VulkanHarness vulkan;
RunCase(&vulkan, VectorAddcWritesPerLaneCarryOut());
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--sdwa-mul-i24-only") == 0) {
VulkanHarness vulkan;
RunCase(&vulkan, Vop2SdwaMulI24Destinations(32));
RunCase(&vulkan, Vop2SdwaMulI24Destinations(64));
RunCase(&vulkan, VectorIntegerOps());
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--sdwa-mul-u24-only") == 0) {
VulkanHarness vulkan;
RunCase(&vulkan, Vop2SdwaMulU24Destinations(32));
RunCase(&vulkan, Vop2SdwaMulU24Destinations(64));
RunCase(&vulkan, Vop2SdwaMulI24Destinations(32));
RunCase(&vulkan, Vop2SdwaMulI24Destinations(64));
RunCase(&vulkan, VectorIntegerOps());
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--cvt-pk-i16-only") == 0) {
VulkanHarness vulkan;
RunCase(&vulkan, Vop3CvtPkI16I32Captured());
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--med3-i16-only") == 0) {
VulkanHarness vulkan;
RunCase(&vulkan, Vop3Min3U16CapturedAndSelectors());
RunCase(&vulkan, Vop3Med3I16Captured());
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--mul-lo-u16-only") == 0) {
VulkanHarness vulkan;
RunCase(&vulkan, Vop3MulLoU16CapturedAndSelectors());
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--clamp-nan-only") == 0) {
VulkanHarness vulkan;
RunCase(&vulkan, Vop3F32ClampFlushesNaN());
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--mad-i16-only") == 0) {
VulkanHarness vulkan;
RunCase(&vulkan, Vop3MadI16CapturedSelectorsAndSaturation());
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--waitcnt-depctr-only") == 0) {
VulkanHarness vulkan;
RunCase(&vulkan, ScalarWaitcntDepctrCapturedVmVsrc());
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--sff1-b64-only") == 0) {
VulkanHarness vulkan;
RunCase(&vulkan, ScalarBitfieldPack());
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--context-state-only") == 0) {
VulkanHarness vulkan;
CheckShaderFusion();
CheckPm4NativeTargetGeometryRegisters(vulkan.RuntimeRenderer());
CheckPm4IndirectShaderRegisters(vulkan.RuntimeRenderer());
CheckPm4IndirectUconfigRegisters(vulkan.RuntimeRenderer());
CheckPm4BlendColorRegisterRanges(vulkan.RuntimeRenderer());
CheckPm4PolygonOffsetRegisters(vulkan.RuntimeRenderer());
CheckPm4DepthControlHighBits(vulkan.RuntimeRenderer());
CheckPm4DepthRenderOverride(vulkan.RuntimeRenderer());
CheckPm4ContextStateOperations(vulkan.RuntimeRenderer());
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--pm4-wait-only") == 0) {
VulkanHarness vulkan;
CheckPm4AcquireMemPackets(vulkan.RuntimeRenderer());
CheckPm4WaitPackets(vulkan.RuntimeRenderer());
CheckPm4WaitResume(vulkan.RuntimeRenderer());
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--pm4-draw-multi-only") == 0) {
VulkanHarness vulkan;
CheckPm4DrawIndirectMultiPacket(vulkan.RuntimeRenderer());
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--rewind-only") == 0) {
VulkanHarness vulkan;
CheckPm4RewindResume(vulkan.RuntimeRenderer());
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--image-upload-only") == 0) {
VulkanHarness vulkan;
vulkan.CheckLargeSparseImageUpload();
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--image-overlap-only") == 0) {
CheckDepthAttachmentWrites();
CheckDepthFeedbackAspects();
CheckDynamicRenderingState();
VulkanHarness vulkan;
vulkan.CheckRasterization(false);
vulkan.CheckRenderExecutorColorDiscovery();
vulkan.CheckRenderExecutorColor1DArrayDiscovery();
vulkan.CheckRenderExecutorColorVolumeDiscovery();
vulkan.CheckRenderExecutorColorMetadataClear();
vulkan.CheckSampledDccClear();
vulkan.CheckRenderExecutorColorDepthTileDiscovery();
vulkan.CheckRenderExecutorStencilBindingDiscovery();
vulkan.CheckUnifiedTextureCacheFlow();
vulkan.CheckBgra16Readback();
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--draw-offset-only") == 0) {
CheckEmbeddedFetchVertexOffset();
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--depth-feedback-only") == 0) {
CheckDepthAttachmentWrites();
CheckDepthFeedbackAspects();
CheckDynamicRenderingState();
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--polygon-mode-only") == 0) {
VulkanHarness vulkan;
vulkan.CheckRasterization(false);
vulkan.CheckRasterization(false, Prospero::BufferFormat::k11_11_10Float);
vulkan.CheckRasterization(false, Prospero::BufferFormat::k10_10_10_2UScaled);
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--htile-clear-only") == 0) {
VulkanHarness vulkan;
vulkan.CheckUnifiedTextureCacheFlow();
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--dcc-clear-only") == 0) {
VulkanHarness vulkan;
vulkan.CheckRenderExecutorColorMetadataClear();
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--compute-meta-clear-only") == 0) {
CheckDynamicRenderingState();
CheckPs5GameExampleImageClearRuntimeShape();
VulkanHarness vulkan;
vulkan.CheckComputeMetaClearClassification();
vulkan.CheckNativeIndirectDispatch();
vulkan.CheckRenderExecutorColorVolumeDiscovery();
vulkan.CheckRenderExecutorColorMetadataClear();
vulkan.CheckSampledDccClear();
vulkan.CheckRenderExecutorStencilBindingDiscovery();
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--native-indirect-only") == 0) {
VulkanHarness vulkan;
vulkan.CheckNativeIndirectDispatch();
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--layered-image-only") == 0) {
VulkanHarness vulkan;
vulkan.CheckUnifiedImageViewCache();
vulkan.CheckCubeFaceStorageExpansion();
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--image-view-cache-only") == 0) {
VulkanHarness vulkan;
vulkan.CheckUnifiedImageViewCache();
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--storage-sampled-only") == 0) {
VulkanHarness vulkan;
vulkan.CheckUnifiedImageViewCache();
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--depth-readback-only") == 0) {
CheckDepthTargetFootprints();
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--bda-page-table-only") == 0) {
VulkanHarness vulkan;
vulkan.CheckBdaPageTableUploads();
vulkan.CheckBdaSyncUploads();
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--buffer-cache-range-only") == 0) {
VulkanHarness vulkan;
vulkan.CheckUnifiedTextureCacheFlow();
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--buffer-cache-gc-only") == 0) {
VulkanHarness vulkan;
vulkan.CheckBufferCacheDirtyGarbageCollection();
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--sampler-border-only") == 0) {
CheckImageSamplerSpecialization();
VulkanHarness vulkan;
vulkan.CheckSamplerBorderColors();
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--sampled-depth-resource-only") == 0) {
VulkanHarness vulkan;
CheckSampledDepthResource();
CheckDepthTextureEncoding();
vulkan.CheckComparisonDepthTexture();
vulkan.CheckRasterization(true);
RunCase(nullptr, ImageSampleA16CompareBiasRdna2AddressOrder());
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--indirect-image-only") == 0) {
CheckImageSamplerSpecialization();
VulkanHarness vulkan;
vulkan.CheckNullTextureDescriptors();
CheckCountedRowImageSelector(vulkan);
CheckPointerImageTable(vulkan);
CheckNestedPointerImageTable(vulkan);
CheckGuardedRecordImageTable(vulkan);
CheckIndirectImageKeySwitch(vulkan);
CheckFiniteFirstLaneImageChoice(vulkan);
CheckFiniteInlineSamplerPhi(vulkan);
RunCase(&vulkan, ImageCubeGradientsPreserveDerivatives());
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--readlane-key-guard-only") == 0) {
VulkanHarness vulkan;
RunCase(&vulkan, VectorReadlaneSelectsTwoKeysWithinWave());
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--rt1d-only") == 0) {
VulkanHarness vulkan;
vulkan.CheckRenderExecutorColorDiscovery();
vulkan.CheckRenderExecutorColor1DArrayDiscovery();
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--image-transition-only") == 0) {
VulkanHarness vulkan;
CheckImageTransitionState(vulkan.RuntimeRenderer());
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--image-rebind-only") == 0) {
VulkanHarness vulkan;
vulkan.CheckRenderExecutorStencilBindingDiscovery();
return 0;
}
#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
if (argc == 2 && std::strcmp(argv[1], "--reverse-rt-death") == 0) {
RunReverseRenderTargetDeathCase();
}
if (argc == 2 && std::strcmp(argv[1], "--reverse-rt-only") == 0) {
CheckRenderTargetFormatContract();
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--standard64-rt-only") == 0) {
CheckStandard64RenderTargetTileRoundTrip();
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--standard-tile-rt-only") == 0) {
VulkanHarness vulkan;
vulkan.CheckRenderExecutorColorStandardTileDiscovery();
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--image-view-only") == 0) {
VulkanHarness vulkan;
CheckSampledColorViews();
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--storage-bgra-only") == 0) {
CheckSampledColorViews();
CheckBasicStorageTextureDescriptor();
VulkanHarness vulkan;
vulkan.CheckUnifiedImageViewCache();
RunCase(&vulkan, ImageStoreBgraUsesInverseSwizzle());
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--storage-mip-host-only") == 0) {
VulkanHarness vulkan;
vulkan.CheckRenderExecutorStencilBindingDiscovery();
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--storage-mip-only") == 0) {
RunCase(nullptr, ImageStoreMipSelectsPpsa01340Descriptor());
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--storage-yzwx-only") == 0) {
CheckSampledColorViews();
CheckBasicStorageTextureDescriptor();
CheckStorageTextureGpuOwnedRebindState();
VulkanHarness vulkan;
RunCase(&vulkan, ImageStoreYzwxUsesInverseSwizzle());
return 0;
}
if (argc == 3 && std::strcmp(argv[1], "--image-view-death") == 0) {
RunImageViewDeathCase(argv[2]);
}
if (argc == 3 &&
std::strcmp(argv[1], "--storage-texture-descriptor-death") == 0) {
RunStorageTextureDescriptorDeathCase(argv[2]);
}
if (argc != 1) {
std::fprintf(stderr, "unknown test selector: %s\n", argv[1]);
return 2;
}
VulkanHarness vulkan;
// Keep compute and host checks running when rasterization features are missing.
const bool rasterization = vulkan.RasterizationSupported();
bool skipped_device_checks = false;
CheckRenderTargetFormatContract();
CheckSampledColorViews();
CheckImageTransitionState(vulkan.RuntimeRenderer());
CheckSampledDepthResource();
CheckDepthTextureEncoding();
vulkan.CheckSamplerBorderColors();
if (rasterization) {
vulkan.CheckComparisonDepthTexture();
vulkan.CheckRasterization(true);
} else {
skipped_device_checks = true;
}
CheckBasicStorageTextureDescriptor();
CheckStorageTextureLinearUploadLayout();
CheckStorageTextureDepthTileUploadLayout();
CheckStandard64RenderTargetTileRoundTrip();
CheckStorageTextureVolumeUploadLayout();
CheckStorageTextureVolumeMipRegions();
CheckStorageTextureGpuOwnedRebindState();
CheckNativeMsaaState();
CheckPs5DepthRegisterDecoding();
CheckDepthHtileStencilCompatibility();
CheckDepthAttachmentWrites();
CheckDepthFeedbackAspects();
CheckDynamicRenderingState();
CheckDepthTargetFootprints();
CheckSlotVectorLifetime();
#else
if (argc != 1) {
std::fprintf(stderr, "unknown test selector: %s\n", argv[1]);
return 2;
}
CheckShaderRecompilerFatalContracts();
CheckDepthFeedbackAspects();
VulkanHarness vulkan;
const bool rasterization = vulkan.RasterizationSupported();
bool skipped_device_checks = false;
#endif
CheckImageSamplerSpecialization();
CheckResourcePlanHandoff();
CheckNativeImageDescriptorTypes();
CheckClipControlDepthClipState();
CheckReferenceClockScale();
CheckErrorDialogLifecycle();
CheckSigninDialogLifecycle();
CheckVulkan13FeatureRequirements();
CheckPm4AcquireMemPackets(vulkan.RuntimeRenderer());
CheckPm4SyntheticOcclusionCounterDump(vulkan.RuntimeRenderer());
CheckPm4Predication(vulkan.RuntimeRenderer());
CheckPm4StencilInfoValueLane(vulkan.RuntimeRenderer());
CheckPm4NativeTargetGeometryRegisters(vulkan.RuntimeRenderer());
CheckPm4IndirectShaderRegisters(vulkan.RuntimeRenderer());
CheckPm4IndirectUconfigRegisters(vulkan.RuntimeRenderer());
CheckPm4DirectShaderRegisterFallback(vulkan.RuntimeRenderer());
CheckPm4GuardBandRegisterRanges(vulkan.RuntimeRenderer());
CheckPm4BlendColorRegisterRanges(vulkan.RuntimeRenderer());
CheckPm4PolygonOffsetRegisters(vulkan.RuntimeRenderer());
CheckPm4DepthControlHighBits(vulkan.RuntimeRenderer());
CheckPm4DepthRenderOverride(vulkan.RuntimeRenderer());
CheckShaderFusion();
CheckAgcSystemTable(vulkan.RuntimeRenderer());
CheckPm4WaitPackets(vulkan.RuntimeRenderer());
CheckPm4DrawIndirectMultiPacket(vulkan.RuntimeRenderer());
CheckPm4ContextStateOperations(vulkan.RuntimeRenderer());
CheckPm4IndirectControlFlow(vulkan.RuntimeRenderer());
CheckPm4WaitResume(vulkan.RuntimeRenderer());
CheckPm4RewindResume(vulkan.RuntimeRenderer());
CheckPm4CeCompletion(vulkan.RuntimeRenderer());
CheckEmbeddedFetchVertexOffset();
CheckEmbeddedFetchLaneSpill();
CheckTessellationPrograms();
CheckPixelParameterAliases();
CheckPixelFrontFaceModes();
CheckRectListShaders();
CheckSigned16ImageStore(vulkan);
CheckBoundedFormattedBufferCopy(vulkan);
CheckIndirectBufferStore(vulkan);
CheckBoundedRecordBufferAccess(vulkan);
CheckGpuSelectedRecordBufferAccess(vulkan);
CheckGuardedWaterfallBufferRecords(vulkan);
CheckGpuSelectedFormattedRecordStore(vulkan);
CheckFiniteBufferSelection(vulkan);
CheckRuntimeBufferRecords(vulkan);
CheckSharedIndirectBufferLoads(vulkan);
CheckComputeThreadDimensions(vulkan);
CheckCountedRowImageSelector(vulkan);
CheckPointerImageTable(vulkan);
CheckNestedPointerImageTable(vulkan);
CheckGuardedRecordImageTable(vulkan);
CheckIndirectImageKeySwitch(vulkan);
CheckDynamicSampleOffsets(vulkan, 32u);
CheckDynamicSampleOffsets(vulkan, 64u);
CheckFiniteFirstLaneImageChoice(vulkan);
CheckFiniteInlineSamplerPhi(vulkan);
CheckWave64WholeWaveResults();
CheckPs5GameExampleImageClearRuntimeShape();
vulkan.CheckSchedulerTimeline();
vulkan.CheckHostImageAllocation();
vulkan.CheckDescriptorHeapLargeSet();
vulkan.CheckGpuMappedRangeLifecycle();
vulkan.CheckStreamBufferRing();
vulkan.CheckGpuTilerCpuParity();
vulkan.CheckNativeIndirectDispatch();
CheckComputeLdsLimit(vulkan);
vulkan.CheckNullTextureDescriptors();
vulkan.CheckBdaPageTableUploads();
vulkan.CheckBdaSyncUploads();
vulkan.CheckLargeSparseImageUpload();
vulkan.CheckUnifiedTextureCacheFlow();
vulkan.CheckUnifiedImageViewCache();
vulkan.CheckPackedTextureComponents();
vulkan.CheckCubeFaceStorageExpansion();
if (rasterization) {
vulkan.CheckGraphicsPushConstantBank();
vulkan.CheckRenderExecutorColorDiscovery();
vulkan.CheckRenderExecutorColor1DArrayDiscovery();
vulkan.CheckRenderExecutorColorVolumeDiscovery();
vulkan.CheckRenderExecutorColorMetadataClear();
vulkan.CheckSampledDccClear();
vulkan.CheckRenderExecutorColorStandardTileDiscovery();
vulkan.CheckRenderExecutorColorDepthTileDiscovery();
vulkan.CheckRenderExecutorStencilBindingDiscovery();
vulkan.CheckBgra16Readback();
vulkan.CheckRasterization(false);
vulkan.CheckRasterization(false, Prospero::BufferFormat::k11_11_10Float);
vulkan.CheckRasterization(false, Prospero::BufferFormat::k10_10_10_2UScaled);
vulkan.CheckBufferCacheDirtyGarbageCollection();
} else {
skipped_device_checks = true;
}
const auto tests = MakeCases();
const auto graphics_tests = MakeGraphicsCases();
CheckOpcodeCoverage(tests, graphics_tests);
for (const auto &test : tests) {
RunCase(&vulkan, test);
}
for (const auto &test : graphics_tests) {
RunGraphicsCase(&vulkan, test);
}
vulkan.CheckGpuSuspendPoint();
vulkan.CheckGpuCommandLane();
vulkan.CheckScalarLeafCall();
if (skipped_device_checks) {
std::printf(
"ShaderRecompilerComputeTests: device rasterization checks skipped, this "
"device does not support the production rasterization features\n");
}
if (vulkan.SkippedCaseCount() > 0) {
std::printf("ShaderRecompilerComputeTests: %u graphics cases skipped\n",
vulkan.SkippedCaseCount());
}
std::printf("ShaderRecompilerComputeTests: all cases passed\n");
return 0;
}
Reference in New Issue View Git Blame Copy Permalink