diff --git a/CMakeLists.txt b/CMakeLists.txt index c8f7700eb..5a98c1194 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -657,6 +657,8 @@ if(BUILD_TESTING) COMMAND $ --context-state-only) add_test(NAME gpu_tiler COMMAND $ --gpu-tiler-only) + add_test(NAME image_transfer_layout + COMMAND $ --image-layout-only) add_test(NAME texture_cache_layered_image COMMAND $ --layered-image-only) diff --git a/src/graphics/host_gpu/renderer/cache/textureCache.cpp b/src/graphics/host_gpu/renderer/cache/textureCache.cpp index d4902f99b..77e7f9aac 100644 --- a/src/graphics/host_gpu/renderer/cache/textureCache.cpp +++ b/src/graphics/host_gpu/renderer/cache/textureCache.cpp @@ -110,37 +110,6 @@ constexpr uint64_t NumFramesBeforeRemoval = 32; return true; } -[[nodiscard]] std::vector BuildDepthCopies(const ImageInfo& info, - uint64_t slice_stride, - vk::ImageAspectFlags aspect) { - std::vector copies(info.resources.layers); - for (uint32_t layer = 0; layer < info.resources.layers; ++layer) { - auto& copy = copies[layer]; - copy.bufferOffset = slice_stride * layer; - copy.bufferRowLength = info.pitch; - copy.bufferImageHeight = info.extent.height; - copy.imageSubresource = {aspect, 0, layer, 1}; - copy.imageExtent = {info.extent.width, info.extent.height, 1}; - } - return copies; -} - -[[nodiscard]] std::vector BuildDepthTiles(const ImageInfo& info) { - TileBlockLayout block {}; - EXIT_NOT_IMPLEMENTED( - !TileGetBlockLayout(TileBlockFamily::Depth64KB, info.bytes_per_block, block)); - const auto full_slice_size = info.data.size / info.resources.layers; - std::vector tiles; - tiles.reserve(info.resources.layers); - for (uint32_t layer = 0; layer < info.resources.layers; ++layer) { - const auto offset = full_slice_size * layer; - tiles.push_back({block.family, block.bytes_per_element, offset, full_slice_size, offset, - full_slice_size, 0, info.extent.width, info.extent.height, 1, info.pitch}); - tiles.back().surface_z = layer; - } - return tiles; -} - } // namespace TextureCache::TextureCache(GraphicContext& graphics, CommandScheduler& scheduler, @@ -570,20 +539,20 @@ bool TextureCache::CopyD16(Image& destination, Image& source) { EXIT_IF(layers == 0 || depth_slice > UINT64_MAX / layers || color_slice > UINT64_MAX / layers); const auto depth_size = depth_slice * layers; const auto color_size = color_slice * layers; - std::vector depth_copies(layers); - std::vector color_copies(layers); - for (uint32_t layer = 0; layer < layers; layer++) { - depth_copies[layer].bufferOffset = depth_slice * layer; - depth_copies[layer].bufferRowLength = depth.info.pitch; - depth_copies[layer].bufferImageHeight = depth.info.extent.height; - depth_copies[layer].imageSubresource = {vk::ImageAspectFlagBits::eDepth, 0, layer, 1}; - depth_copies[layer].imageExtent = depth.info.extent; - color_copies[layer].bufferOffset = color_slice * layer; - color_copies[layer].bufferRowLength = color.info.pitch; - color_copies[layer].bufferImageHeight = color.info.extent.height; - color_copies[layer].imageSubresource = {vk::ImageAspectFlagBits::eColor, 0, layer, 1}; - color_copies[layer].imageExtent = color.info.extent; - } + const std::array depth_copies { + vk::BufferImageCopy {0, + depth.info.pitch, + depth.info.extent.height, + {vk::ImageAspectFlagBits::eDepth, 0, 0, layers}, + {}, + depth.info.extent}}; + const std::array color_copies { + vk::BufferImageCopy {0, + color.info.pitch, + color.info.extent.height, + {vk::ImageAspectFlagBits::eColor, 0, 0, layers}, + {}, + color.info.extent}}; auto depth_buffer = m_tiler.GetScratchBuffer(depth_size); auto color_buffer = m_tiler.GetScratchBuffer(color_size, depth_buffer.buffer); @@ -740,7 +709,11 @@ ImageId TextureCache::ResolveDepthOverlap(const ImageInfo& requested, BindingTyp if (retain_cached_layout) { info.data = cached.info.data; info.resources = cached.info.resources; - info.mip_layout = cached.info.mip_layout; + info.mip_layout = cached.info.mip_layout; + info.tiling = cached.info.tiling; + info.tiled_slice_stride = cached.info.tiled_slice_stride; + info.linear_slice_stride = cached.info.linear_slice_stride; + info.first_tail_level = cached.info.first_tail_level; } info.htile_clear_mask = 0; const auto replacement_id = InsertImage(info); @@ -801,7 +774,14 @@ TextureCache::OverlapResult TextureCache::ResolveOverlap(const ImageInfo& reques // a level into or out of the mip tail. These are separate guest layouts. if (requested.tile_mode != cached.info.tile_mode || (requested.resources == cached.info.resources && - requested.mip_layout != cached.info.mip_layout)) { + !std::equal(requested.mip_layout.begin(), requested.mip_layout.end(), + cached.info.mip_layout.begin(), + [](const ImageMipInfo& left, const ImageMipInfo& right) { + return left.offset == right.offset && left.size == right.size && + left.pitch == right.pitch && left.height == right.height && + left.tail_x == right.tail_x && left.tail_y == right.tail_y && + left.surface_z == right.surface_z; + }))) { if (safe_to_delete) { FreeImage(cached_id); } @@ -913,168 +893,55 @@ ImageId TextureCache::ExpandImage(const ImageInfo& info, ImageId source_id) { return expanded_id; } -struct TextureCache::TextureTransfer { - TextureUploadLayout layout; - std::vector regions; - std::vector tiles; - ColorTransform color_transform = ColorTransform::None; - bool valid = false; - - [[nodiscard]] uint64_t LinearSize() const { - uint64_t size = 0; - for (const auto& tile: tiles) { - size = std::max(size, tile.linear_offset + tile.linear_size); - } - return size; +bool TextureCache::CanDownload(const Image& image) const { + const auto& info = image.info; + if (info.samples != 1 || image.backing.samples != 1 || info.resources.layers == 0 || + info.data.size % info.resources.layers != 0) { + return false; } -}; - -struct TextureCache::ImageDownload { - TextureTransfer texture; - bool depth_target = false; - bool valid = false; -}; - -TextureCache::TextureTransfer -TextureCache::BuildTextureTransfer(const Image& image, BindingType binding, - TransferDirection direction) const { - const auto& info = image.info; - const bool upload = direction == TransferDirection::Upload; - const bool render_target = binding == BindingType::RenderTarget; - const bool video_out = binding == BindingType::VideoOut; - uint32_t layers = info.TransferLayers(); - bool volume = info.IsVolume(); - const char* owner = "TextureCache readback"; - - TextureTransfer transfer; - transfer.color_transform = info.GetColorTransform(); - if (upload) { - if ((render_target || video_out) && - (info.resources.layers == 0 || info.data.size % info.resources.layers != 0 || - info.samples != 1 || image.backing.samples != 1)) { - EXIT("TextureCache: invalid color-attachment upload\n"); - } - owner = "TextureCache"; - if (render_target) { - owner = "RenderTarget"; - } else if (binding == BindingType::Storage) { - owner = "StorageTextureCache"; - } else if (video_out) { - if (info.metadata.compression != VideoOutCompression::Uncompressed) { - EXIT("TextureCache: invalid color-attachment upload\n"); - } - layers = info.resources.layers; - volume = false; - owner = "VideoOut"; - } + if (UploadBinding(image) == BindingType::DepthTarget) { + return IsSupportedDepthPlaneReadback(info) && + Prospero::NumBytesPerElement(info.guest_format) == info.bytes_per_block; } - - transfer.layout = TextureCalcUploadLayout(info.guest_format, info.extent.width, - info.extent.height, info.resources.levels, layers, - info.tile_mode, info.data.size, volume, owner); - transfer.regions = TextureBuildImageCopies(transfer.layout); - if (info.IsDepth()) { - for (auto& region: transfer.regions) { - region.imageSubresource.aspectMask = vk::ImageAspectFlagBits::eDepth; - } - } - if (transfer.layout.surface.description.tile_mode != Prospero::TileMode::kLinear) { - if (!TextureBuildGpuTileInfos(info.data.size, transfer.regions, transfer.layout, - info.resources.levels, transfer.tiles)) { - return transfer; - } - } - transfer.valid = true; - return transfer; -} - -TextureCache::ImageDownload TextureCache::BuildDownload(const Image& image) const { - const auto& info = image.info; - const auto binding = UploadBinding(image); - ImageDownload transfer {.depth_target = binding == BindingType::DepthTarget}; - if (info.samples != 1 || image.backing.samples != 1) { - return transfer; - } - if (transfer.depth_target) { - transfer.valid = IsSupportedDepthPlaneReadback(info) && info.resources.layers != 0 && - info.data.size % info.resources.layers == 0 && - Prospero::NumBytesPerElement(info.guest_format) == info.bytes_per_block; - return transfer; - } - if (info.metadata.compression != VideoOutCompression::Uncompressed) { - return transfer; - } - transfer.texture = BuildTextureTransfer(image, binding, TransferDirection::Download); - transfer.valid = transfer.texture.valid; - return transfer; + return info.metadata.compression == VideoOutCompression::Uncompressed && + (!info.IsTiled() || !Prospero::IsFmaskTextureFormat(info.guest_format)) && + info.LinearSize() != 0; } void TextureCache::UploadImage(Image& image, Buffer& source, uint64_t source_offset) { auto& destination = image.depth_id ? m_slot_images[image.depth_id] : image; - const auto binding = image.depth_id ? BindingType::DepthTarget : UploadBinding(image); - const auto upload = [&](std::vector& copies, TileManager::Result linear) { - for (auto& copy: copies) { - copy.bufferOffset += linear.offset; - } - destination.Upload(copies, linear.buffer, linear.offset, linear.size); - }; - + const auto& info = image.info; + const auto binding = image.depth_id ? BindingType::DepthTarget : UploadBinding(image); if (binding != BindingType::DepthTarget) { - const auto& info = image.info; - auto transfer = BuildTextureTransfer(image, binding, TransferDirection::Upload); - if (!transfer.valid) { - EXIT("TextureCache: invalid texture upload: binding=%u addr=0x%016" PRIx64 - " size=0x%016" PRIx64 " format=%u tile=%u family=%u extent=%ux%ux%u " - "pitch=%u levels=%u layers=%u samples=%u\n", - static_cast(binding), info.data.address, info.data.size, - static_cast(info.guest_format), static_cast(info.tile_mode), - static_cast(transfer.layout.surface.texture.block.family), info.extent.width, - info.extent.height, info.extent.depth, info.pitch, info.resources.levels, - info.resources.layers, info.samples); + if ((binding == BindingType::RenderTarget || binding == BindingType::VideoOut) && + (info.resources.layers == 0 || info.data.size % info.resources.layers != 0 || + info.samples != 1 || image.backing.samples != 1)) { + EXIT("TextureCache: invalid color-attachment upload\n"); } - TileManager::Result linear {source.Handle(), source_offset, info.data.size}; - if (!transfer.tiles.empty()) { - linear = m_tiler.Detile(source.Handle(), source_offset, info.data.size, - transfer.LinearSize(), transfer.tiles, transfer.color_transform); - } else if (transfer.color_transform != ColorTransform::None) { - linear = m_tiler.TransformColor(linear, transfer.color_transform, true); + if (binding == BindingType::VideoOut && + info.metadata.compression != VideoOutCompression::Uncompressed) { + EXIT("TextureCache: invalid video-out upload\n"); } - upload(transfer.regions, linear); + const auto linear = m_tiler.Detile(source.Handle(), source_offset, info); + const auto copies = info.BufferCopies(linear.offset); + destination.Upload(copies, linear.buffer, linear.offset, linear.size); return; } - // The stencil plane has its own row pitch and shares the native image - // with the depth plane. - auto info = destination.info; - if (image.depth_id) { - info.data = image.info.data; - info.resources = {image.stencil_subresources.level_count, - image.stencil_subresources.layer_count}; - info.guest_format = Prospero::BufferFormat::k8UInt; - info.bytes_per_block = 1; - if (info.IsTiled()) info.pitch = TileGetDepthPitch(info.extent.width, 1, 0); - } if (info.samples != 1 || destination.backing.samples != 1 || info.resources.layers == 0 || info.data.size % info.resources.layers != 0 || Prospero::NumBytesPerElement(info.guest_format) != info.bytes_per_block) { EXIT("TextureCache: invalid depth upload\n"); } - const auto layers = info.resources.layers; - const auto full_slice_size = info.data.size / layers; - auto copies = BuildDepthCopies(info, full_slice_size, image.depth_id - ? vk::ImageAspectFlagBits::eStencil - : vk::ImageAspectFlagBits::eDepth); + const auto layers = info.resources.layers; + const auto full_slice_size = info.linear_slice_stride; + auto linear = m_tiler.Detile(source.Handle(), source_offset, info); + auto copies = info.BufferCopies(linear.offset); if (image.depth_id) { for (auto& copy: copies) { copy.imageSubresource.baseArrayLayer += image.stencil_subresources.base_layer; } } - TileManager::Result linear {source.Handle(), source_offset, source.Size() - source_offset}; - if (info.IsTiled()) { - const auto tiles = BuildDepthTiles(info); - linear = - m_tiler.Detile(source.Handle(), source_offset, info.data.size, info.data.size, tiles); - } const auto transfer_bytes = image.depth_id ? 1u : DepthAspectTransferBytes(info.pixel_format); if (transfer_bytes != info.bytes_per_block) { const uint64_t texels_per_slice = static_cast(info.pitch) * info.extent.height; @@ -1095,11 +962,14 @@ void TextureCache::UploadImage(Image& image, Buffer& source, uint64_t source_off .source_slice_stride = full_slice_size, .target_slice_stride = transfer_slice}); linear = promoted; - for (uint32_t layer = 0; layer < layers; layer++) { - copies[layer].bufferOffset = transfer_slice * layer; - } + copies.resize(1); + copies[0].bufferOffset = linear.offset; + copies[0].bufferRowLength = info.pitch; + copies[0].bufferImageHeight = info.extent.height; + copies[0].imageSubresource = {vk::ImageAspectFlagBits::eDepth, 0, 0, layers}; + copies[0].imageExtent = info.extent; } - upload(copies, linear); + destination.Upload(copies, linear.buffer, linear.offset, linear.size); } void TextureCache::InitializeImage(ImageId id) { @@ -1257,8 +1127,25 @@ ImageId TextureCache::AssociateStencil(ImageId depth_id, GuestRange stencil) { } auto& record = m_slot_images[association]; TouchImage(record); - record.depth_id = depth_id; - record.stencil_subresources = depth.stencil_subresources; + if (record.depth_id != depth_id || record.stencil_subresources != depth.stencil_subresources) { + ImageInfo plane {}; + plane.data = stencil; + plane.pixel_format = vk::Format::eS8Uint; + plane.guest_format = Prospero::BufferFormat::k8UInt; + plane.type = depth.info.type; + plane.extent = depth.info.extent; + plane.resources = {depth.stencil_subresources.level_count, + depth.stencil_subresources.layer_count}; + plane.tile_mode = depth.info.tile_mode; + plane.pitch = + plane.IsTiled() ? TileGetDepthPitch(plane.extent.width, 1, 0) : depth.info.pitch; + plane.bytes_per_block = 1; + plane.UpdateSize(); + record.info = plane; + record.depth_id = depth_id; + record.stencil_subresources = depth.stencil_subresources; + } + return association; } @@ -1282,6 +1169,19 @@ ImageId TextureCache::FindImage(ImageDesc& desc, bool exact_format) { for (const auto id: candidates) { const auto& image = m_slot_images[id]; + // Stencil descriptors name the associated plane; its depth owner supplies + // the native image and the canonical guest tiling. + if ((desc.type == BindingType::Texture || desc.type == BindingType::Storage) && + image.depth_id && image.info.data == desc.info.data && + m_slot_images[image.depth_id].info.samples == desc.info.samples && + image.info.extent == desc.info.extent && image.info.type == desc.info.type && + image.info.resources.levels >= desc.info.resources.levels && + image.info.resources.layers >= desc.info.resources.layers && + desc.info.bytes_per_block == 1 && + desc.info.guest_format == Prospero::BufferFormat::k8UInt) { + result = id; + break; + } if (SameBacking(image.info, desc.info, exact_format)) { result = id; } @@ -1303,7 +1203,7 @@ ImageId TextureCache::FindImage(ImageDesc& desc, bool exact_format) { } } - if (result) { + if (result && !m_slot_images[result].depth_id) { auto& resolved = m_slot_images[result]; if (exact_format && resolved.info.pixel_format != desc.info.pixel_format) { result = {}; @@ -1685,7 +1585,7 @@ void TextureCache::InvalidateMemory(uint64_t address, uint64_t size) { void TextureCache::DownloadDepth(Image& image, Buffer& destination, uint64_t destination_offset) { const auto& info = image.info; const auto layers = info.resources.layers; - const auto full_slice_size = info.data.size / layers; + const auto full_slice_size = info.linear_slice_stride; const auto transfer_bytes = DepthAspectTransferBytes(info.pixel_format); const uint64_t texels_per_slice = static_cast(info.pitch) * info.extent.height; EXIT_NOT_IMPLEMENTED(transfer_bytes == 0 || texels_per_slice > UINT32_MAX || @@ -1696,31 +1596,31 @@ void TextureCache::DownloadDepth(Image& image, Buffer& destination, uint64_t des EXIT_NOT_IMPLEMENTED(transfer_slice > UINT64_MAX / layers); const uint64_t transfer_size = transfer_slice * layers; EXIT_NOT_IMPLEMENTED(guest_slice > full_slice_size); - auto copies = BuildDepthCopies(info, full_slice_size, vk::ImageAspectFlagBits::eDepth); if (transfer_bytes == info.bytes_per_block) { if (!info.IsTiled()) { - for (auto& copy: copies) { - copy.bufferOffset += destination_offset; - } + const auto copies = info.BufferCopies(destination_offset); image.Download(copies, destination.Handle(), destination_offset, info.data.size); - return; + } else { + const auto copies = info.BufferCopies(); + m_tiler.TileImage(image, copies, destination.Handle(), destination_offset, + info.data.size); } - const auto tiles = BuildDepthTiles(info); - m_tiler.TileImage(image, copies, destination.Handle(), destination_offset, info.data.size, - info.data.size, tiles); return; } EXIT_NOT_IMPLEMENTED(info.bytes_per_block != sizeof(uint16_t) || transfer_bytes != sizeof(uint32_t)); - for (uint32_t layer = 0; layer < layers; layer++) { - copies[layer].bufferOffset = transfer_slice * layer; - } + const std::array copies {vk::BufferImageCopy {0, + info.pitch, + info.extent.height, + {vk::ImageAspectFlagBits::eDepth, 0, 0, layers}, + {}, + info.extent}}; auto host_linear = m_tiler.GetScratchBuffer(transfer_size); image.Download(copies, host_linear.buffer, 0, host_linear.size); const bool tiled = info.IsTiled(); - auto guest_linear = tiled ? m_tiler.GetScratchBuffer(info.data.size, host_linear.buffer) - : TileManager::Result {destination.Handle(), destination_offset, - destination.Size() - destination_offset}; + auto guest_linear = tiled ? m_tiler.GetScratchBuffer(info.LinearSize(), host_linear.buffer) + : TileManager::Result {destination.Handle(), destination_offset, + destination.Size() - destination_offset}; m_tiler.ConvertD16(host_linear, guest_linear, TileManager::D16Direction::Demote, DepthAspectTransferFormat(info.pixel_format) == vk::Format::eD32Sfloat, {.width = info.extent.width, @@ -1730,47 +1630,39 @@ void TextureCache::DownloadDepth(Image& image, Buffer& destination, uint64_t des .target_row_stride = static_cast(info.pitch) * sizeof(uint16_t), .source_slice_stride = transfer_slice, .target_slice_stride = full_slice_size}); - if (!tiled) { - return; + if (tiled) { + m_tiler.Tile(guest_linear.buffer, guest_linear.offset, guest_linear.size, + destination.Handle(), destination_offset, info.data.size, info); } - const auto tiles = BuildDepthTiles(info); - m_tiler.Tile(guest_linear.buffer, guest_linear.offset, info.data.size, destination.Handle(), - destination_offset, info.data.size, tiles); } void TextureCache::DownloadImage(Image& image, Buffer& destination, uint64_t destination_offset, - uint64_t destination_size, ImageDownload transfer) { - if (!transfer.valid) { - EXIT("TextureCache: invalid image download transfer\n"); + uint64_t destination_size, uint32_t levels) { + if (!CanDownload(image)) { + EXIT("TextureCache: invalid image download\n"); } - if (transfer.depth_target) { - if (destination_size != image.info.data.size) { - EXIT("TextureCache: partial depth image download is unsupported\n"); - } + if (UploadBinding(image) == BindingType::DepthTarget) { + EXIT_IF(destination_size != image.info.data.size || levels != image.info.resources.levels); DownloadDepth(image, destination, destination_offset); return; } - auto& texture = transfer.texture; - const auto transform = texture.color_transform; - if (texture.tiles.empty()) { - if (transform != ColorTransform::None) { - auto linear = m_tiler.GetScratchBuffer(destination_size); - image.Download(texture.regions, linear.buffer, 0, linear.size); - m_tiler.TransformColor(linear, - {destination.Handle(), destination_offset, destination_size}, - transform, false); - return; - } - for (auto& copy: texture.regions) { - copy.bufferOffset += destination_offset; - } - image.Download(texture.regions, destination.Handle(), destination_offset, destination_size); - return; + const auto& info = image.info; + const auto transform = info.GetColorTransform(); + if (info.IsTiled()) { + const auto copies = info.BufferCopies(0, levels); + m_tiler.TileImage(image, copies, destination.Handle(), destination_offset, destination_size, + levels); + } else if (transform != ColorTransform::None) { + auto linear = m_tiler.GetScratchBuffer(destination_size); + const auto copies = info.BufferCopies(0, levels); + image.Download(copies, linear.buffer, 0, linear.size); + m_tiler.TransformColor(linear, {destination.Handle(), destination_offset, destination_size}, + transform, false); + } else { + const auto copies = info.BufferCopies(destination_offset, levels); + image.Download(copies, destination.Handle(), destination_offset, destination_size); } - - m_tiler.TileImage(image, texture.regions, destination.Handle(), destination_offset, - destination_size, texture.LinearSize(), texture.tiles, transform); } bool BufferCache::SynchronizeBufferFromImage(Buffer& buffer, uint64_t vaddr, uint64_t size) { @@ -1810,30 +1702,12 @@ bool BufferCache::SynchronizeBufferFromImage(Buffer& buffer, uint64_t vaddr, uin if (copy_size == 0) { return false; } - auto transfer = m_texture_cache.BuildDownload(image); - if (!transfer.valid) { + if (!m_texture_cache.CanDownload(image) || + (TextureCache::UploadBinding(image) == TextureCache::BindingType::DepthTarget && + copy_size != image.info.data.size)) { return false; } - if (transfer.depth_target && copy_size != image.info.data.size) { - return false; - } - if (!transfer.depth_target && levels < image.info.resources.levels) { - auto& texture = transfer.texture; - std::erase_if(texture.regions, [levels](const vk::BufferImageCopy& region) { - return region.imageSubresource.mipLevel >= levels; - }); - if (texture.regions.empty()) { - return false; - } - if (!texture.tiles.empty()) { - texture.tiles.clear(); - if (!TextureBuildGpuTileInfos(copy_size, texture.regions, texture.layout, levels, - texture.tiles)) { - return false; - } - } - } - m_texture_cache.DownloadImage(image, buffer, buf_offset, copy_size, std::move(transfer)); + m_texture_cache.DownloadImage(image, buffer, buf_offset, copy_size, levels); return true; } @@ -1842,8 +1716,7 @@ bool TextureCache::DownloadImageMemory(ImageId id) { if (image.depth_id) { return false; } - auto transfer = BuildDownload(image); - if (!transfer.valid || !image.SafeToDownload()) { + if (!image.SafeToDownload() || !CanDownload(image)) { return false; } const auto range = image.info.data; @@ -1859,7 +1732,7 @@ bool TextureCache::DownloadImageMemory(ImageId id) { } download.Flush(offset, range.size); - DownloadImage(image, download, offset, range.size, std::move(transfer)); + DownloadImage(image, download, offset, range.size, image.info.resources.levels); vk::BufferMemoryBarrier barrier {}; barrier.srcAccessMask = vk::AccessFlagBits::eMemoryWrite | vk::AccessFlagBits::eTransferWrite | vk::AccessFlagBits::eShaderWrite; diff --git a/src/graphics/host_gpu/renderer/cache/textureCache.h b/src/graphics/host_gpu/renderer/cache/textureCache.h index 5be210e58..7f426a0d2 100644 --- a/src/graphics/host_gpu/renderer/cache/textureCache.h +++ b/src/graphics/host_gpu/renderer/cache/textureCache.h @@ -73,10 +73,6 @@ public: void RunGarbageCollector(); private: - enum class TransferDirection { Upload, Download }; - struct TextureTransfer; - struct ImageDownload; - struct MetaDataInfo { enum class Type : uint8_t { CMask, FMask, HTile }; @@ -141,12 +137,10 @@ private: void MaterializeColorClear(ImageId id, const ImageDesc& desc, uint32_t metadata_base_layer); void InitializeImage(ImageId id); - [[nodiscard]] TextureTransfer - BuildTextureTransfer(const Image& image, BindingType binding, TransferDirection direction) const; - [[nodiscard]] ImageDownload BuildDownload(const Image& image) const; + [[nodiscard]] bool CanDownload(const Image& image) const; void UploadImage(Image& image, Buffer& source, uint64_t source_offset); void DownloadImage(Image& image, Buffer& destination, uint64_t destination_offset, - uint64_t destination_size, ImageDownload transfer); + uint64_t destination_size, uint32_t levels); void DownloadDepth(Image& image, Buffer& destination, uint64_t destination_offset); void CommitGpuWrite(Image& image); // Caller holds m_lock. Volume layer ranges select depth slices. diff --git a/src/graphics/host_gpu/renderer/colorRenderTarget.cpp b/src/graphics/host_gpu/renderer/colorRenderTarget.cpp index 8af8b5e16..1202db0e8 100644 --- a/src/graphics/host_gpu/renderer/colorRenderTarget.cpp +++ b/src/graphics/host_gpu/renderer/colorRenderTarget.cpp @@ -139,9 +139,7 @@ void RenderExecutor::ResolveRenderColorTarget(CommandBuffer& buffer, RenderColor // Nonlinear clear values are still stored as normalized components. // Metadata clears are materialized during image discovery; render-pass loads preserve contents. uint32_t width = 0; - uint32_t height = 0; - uint32_t pitch = 0; - uint64_t size = 0; + uint32_t height = 0; bool tile = false; const bool standard4 = rt.attrib3.tile_mode == Prospero::TileMode::kStandard4KB; const bool standard64 = rt.attrib3.tile_mode == Prospero::TileMode::kStandard64KB; @@ -164,6 +162,9 @@ void RenderExecutor::ResolveRenderColorTarget(CommandBuffer& buffer, RenderColor if (texture_tile && samples != 1) { EXIT("texture-tiled color render targets do not support multisampling\n"); } + if (volume && !tile) { + EXIT("linear 3D render targets are unsupported\n"); + } width = rt.attrib2.width + 1; height = rt.attrib2.height + 1; @@ -198,80 +199,25 @@ void RenderExecutor::ResolveRenderColorTarget(CommandBuffer& buffer, RenderColor " layer=%u/%u\n", rt.attrib3.dimension, rt.attrib3.depth, view.base_layer, view.image_layers); } - if (samples == 1) { - pitch = TileGetTexturePitch(transfer_format, width, rt.attrib3.tile_mode); - } else { - pitch = TileGetRenderTargetPitch(width, bytes_per_element, rt.attrib.num_fragments); - } - if (pitch == 0) { - EXIT("unsupported render-target pitch: width=%u bytes=%u\n", width, bytes_per_element); - } - - TileSizeOffset mip_sizes[16] {}; - TilePaddedSize mip_padded[16] {}; - TileSurfaceLayout volume_layout {}; - uint64_t backing_size = 0; - if (volume) { - const TileSurfaceDescription description {transfer_format, - rt.attrib3.tile_mode, - TileSurfaceDimension::Dim3D, - width, - height, - depth, - levels, - 1}; - if (!tile || !TileGetTiledTextureLayout(description, volume_layout)) { - EXIT("unsupported 3D render-target layout: %ux%ux%u levels=%u tile=%u\n", width, height, - depth, levels, static_cast(rt.attrib3.tile_mode)); - } - size = volume_layout.block_slice_size; - backing_size = volume_layout.total_size; - } else { - TileSizeAlign layout {}; - bool valid_layout = false; - if (samples == 1) { - TileGetTextureSize(transfer_format, width, height, levels, rt.attrib3.tile_mode, - &layout, mip_sizes, mip_padded); - valid_layout = layout.size != 0 && layout.align != 0 && - (rt.attrib3.tile_mode != Prospero::TileMode::kRenderTarget || - levels <= std::bit_width(std::max(width, height))); - } else { - valid_layout = TileGetRenderTargetSize(width, height, pitch, bytes_per_element, - layout, rt.attrib.num_fragments); - mip_sizes[0] = {layout.size, 0, 0, 0, 0, 0}; - mip_padded[0] = {pitch, height}; - } - if (!valid_layout) { - EXIT("unsupported render-target layout: %ux%u pitch=%u bytes=%u levels=%u\n", width, - height, pitch, bytes_per_element, levels); - } - size = layout.size; - } - if (size == 0 || (!volume && size > UINT64_MAX / view.image_layers)) { - EXIT("render-target memory footprint is invalid\n"); - } - if (!volume) { - backing_size = size * view.image_layers; - } - if (backing_size == 0) { - EXIT("render-target backing is empty\n"); - } - if (!GuestRange {rt.base.addr, backing_size}.Valid()) { - EXIT("render-target backing range is invalid\n"); - } - auto& desc = r.desc; desc.type = TextureCache::BindingType::RenderTarget; - desc.info.data = {rt.base.addr, backing_size}; + desc.info.data = {rt.base.addr, 0}; desc.info.pixel_format = target_format.format; desc.info.guest_format = target_format.guest_format; desc.info.type = image_type; desc.info.extent = {width, height, depth}; - desc.info.resources = {levels, volume ? 1u : view.image_layers}; - desc.info.pitch = pitch; + desc.info.resources = {levels, volume ? 1u : view.image_layers}; desc.info.bytes_per_block = bytes_per_element; desc.info.samples = samples; desc.info.tile_mode = rt.attrib3.tile_mode; + if (!volume && rt.attrib3.tile_mode == Prospero::TileMode::kRenderTarget && + levels > std::bit_width(std::max(width, height))) { + EXIT("unsupported render-target mip count\n"); + } + desc.info.UpdateSize(); + if (!desc.info.data.Valid()) { + EXIT("render-target backing range is invalid\n"); + } const bool has_dcc = rt.info.dcc_compression_enable && rt.dcc_addr.addr != 0; const bool has_cmask = !rt.info.dcc_compression_enable && rt.info.cmask_fast_clear_enable && rt.cmask.addr != 0 && samples == 1 && @@ -293,30 +239,6 @@ void RenderExecutor::ResolveRenderColorTarget(CommandBuffer& buffer, RenderColor desc.info.metadata.clear_word = rt.clear_word0.word0; desc.info.metadata.clear_register_valid = true; } - for (uint32_t level = 0; level < levels; level++) { - if (volume) { - const auto& mip = volume_layout.mips[level]; - desc.info.mip_layout[level] = { - mip.offset, - mip.size, - mip.padded_width, - mip.padded_height, - }; - continue; - } - const auto level_offset = - mip_sizes[level].src_size != 0 ? mip_sizes[level].src_offset : mip_sizes[level].offset; - const auto level_size = - static_cast(mip_sizes[level].src_size != 0 ? mip_sizes[level].src_size - : mip_sizes[level].size) * - view.image_layers; - desc.info.mip_layout[level] = { - level_offset, - level_size, - mip_padded[level].width, - mip_padded[level].height, - }; - } desc.view_info.format = target_format.format; if (is_1d) { desc.view_info.type = diff --git a/src/graphics/host_gpu/renderer/depthRenderTarget.cpp b/src/graphics/host_gpu/renderer/depthRenderTarget.cpp index 482334af2..8ac58043a 100644 --- a/src/graphics/host_gpu/renderer/depthRenderTarget.cpp +++ b/src/graphics/host_gpu/renderer/depthRenderTarget.cpp @@ -239,7 +239,7 @@ static TextureCache::ImageDesc MakeDepthTargetDesc(const CommandBuffer& buffer, desc.info.bytes_per_block = bytes; desc.info.samples = samples; desc.info.tile_mode = Prospero::TileMode::kDepth; - desc.info.mip_layout[0] = {0, depth_backing_size, pitch, height}; + desc.info.UpdateSize(); desc.info.metadata.range = has_htile ? GuestRange {z.htile_data_base_addr, htile_backing_size} : GuestRange {}; desc.info.metadata.kind = has_htile ? ImageMetadataKind::Htile : ImageMetadataKind::None; diff --git a/src/graphics/host_gpu/renderer/image/imageInfo.cpp b/src/graphics/host_gpu/renderer/image/imageInfo.cpp new file mode 100644 index 000000000..780d2e447 --- /dev/null +++ b/src/graphics/host_gpu/renderer/image/imageInfo.cpp @@ -0,0 +1,246 @@ +#include "graphics/host_gpu/renderer/image/imageInfo.h" + +#include "common/alignment.h" + +namespace Libs::Graphics { +namespace { + +uint64_t Multiply(uint64_t left, uint64_t right) { + EXIT_IF(right != 0 && left > UINT64_MAX / right); + return left * right; +} + +uint64_t Add(uint64_t left, uint64_t right) { + EXIT_IF(right > UINT64_MAX - left); + return left + right; +} + +bool IsDepthPlane(const ImageInfo& info) { + return (info.IsDepth() || info.pixel_format == vk::Format::eS8Uint) && + (info.tile_mode == Prospero::TileMode::kDepth || + info.tile_mode == Prospero::TileMode::kLinear); +} + +} // namespace + +vk::Extent3D ImageInfo::MipExtent(uint32_t level) const { + EXIT_IF(level >= resources.levels || level >= mip_layout.size()); + return {std::max(extent.width >> level, 1u), std::max(extent.height >> level, 1u), + IsVolume() ? std::max(extent.depth >> level, 1u) : resources.layers}; +} + +void ImageInfo::UpdateSize() { + EXIT_IF(guest_format == Prospero::BufferFormat::kInvalid || extent.width == 0 || + extent.height == 0 || extent.depth == 0 || resources.levels == 0 || + resources.levels > mip_layout.size() || resources.layers == 0 || samples == 0 || + !std::has_single_bit(samples)); + TileTextureElementLayout element {}; + EXIT_NOT_IMPLEMENTED(!TileGetTextureElementLayout(guest_format, element)); + if (bytes_per_block == 0) { + bytes_per_block = element.bytes; + } + EXIT_IF(bytes_per_block != element.bytes); + mip_layout = {}; + tiling = {}; + tiling.texel_width = element.texel_width; + tiling.texel_height = element.texel_height; + tiling.block.bytes_per_element = element.bytes; + tiled_slice_stride = 0; + linear_slice_stride = 0; + first_tail_level = 16; + + if (samples > 1) { + EXIT_NOT_IMPLEMENTED(IsVolume() || resources.levels != 1 || + (tile_mode != Prospero::TileMode::kDepth && + tile_mode != Prospero::TileMode::kRenderTarget)); + const auto fragments = std::countr_zero(samples); + if (pitch == 0) { + pitch = tile_mode == Prospero::TileMode::kDepth + ? TileGetDepthPitch(extent.width, bytes_per_block, fragments) + : TileGetRenderTargetPitch(extent.width, bytes_per_block, fragments); + } + EXIT_NOT_IMPLEMENTED(!TileGetTextureBlockLayout(guest_format, tile_mode, false, tiling)); + if (data.size == 0) { + TileSizeAlign slice {}; + EXIT_NOT_IMPLEMENTED(!TileGetRenderTargetSize(extent.width, extent.height, pitch, + bytes_per_block, slice, fragments)); + data.size = Multiply(slice.size, resources.layers); + } + EXIT_IF(data.size % resources.layers != 0); + tiled_slice_stride = data.size / resources.layers; + mip_layout[0] = {0, data.size, pitch, extent.height}; + return; + } + + if (pitch == 0) { + pitch = TileGetTexturePitch(guest_format, extent.width, tile_mode); + } + EXIT_IF(pitch == 0); + if (IsDepthPlane(*this) && resources.levels == 1) { + EXIT_NOT_IMPLEMENTED(IsVolume()); + if (IsTiled()) { + EXIT_NOT_IMPLEMENTED( + !TileGetTextureBlockLayout(guest_format, tile_mode, false, tiling)); + } + if (data.size == 0) { + TileSizeAlign slice {}; + TileGetTextureSize(guest_format, extent.width, extent.height, 1, tile_mode, &slice, + nullptr, nullptr); + data.size = Multiply(slice.size, resources.layers); + } + EXIT_IF(data.size % resources.layers != 0); + const auto slice = data.size / resources.layers; + EXIT_IF(Multiply(Multiply(pitch, extent.height), bytes_per_block) > slice); + linear_slice_stride = slice; + tiled_slice_stride = IsTiled() ? slice : 0; + mip_layout[0] = {0, data.size, pitch, extent.height, 0, slice, pitch}; + return; + } + if (!IsTiled()) { + TileSizeAlign total {}; + TileSizeOffset offsets[16] {}; + TilePaddedSize padded[16] {}; + TileGetTextureSize(guest_format, extent.width, extent.height, resources.levels, tile_mode, + &total, offsets, padded); + if (data.size == 0) { + data.size = Multiply(total.size, TransferLayers()); + } + for (uint32_t level = 0; level < resources.levels; ++level) { + auto& mip = mip_layout[level]; + mip.offset = offsets[level].offset; + mip.size = Multiply(offsets[level].size, MipExtent(level).depth); + mip.pitch = padded[level].width / element.texel_width; + mip.height = padded[level].height / element.texel_height; + mip.linear_offset = offsets[level].offset; + mip.linear_size = offsets[level].size; + mip.linear_pitch = mip.pitch; + linear_slice_stride = + std::max(linear_slice_stride, Add(mip.linear_offset, mip.linear_size)); + } + if (TransferLayers() > 1 && data.size % TransferLayers() == 0) { + linear_slice_stride = std::max(linear_slice_stride, data.size / TransferLayers()); + } + } else { + TileSurfaceLayout surface {}; + const TileSurfaceDescription description {guest_format, + tile_mode, + IsVolume() ? TileSurfaceDimension::Dim3D + : TileSurfaceDimension::Dim2D, + extent.width, + extent.height, + IsVolume() ? extent.depth : 1u, + resources.levels, + resources.layers}; + EXIT_NOT_IMPLEMENTED(!TileGetTiledTextureLayout(description, surface)); + tiling = surface.texture; + first_tail_level = surface.first_tail_level; + tiled_slice_stride = surface.block_slice_size; + if (data.size == 0) { + data.size = surface.total_size; + } + if (!IsVolume() && resources.layers > 1 && data.size % resources.layers == 0) { + tiled_slice_stride = std::max(tiled_slice_stride, data.size / resources.layers); + } + uint64_t linear_offset = 0; + uint32_t mip_pitch = IsVolume() ? extent.width : pitch; + for (uint32_t level = 0; level < resources.levels; ++level) { + const auto& source = surface.mips[level]; + auto& mip = mip_layout[level]; + const auto active = MipExtent(level); + mip.offset = source.offset; + mip.size = Multiply(source.size, IsVolume() ? 1u : resources.layers); + mip.pitch = source.padded_width; + mip.height = source.padded_height; + mip.tail_x = source.tail_x; + mip.tail_y = source.tail_y; + mip.linear_pitch = + Common::AlignUp(mip_pitch, element.texel_width) / element.texel_width; + const auto rows = + Common::AlignUp(active.height, element.texel_height) / element.texel_height; + mip.linear_size = Multiply(Multiply(mip.linear_pitch, rows), element.bytes); + const auto alignment = std::max(element.bytes, 4u); + EXIT_IF(linear_offset > UINT64_MAX - (alignment - 1u)); + mip.linear_offset = Common::AlignUp(linear_offset, alignment); + linear_offset = Add(mip.linear_offset, Multiply(mip.linear_size, active.depth)); + mip_pitch = std::max(mip_pitch >> 1u, 1u); + } + } +} + +uint64_t ImageInfo::LinearSize(uint32_t levels) const { + levels = levels == 0 ? resources.levels : levels; + EXIT_IF(levels == 0 || levels > resources.levels || levels > mip_layout.size()); + if (linear_slice_stride == 0) { + const auto& mip = mip_layout[levels - 1u]; + return Add(mip.linear_offset, Multiply(mip.linear_size, MipExtent(levels - 1u).depth)); + } + uint64_t size = 0; + for (uint32_t level = 0; level < levels; ++level) { + const auto& mip = mip_layout[level]; + const auto slices = MipExtent(level).depth; + const auto stride = linear_slice_stride != 0 ? linear_slice_stride : mip.linear_size; + size = std::max( + size, Add(Add(mip.linear_offset, Multiply(slices - 1u, stride)), mip.linear_size)); + } + return size; +} + +std::vector ImageInfo::BufferCopies(uint64_t buffer_offset, + uint32_t levels) const { + levels = levels == 0 ? resources.levels : levels; + EXIT_IF(levels > resources.levels || levels > mip_layout.size() || tiling.texel_width == 0 || + tiling.texel_height == 0); + const auto aspect = IsDepth() ? vk::ImageAspectFlagBits::eDepth + : pixel_format == vk::Format::eS8Uint ? vk::ImageAspectFlagBits::eStencil + : vk::ImageAspectFlagBits::eColor; + std::vector copies; + std::array image_heights {}; + uint64_t count = 0; + for (uint32_t level = 0; level < levels; ++level) { + const auto& mip = mip_layout[level]; + const auto active = MipExtent(level); + if (linear_slice_stride == 0) { + image_heights[level] = Common::AlignUp(active.height, tiling.texel_height); + } else { + const auto row_bytes = Multiply(mip.linear_pitch, tiling.block.bytes_per_element); + EXIT_IF(row_bytes == 0); + const auto rows = linear_slice_stride / row_bytes; + if (linear_slice_stride % row_bytes == 0 && + rows >= + (uint64_t {active.height} + tiling.texel_height - 1u) / tiling.texel_height && + rows <= UINT32_MAX / tiling.texel_height) { + image_heights[level] = static_cast(rows) * tiling.texel_height; + } + } + count += image_heights[level] != 0 ? 1u : active.depth; + } + copies.reserve(static_cast(count)); + for (uint32_t level = 0; level < levels; ++level) { + const auto& mip = mip_layout[level]; + const auto active = MipExtent(level); + vk::BufferImageCopy copy {}; + copy.bufferOffset = Add(buffer_offset, mip.linear_offset); + copy.bufferRowLength = mip.linear_pitch * tiling.texel_width; + copy.bufferImageHeight = image_heights[level] != 0 ? image_heights[level] + : !IsTiled() ? mip.height * tiling.texel_height + : Common::AlignUp(active.height, tiling.texel_height); + copy.imageSubresource = {aspect, level, 0, 1}; + copy.imageExtent = {active.width, active.height, 1}; + if (image_heights[level] == 0) { + for (uint32_t slice = 0; slice < active.depth; ++slice) { + copy.bufferOffset = Add(Add(buffer_offset, mip.linear_offset), + Multiply(slice, linear_slice_stride)); + copy.imageSubresource.baseArrayLayer = IsVolume() ? 0u : slice; + copy.imageOffset.z = IsVolume() ? static_cast(slice) : 0; + copies.push_back(copy); + } + } else { + copy.imageSubresource.layerCount = IsVolume() ? 1u : active.depth; + copy.imageExtent.depth = IsVolume() ? active.depth : 1u; + copies.push_back(copy); + } + } + return copies; +} + +} // namespace Libs::Graphics diff --git a/src/graphics/host_gpu/renderer/image/imageInfo.h b/src/graphics/host_gpu/renderer/image/imageInfo.h index e9239ae37..b8ae73e30 100644 --- a/src/graphics/host_gpu/renderer/image/imageInfo.h +++ b/src/graphics/host_gpu/renderer/image/imageInfo.h @@ -4,6 +4,7 @@ #include "common/assert.h" #include "graphics/guest_gpu/gpu_defs.h" #include "graphics/guest_gpu/gpu_format.h" +#include "graphics/guest_gpu/tile.h" #include "graphics/host_gpu/regionDefinitions.h" #include "graphics/host_gpu/vulkanCommon.h" @@ -12,6 +13,7 @@ #include #include #include +#include namespace Libs::Graphics { @@ -47,11 +49,19 @@ struct ImageSubresourceRange { }; struct ImageMipInfo { + // Guest mip chains run backwards within each layer or volume block-slice. + // Tail levels share an offset and use tail_x/tail_y within that block. uint64_t offset = 0; uint64_t size = 0; // Padded dimensions in storage elements (compressed blocks for BC formats). uint32_t pitch = 0; uint32_t height = 0; + uint64_t linear_offset = 0; + uint64_t linear_size = 0; // Per transfer slice. + uint32_t linear_pitch = 0; // Storage elements. + uint32_t tail_x = 0; + uint32_t tail_y = 0; + uint32_t surface_z = 0; auto operator<=>(const ImageMipInfo&) const = default; }; @@ -71,6 +81,17 @@ struct ImageInfo { Prospero::TileMode tile_mode = Prospero::TileMode::kLinear; bool bgra16 = false; std::array mip_layout {}; + TileTextureBlockLayout tiling; + uint64_t tiled_slice_stride = 0; + // Nonzero for guest-linear and depth-plane transfers; tiled colors are mip-major. + uint64_t linear_slice_stride = 0; + uint32_t first_tail_level = 16; + + void UpdateSize(); + [[nodiscard]] uint64_t LinearSize(uint32_t levels = 0) const; + [[nodiscard]] vk::Extent3D MipExtent(uint32_t level) const; + [[nodiscard]] std::vector BufferCopies(uint64_t buffer_offset = 0, + uint32_t levels = 0) const; [[nodiscard]] ColorTransform GetColorTransform() const noexcept { if (bgra16) return ColorTransform::SwapBgra16; diff --git a/src/graphics/host_gpu/renderer/image/textureCommon.cpp b/src/graphics/host_gpu/renderer/image/textureCommon.cpp index 7551a64cd..794b62fd3 100644 --- a/src/graphics/host_gpu/renderer/image/textureCommon.cpp +++ b/src/graphics/host_gpu/renderer/image/textureCommon.cpp @@ -1,16 +1,13 @@ #include "graphics/host_gpu/renderer/image/textureCommon.h" -#include "common/alignment.h" #include "common/assert.h" #include "graphics/guest_gpu/gpu_defs.h" #include "graphics/guest_gpu/gpu_format.h" -#include "graphics/host_gpu/renderer/image/tiler.h" #include "graphics/host_gpu/vulkanCommon.h" #include "graphics/shader/shader.h" #include #include -#include namespace Libs::Graphics { namespace { @@ -60,68 +57,6 @@ HostFormatInfo ResolveHostFormat(Prospero::BufferFormat guest_format, } } -uint32_t GetTextureLevelDepth(uint32_t depth, uint32_t level, bool volume_texture) { - return volume_texture ? std::max(depth >> level, 1u) : depth; -} - -size_t GetTextureRegionCount(uint32_t depth, uint32_t levels, bool volume_texture) { - size_t count = 0; - for (uint32_t level = 0; level < levels; level++) { - count += GetTextureLevelDepth(depth, level, volume_texture); - } - return count; -} - -uint64_t CalcTextureSliceStride(const TextureUploadMipLayout* mips, uint32_t levels, - uint64_t total_size, uint32_t depth) { - uint64_t stride = 0; - for (uint32_t mip_level = 0; mip_level < levels; ++mip_level) { - stride = std::max(stride, mips[mip_level].offset + mips[mip_level].size); - } - - if (depth > 1 && total_size != 0 && total_size % depth == 0) { - stride = std::max(stride, total_size / depth); - } - - return stride; -} - -uint64_t CalcLinearUploadLevelSize(const TileTextureElementLayout& element, uint32_t pitch, - uint32_t height) { - const uint32_t element_columns = - std::max((pitch + element.texel_width - 1u) / element.texel_width, 1u); - const uint32_t element_rows = - std::max((height + element.texel_height - 1u) / element.texel_height, 1u); - return static_cast(element_columns) * element_rows * element.bytes; -} - -uint64_t SetLinearUploadLevels(TextureUploadMipLayout* mips, - const TileTextureElementLayout& element, uint32_t height, - uint32_t levels, uint32_t base_pitch) { - uint64_t offset = 0; - auto mip_pitch = base_pitch; - auto mip_height = height; - - for (uint32_t mip_level = 0; mip_level < levels; ++mip_level) { - const auto size = CalcLinearUploadLevelSize(element, mip_pitch, mip_height); - mips[mip_level] = {offset, size}; - - offset += size; - if (mip_pitch > 1) { - mip_pitch /= 2; - } - if (mip_height > 1) { - mip_height /= 2; - } - } - - return offset; -} - -bool FitsBufferRange(uint64_t offset, uint64_t size, uint64_t capacity) { - return offset <= capacity && size <= capacity - offset; -} - } // namespace RenderTargetFormatInfo TextureGetRenderTargetFormat(Prospero::ChannelLayout layout, @@ -176,203 +111,4 @@ SurfaceFormatInfo TextureGetSurfaceFormatInfo(Prospero::BufferFormat format) { EXIT("unknown format: fmt = %u\n", static_cast(format)); } -TextureUploadLayout TextureCalcUploadLayout(Prospero::BufferFormat format, uint32_t width, - uint32_t height, uint32_t levels, uint32_t depth, - Prospero::TileMode tile_mode, uint64_t upload_size, - bool volume_texture, const char* owner) { - TextureUploadLayout layout {}; - layout.surface.description = { - format, - tile_mode, - volume_texture ? TileSurfaceDimension::Dim3D : TileSurfaceDimension::Dim2D, - width, - height, - volume_texture ? depth : 1u, - levels, - volume_texture ? 1u : depth, - }; - const auto& description = layout.surface.description; - TileTextureElementLayout element {}; - - if (format == Prospero::BufferFormat::kInvalid) { - EXIT("%s: legacy texture upload format unsupported: fmt=0 tile=%u size=%" PRIu64 - " extent=%ux%u levels=%u\n", - owner, static_cast(description.tile_mode), upload_size, width, height, - levels); - } - - if (tile_mode == Prospero::TileMode::kLinear) { - if (!TileGetTextureElementLayout(format, element)) { - EXIT("%s: unsupported linear texture format: fmt=%u\n", owner, - static_cast(format)); - } - } else { - if (!TileGetTiledTextureLayout(description, layout.surface)) { - EXIT("%s: unsupported typed tiled upload: fmt=%u tile=%u " - "size=%" PRIu64 " extent=%ux%u levels=%u\n", - owner, static_cast(format), static_cast(tile_mode), - upload_size, width, height, levels); - } - element = {layout.surface.texture.block.bytes_per_element, - layout.surface.texture.texel_width, layout.surface.texture.texel_height}; - } - layout.pitch = TileGetTexturePitch(format, width, description.tile_mode); - if (tile_mode == Prospero::TileMode::kLinear) { - TileSizeOffset level_sizes[16] {}; - TilePaddedSize padded_sizes[16] {}; - TileGetTextureSize(format, width, height, levels, description.tile_mode, nullptr, - level_sizes, padded_sizes); - for (uint32_t level = 0; level < levels; ++level) { - layout.mips[level] = {level_sizes[level].offset, level_sizes[level].size, - padded_sizes[level].width, padded_sizes[level].height}; - } - } else if (volume_texture) { - layout.slice_stride = SetLinearUploadLevels(layout.mips, element, height, levels, width); - } else { - layout.source_slice_stride = layout.surface.block_slice_size; - if (depth > 1 && upload_size != 0 && upload_size % depth == 0) { - layout.source_slice_stride = std::max(layout.source_slice_stride, upload_size / depth); - } - SetLinearUploadLevels(layout.mips, element, height, levels, layout.pitch); - } - - if (!volume_texture || tile_mode == Prospero::TileMode::kLinear) { - layout.slice_stride = CalcTextureSliceStride(layout.mips, levels, upload_size, depth); - } - return layout; -} - -std::vector TextureBuildImageCopies(const TextureUploadLayout& layout) { - const auto& description = layout.surface.description; - const bool volume_texture = description.dimension == TileSurfaceDimension::Dim3D; - const bool linear_texture = description.tile_mode == Prospero::TileMode::kLinear; - const auto depth = volume_texture ? description.depth : description.layers; - uint32_t mip_width = description.width; - uint32_t mip_height = description.height; - uint32_t mip_pitch = volume_texture && !linear_texture ? description.width : layout.pitch; - - std::vector regions; - regions.reserve(GetTextureRegionCount(depth, description.levels, volume_texture)); - for (uint32_t mip_level = 0; mip_level < description.levels; ++mip_level) { - EXIT_NOT_IMPLEMENTED(layout.mips[mip_level].size == 0); - const auto mip_depth = GetTextureLevelDepth(depth, mip_level, volume_texture); - vk::BufferImageCopy region {}; - region.imageSubresource = {vk::ImageAspectFlagBits::eColor, mip_level, 0, 1}; - region.imageExtent = {mip_width, mip_height, 1}; - if (linear_texture) { - region.bufferRowLength = layout.mips[mip_level].row_length; - region.bufferImageHeight = layout.mips[mip_level].image_height; - } else { - const auto texel_width = layout.surface.texture.texel_width; - const auto aligned_pitch = Common::AlignUp(mip_pitch, texel_width); - region.bufferRowLength = - aligned_pitch > Common::AlignUp(mip_width, texel_width) ? aligned_pitch : 0; - } - for (uint32_t slice_index = 0; slice_index < mip_depth; ++slice_index) { - region.bufferOffset = layout.mips[mip_level].offset + slice_index * layout.slice_stride; - region.imageSubresource.baseArrayLayer = volume_texture ? 0u : slice_index; - region.imageOffset.z = volume_texture ? static_cast(slice_index) : 0; - regions.push_back(region); - } - - if (mip_width > 1) { - mip_width /= 2; - } - if (mip_height > 1) { - mip_height /= 2; - } - if (mip_pitch > 1) { - mip_pitch /= 2; - } - } - - return regions; -} - -bool TextureBuildGpuTileInfos(uint64_t tiled_size, const std::vector& regions, - const TextureUploadLayout& layout, uint32_t levels, - std::vector& out_tile_infos) { - const auto& description = layout.surface.description; - const bool volume_texture = description.dimension == TileSurfaceDimension::Dim3D; - const auto depth = volume_texture ? description.depth : description.layers; - if (tiled_size == 0 || levels == 0 || levels > 16 || depth == 0 || - regions.size() != GetTextureRegionCount(depth, levels, volume_texture) || - Prospero::IsFmaskTextureFormat(description.format)) { - return false; - } - - const auto& surface = layout.surface; - const auto& texture = surface.texture; - const auto& block = texture.block; - if (surface.description.levels < levels || block.bytes_per_element == 0 || - block.family == TileBlockFamily::Count) { - return false; - } - - std::vector tile_infos; - tile_infos.reserve(regions.size()); - const auto slices_per_copy = volume_texture ? block.block_depth : 1u; - size_t region_base = 0; - for (uint32_t mip_level = 0; mip_level < levels; ++mip_level) { - const auto& mip = surface.mips[mip_level]; - const auto mip_depth = GetTextureLevelDepth(depth, mip_level, volume_texture); - for (uint32_t slice_index = 0; slice_index < mip_depth; slice_index += slices_per_copy) { - const auto& region = regions[region_base + slice_index]; - GpuTileInfo tile_info {}; - tile_info.family = block.family; - tile_info.bytes_per_element = block.bytes_per_element; - tile_info.linear_offset = region.bufferOffset; - tile_info.linear_size = layout.mips[mip_level].size; - tile_info.tiled_size = mip.size; - if (volume_texture) { - tile_info.depth = std::min(slices_per_copy, mip_depth - slice_index); - tile_info.linear_slice_stride = layout.slice_stride; - tile_info.linear_size += (tile_info.depth - 1u) * layout.slice_stride; - tile_info.tiled_offset = - mip.offset + - static_cast(slice_index / slices_per_copy) * surface.block_slice_size; - tile_info.surface_z = - block.block_depth == 1 ? static_cast(region.imageOffset.z) : 0; - } else { - const auto source_stride = layout.source_slice_stride != 0 - ? layout.source_slice_stride - : surface.block_slice_size; - if (slice_index > (UINT64_MAX - mip.offset) / source_stride) { - return false; - } - tile_info.tiled_offset = - mip.offset + static_cast(slice_index) * source_stride; - tile_info.surface_z = block.family == TileBlockFamily::RenderTarget64KB || - block.family == TileBlockFamily::Depth64KB - ? region.imageSubresource.baseArrayLayer - : 0; - } - if (!FitsBufferRange(tile_info.linear_offset, tile_info.linear_size, UINT64_MAX) || - !FitsBufferRange(tile_info.tiled_offset, tile_info.tiled_size, tiled_size)) { - return false; - } - const auto row_length = - region.bufferRowLength != 0 ? region.bufferRowLength : region.imageExtent.width; - const auto image_height = region.bufferImageHeight != 0 ? region.bufferImageHeight - : region.imageExtent.height; - tile_info.width = std::max( - (region.imageExtent.width + texture.texel_width - 1u) / texture.texel_width, 1u); - tile_info.height = - std::max((image_height + texture.texel_height - 1u) / texture.texel_height, 1u); - tile_info.pitch = - std::max((row_length + texture.texel_width - 1u) / texture.texel_width, 1u); - tile_info.tail = mip_level >= surface.first_tail_level; - tile_info.tail_x = tile_info.tail ? mip.tail_x : 0; - tile_info.tail_y = tile_info.tail ? mip.tail_y : 0; - tile_info.tiled_width = mip.padded_width; - tile_info.tiled_height = mip.padded_height; - tile_infos.push_back(tile_info); - } - region_base += mip_depth; - } - - out_tile_infos = std::move(tile_infos); - return true; -} - } // namespace Libs::Graphics diff --git a/src/graphics/host_gpu/renderer/image/textureCommon.h b/src/graphics/host_gpu/renderer/image/textureCommon.h index 50bb31fe8..de85f542c 100644 --- a/src/graphics/host_gpu/renderer/image/textureCommon.h +++ b/src/graphics/host_gpu/renderer/image/textureCommon.h @@ -2,15 +2,10 @@ #define EMULATOR_INCLUDE_EMULATOR_GRAPHICS_HOST_GPU_RENDERER_IMAGE_TEXTURECOMMON_H_ #include "graphics/guest_gpu/gpu_defs.h" -#include "graphics/guest_gpu/tile.h" #include "graphics/host_gpu/vulkanCommon.h" -#include - namespace Libs::Graphics { -struct GpuTileInfo; - struct RenderTargetFormatInfo { vk::Format format = vk::Format::eUndefined; uint32_t bytes_per_element = 0; @@ -24,35 +19,12 @@ struct SurfaceFormatInfo { Prospero::ColorComponentMapping host_to_storage; }; -struct TextureUploadMipLayout { - uint64_t offset = 0; - uint64_t size = 0; - uint32_t row_length = 0; - uint32_t image_height = 0; -}; - -struct TextureUploadLayout { - uint32_t pitch = 0; - uint64_t slice_stride = 0; - uint64_t source_slice_stride = 0; - TileSurfaceLayout surface; - TextureUploadMipLayout mips[16] = {}; -}; - vk::ComponentMapping TextureGetComponentMapping(uint32_t swizzle, Prospero::ColorComponentMapping host_to_storage); SurfaceFormatInfo TextureGetSurfaceFormatInfo(Prospero::BufferFormat format); RenderTargetFormatInfo TextureGetRenderTargetFormat(Prospero::ChannelLayout layout, Prospero::ChannelType type, Prospero::ChannelOrder order); -TextureUploadLayout TextureCalcUploadLayout(Prospero::BufferFormat format, uint32_t width, - uint32_t height, uint32_t levels, uint32_t depth, - Prospero::TileMode tile_mode, uint64_t upload_size, - bool volume_texture, const char* owner); -std::vector TextureBuildImageCopies(const TextureUploadLayout& layout); -bool TextureBuildGpuTileInfos(uint64_t tiled_size, const std::vector& regions, - const TextureUploadLayout& layout, uint32_t levels, - std::vector& out_tile_infos); } // namespace Libs::Graphics diff --git a/src/graphics/host_gpu/renderer/image/tiler.cpp b/src/graphics/host_gpu/renderer/image/tiler.cpp index a0332e4c2..31ad7baf5 100644 --- a/src/graphics/host_gpu/renderer/image/tiler.cpp +++ b/src/graphics/host_gpu/renderer/image/tiler.cpp @@ -31,7 +31,12 @@ TileManager::TileManager(GraphicContext& graphics, CommandScheduler& scheduler, StreamBuffer& stream_buffer) : m_graphics(graphics), m_scheduler(scheduler), m_stream_buffer(stream_buffer) { static_assert(FamilyCount == 9); - static_assert(sizeof(Push) == 60); + static_assert(sizeof(Mip) == 48); + static_assert(offsetof(TilingParams, mips) == 16); + static_assert(sizeof(TilingParams) == 784); + static_assert(sizeof(TilingParams) <= UINT16_MAX && PipelineCount <= UINT16_MAX); + static_assert(sizeof(Tiling) == 16); + static_assert(sizeof(Conversion) == 32); std::array bindings {}; for (uint32_t index = 0; index < 2; index++) { bindings[index] = {index, vk::DescriptorType::eStorageBuffer, 1, @@ -48,7 +53,8 @@ TileManager::TileManager(GraphicContext& graphics, CommandScheduler& scheduler, &m_descriptor_layout), "create TileManager descriptor layout"); - const vk::PushConstantRange push_range {vk::ShaderStageFlagBits::eCompute, 0, sizeof(Push)}; + const vk::PushConstantRange push_range {vk::ShaderStageFlagBits::eCompute, 0, + sizeof(Conversion)}; vk::PipelineLayoutCreateInfo layout_info {}; layout_info.setLayoutCount = 1; layout_info.pSetLayouts = &m_descriptor_layout; @@ -88,14 +94,25 @@ TileManager::~TileManager() { } } -void TileManager::Prepare(bool tile, uint64_t tiled_capacity, uint64_t linear_capacity, - std::span infos, uint64_t source_base, - uint64_t target_base, std::vector& dispatches, - ColorTransform transform) { - EXIT_IF(infos.empty() || tiled_capacity == 0 || linear_capacity == 0); - const auto& limits = m_graphics.GetPhysicalDeviceProperties().limits; +TileManager::Tiling TileManager::Prepare(bool tile, uint64_t tiled_capacity, + uint64_t linear_capacity, const ImageInfo& info, + uint32_t levels, uint64_t source_base, + uint64_t target_base) { + levels = levels != 0 ? levels : info.resources.levels; + EXIT_IF(levels == 0 || levels > MaxMipLevels || levels > info.resources.levels || + tiled_capacity == 0 || linear_capacity == 0); EXIT_NOT_IMPLEMENTED(tiled_capacity > UINT32_MAX || linear_capacity > UINT32_MAX); + const auto& block = info.tiling.block; + const auto transform = info.GetColorTransform(); + EXIT_NOT_IMPLEMENTED( + !info.IsTiled() || info.samples != 1 || Prospero::IsFmaskTextureFormat(info.guest_format) || + block.family >= TileBlockFamily::Count || !std::has_single_bit(block.bytes_per_element) || + block.bytes_per_element > 16 || block.block_width == 0 || block.block_height == 0 || + block.block_depth == 0 || block.block_size == 0 || info.tiling.texel_width == 0 || + info.tiling.texel_height == 0 || info.tiled_slice_stride == 0); + EXIT_IF((transform == ColorTransform::SwapBgra16 && block.bytes_per_element != 8u) || + (transform == ColorTransform::Reverse10_11_11 && block.bytes_per_element != 4u)); const auto checked_multiply = [](uint64_t left, uint64_t right, uint64_t& result) { return (left == 0 || right <= UINT64_MAX / left) && (result = left * right, true); }; @@ -106,112 +123,96 @@ void TileManager::Prepare(bool tile, uint64_t tiled_capacity, uint64_t linear_ca return size != 0 && offset <= capacity && size <= capacity - offset; }; - dispatches.clear(); - dispatches.reserve(infos.size()); - for (const auto& info: infos) { - EXIT_IF((transform == ColorTransform::SwapBgra16 && info.bytes_per_element != 8u) || - (transform == ColorTransform::Reverse10_11_11 && info.bytes_per_element != 4u)); - TileBlockLayout block {}; - const uint32_t tiled_width = info.tiled_width != 0 ? info.tiled_width : info.pitch; - const uint32_t tiled_height = info.tiled_height != 0 ? info.tiled_height : info.height; - const uint64_t groups_x = (static_cast(info.width) + 7u) / 8u; - const uint64_t groups_y = (static_cast(info.height) + 7u) / 8u; - EXIT_NOT_IMPLEMENTED( - !TileGetBlockLayout(info.family, info.bytes_per_element, block) || info.width == 0 || - info.height == 0 || info.depth == 0 || info.pitch < info.width || - groups_x > limits.maxComputeWorkGroupCount[0] || - groups_y > limits.maxComputeWorkGroupCount[1] || - info.depth > limits.maxComputeWorkGroupCount[2] || - (!info.tail && (tiled_width < info.width || tiled_height < info.height)) || - !valid_range(info.linear_offset, info.linear_size, linear_capacity) || - !valid_range(info.tiled_offset, info.tiled_size, tiled_capacity) || - (block.block_depth == 1 && info.depth != 1)); + // ImageInfo owns the layout. Upload only the header and active mip records. + TilingParams params; + params.color_transform = static_cast(transform); + for (uint32_t level = 0; level < levels; ++level) { + const auto& mip = info.mip_layout[level]; + const auto extent = info.MipExtent(level); + const auto width = static_cast( + (uint64_t {extent.width} + info.tiling.texel_width - 1u) / info.tiling.texel_width); + const auto height = static_cast( + (uint64_t {extent.height} + info.tiling.texel_height - 1u) / info.tiling.texel_height); + const auto depth = extent.depth; + const bool tail = level >= info.first_tail_level; + EXIT_NOT_IMPLEMENTED(width == 0 || height == 0 || depth == 0 || mip.linear_pitch < width || + mip.linear_size == 0 || mip.size == 0 || + (!tail && (mip.pitch < width || mip.height < height)) || + (!info.IsVolume() && mip.size % depth != 0)); uint64_t pitch_bytes = 0; - EXIT_NOT_IMPLEMENTED(!checked_multiply(info.pitch, info.bytes_per_element, pitch_bytes) || - pitch_bytes > UINT32_MAX); - uint64_t slice_bytes = info.linear_slice_stride; + EXIT_NOT_IMPLEMENTED( + !checked_multiply(mip.linear_pitch, block.bytes_per_element, pitch_bytes) || + pitch_bytes > UINT32_MAX); + const uint64_t slice_bytes = + info.linear_slice_stride != 0 ? info.linear_slice_stride : mip.linear_size; uint64_t minimum_slice = 0; - EXIT_NOT_IMPLEMENTED(!checked_multiply(pitch_bytes, info.height, minimum_slice)); - if (slice_bytes == 0) { - slice_bytes = minimum_slice; - } uint64_t linear_used = 0; - uint64_t bytes = 0; - EXIT_NOT_IMPLEMENTED((info.depth > 1 && slice_bytes < minimum_slice) || - !checked_multiply(info.depth - 1u, slice_bytes, bytes) || + uint64_t bytes = 0; + uint64_t linear_span = 0; + EXIT_NOT_IMPLEMENTED(!checked_multiply(pitch_bytes, height, minimum_slice) || + (depth > 1 && slice_bytes < minimum_slice) || + !checked_multiply(height - 1u, pitch_bytes, linear_used) || + !checked_multiply(width, block.bytes_per_element, bytes) || !checked_add(linear_used, bytes, linear_used) || - !checked_multiply(info.height - 1u, pitch_bytes, bytes) || - !checked_add(linear_used, bytes, linear_used) || - !checked_multiply(info.width, info.bytes_per_element, bytes) || - !checked_add(linear_used, bytes, linear_used) || - linear_used > info.linear_size || slice_bytes > UINT32_MAX); + linear_used > mip.linear_size || slice_bytes > UINT32_MAX || + !checked_multiply(depth - 1u, slice_bytes, linear_span) || + !checked_add(linear_span, mip.linear_size, linear_span) || + !valid_range(mip.linear_offset, linear_span, linear_capacity)); const uint64_t columns = - (static_cast(tiled_width) + block.block_width - 1u) / block.block_width; - const uint64_t rows = - (static_cast(tiled_height) + block.block_height - 1u) / block.block_height; - uint64_t blocks_per_slice = 0; - EXIT_NOT_IMPLEMENTED(!checked_multiply(columns, rows, blocks_per_slice) || - columns > UINT32_MAX || blocks_per_slice > UINT32_MAX); - if (info.tail) { + (uint64_t {mip.pitch} + block.block_width - 1u) / block.block_width; + if (tail) { EXIT_NOT_IMPLEMENTED( - info.family == TileBlockFamily::Standard256B || info.depth > block.block_depth || - info.tail_x >= block.block_width || info.width > block.block_width - info.tail_x || - info.tail_y >= block.block_height || - info.height > block.block_height - info.tail_y || - info.tiled_size < block.block_size); - } else { - const uint64_t slices = - (static_cast(info.depth) + block.block_depth - 1u) / block.block_depth; - uint64_t tiled_used = 0; - EXIT_NOT_IMPLEMENTED(!checked_multiply(blocks_per_slice, slices, tiled_used) || - !checked_multiply(tiled_used, block.block_size, tiled_used) || - tiled_used > info.tiled_size); + block.family == TileBlockFamily::Standard256B || mip.tail_x >= block.block_width || + width > block.block_width - mip.tail_x || mip.tail_y >= block.block_height || + height > block.block_height - mip.tail_y); } + const uint64_t tiled_plane = info.IsVolume() ? mip.size : mip.size / depth; + const uint64_t tiled_slice = info.tiled_slice_stride; + const uint64_t slices = (uint64_t {depth} + block.block_depth - 1u) / block.block_depth; + uint64_t tiled_span = 0; + EXIT_NOT_IMPLEMENTED(columns > UINT32_MAX || tiled_slice > UINT32_MAX || + (slices > 1 && tiled_slice < tiled_plane) || + !checked_multiply(slices - 1u, tiled_slice, tiled_span) || + !checked_add(tiled_span, tiled_plane, tiled_span) || + !valid_range(mip.offset, tiled_span, tiled_capacity)); - const uint32_t alignment = std::min(info.bytes_per_element, 4u); - EXIT_NOT_IMPLEMENTED(((info.linear_offset | info.tiled_offset | pitch_bytes | slice_bytes) & - (alignment - 1u)) != 0); - const uint64_t src = source_base + (tile ? info.linear_offset : info.tiled_offset); - const uint64_t dst = target_base + (tile ? info.tiled_offset : info.linear_offset); - EXIT_NOT_IMPLEMENTED(src > UINT32_MAX || dst > UINT32_MAX); - - const uint32_t family_index = static_cast(info.family); - const uint32_t element_index = std::countr_zero(info.bytes_per_element); - EXIT_NOT_IMPLEMENTED(family_index >= FamilyCount || element_index >= BytesPerElementCount); - Dispatch dispatch {}; - dispatch.pipeline_slot = - ((tile ? FamilyCount : 0u) + family_index) * BytesPerElementCount + element_index; - dispatch.push.src_base = static_cast(src); - dispatch.push.dst_base = static_cast(dst); - dispatch.push.width = info.width; - dispatch.push.height = info.height; - dispatch.push.depth = info.depth; - dispatch.push.surface_z = info.surface_z; - dispatch.push.pitch_bytes = static_cast(pitch_bytes); - dispatch.push.slice_bytes = static_cast(slice_bytes); - dispatch.push.blocks_per_row = static_cast(columns); - dispatch.push.blocks_per_slice = static_cast(blocks_per_slice); - dispatch.push.tail_x = info.tail_x; - dispatch.push.tail_y = info.tail_y; - dispatch.push.tail = info.tail; - dispatch.push.color_transform = static_cast(transform); - dispatches.push_back(dispatch); + const uint32_t alignment = std::min(block.bytes_per_element, 4u); + EXIT_NOT_IMPLEMENTED( + ((mip.linear_offset | mip.offset | pitch_bytes | slice_bytes | tiled_slice) & + (alignment - 1u)) != 0); + uint64_t src = 0, dst = 0, texels = 0; + EXIT_NOT_IMPLEMENTED( + !checked_add(source_base, tile ? mip.linear_offset : mip.offset, src) || + !checked_add(target_base, tile ? mip.offset : mip.linear_offset, dst) || + src > UINT32_MAX || dst > UINT32_MAX || !checked_multiply(width, height, texels) || + !checked_multiply(texels, depth, texels) || texels > UINT32_MAX - params.num_texels); + params.mips[level] = {static_cast(src), + static_cast(dst), + width, + height, + static_cast(pitch_bytes), + static_cast(slice_bytes), + static_cast(tiled_slice), + static_cast(columns), + mip.surface_z, + tail ? mip.tail_x : 0u, + tail ? mip.tail_y : 0u, + static_cast(texels)}; + params.num_texels += static_cast(texels); } - const uint64_t uniform_alignment = - std::max(limits.minUniformBufferOffsetAlignment, 1); - const uint64_t stride = Common::AlignUp(sizeof(Push), uniform_alignment); - EXIT_NOT_IMPLEMENTED(dispatches.size() > UINT64_MAX / stride); - const uint64_t bytes = dispatches.size() * stride; - auto [mapped, offset] = m_stream_buffer.Map(bytes, uniform_alignment); - EXIT_IF(mapped == nullptr); - for (size_t index = 0; index < dispatches.size(); index++) { - std::memcpy(mapped + index * stride, &dispatches[index].push, sizeof(Push)); - dispatches[index].params_offset = offset + index * stride; - } - m_stream_buffer.Commit(); + const auto& limits = m_graphics.GetPhysicalDeviceProperties().limits; + const auto params_size = + static_cast(offsetof(TilingParams, mips) + levels * sizeof(Mip)); + const auto offset = m_stream_buffer.Copy( + ¶ms, params_size, + std::max(limits.minUniformBufferOffsetAlignment, alignof(TilingParams))); + const auto slot = + ((tile ? FamilyCount : 0u) + static_cast(block.family)) * BytesPerElementCount + + std::countr_zero(block.bytes_per_element); + return {offset, params.num_texels, params_size, static_cast(slot)}; } vk::Pipeline TileManager::GetPipeline(uint32_t slot) { @@ -257,10 +258,9 @@ vk::Pipeline TileManager::GetPipeline(uint32_t slot) { return m_pipelines[slot]; } -void TileManager::Record(vk::Buffer source, uint64_t source_offset, - uint64_t source_capacity, vk::Buffer target, uint64_t target_offset, - uint64_t target_capacity, std::span dispatches, - bool clear_target) { +void TileManager::Record(vk::Buffer source, uint64_t source_offset, uint64_t source_capacity, + vk::Buffer target, uint64_t target_offset, uint64_t target_capacity, + const Tiling& tiling, bool clear_target) { const auto& limits = m_graphics.GetPhysicalDeviceProperties().limits; const uint64_t descriptor_alignment = std::max(limits.minStorageBufferOffsetAlignment, 4); @@ -295,9 +295,8 @@ void TileManager::Record(vk::Buffer source, uint64_t source_offset, barriers[2].srcAccessMask = vk::AccessFlagBits::eHostWrite; barriers[2].dstAccessMask = vk::AccessFlagBits::eUniformRead; barriers[2].buffer = m_stream_buffer.Handle(); - barriers[2].offset = dispatches.front().params_offset; - barriers[2].size = - dispatches.back().params_offset - dispatches.front().params_offset + sizeof(Push); + barriers[2].offset = tiling.params_offset; + barriers[2].size = tiling.params_size; command.pipelineBarrier( vk::PipelineStageFlagBits::eAllCommands | vk::PipelineStageFlagBits::eHost, vk::PipelineStageFlagBits::eComputeShader | vk::PipelineStageFlagBits::eTransfer, {}, 0, @@ -314,24 +313,26 @@ void TileManager::Record(vk::Buffer source, uint64_t source_offset, const vk::DescriptorBufferInfo source_info {source, source_descriptor_offset, source_range}; const vk::DescriptorBufferInfo target_info {target, target_descriptor_offset, target_range}; - for (auto& dispatch: dispatches) { - const vk::DescriptorBufferInfo params_info {m_stream_buffer.Handle(), - dispatch.params_offset, sizeof(Push)}; - const vk::DescriptorBufferInfo infos[] {source_info, target_info, params_info}; - std::array writes {}; - for (uint32_t index = 0; index < writes.size(); index++) { - writes[index].dstBinding = index; - writes[index].descriptorCount = 1; - writes[index].descriptorType = index == 2 ? vk::DescriptorType::eUniformBuffer - : vk::DescriptorType::eStorageBuffer; - writes[index].pBufferInfo = &infos[index]; - } - command.pushDescriptorSetKHR(vk::PipelineBindPoint::eCompute, m_pipeline_layout, 0, - static_cast(writes.size()), writes.data()); - command.bindPipeline(vk::PipelineBindPoint::eCompute, GetPipeline(dispatch.pipeline_slot)); - command.dispatch((dispatch.push.width + 7u) / 8u, (dispatch.push.height + 7u) / 8u, - dispatch.push.depth); + const vk::DescriptorBufferInfo params_info {m_stream_buffer.Handle(), tiling.params_offset, + tiling.params_size}; + const vk::DescriptorBufferInfo infos[] {source_info, target_info, params_info}; + std::array writes {}; + for (uint32_t index = 0; index < writes.size(); index++) { + writes[index].dstBinding = index; + writes[index].descriptorCount = 1; + writes[index].descriptorType = + index == 2 ? vk::DescriptorType::eUniformBuffer : vk::DescriptorType::eStorageBuffer; + writes[index].pBufferInfo = &infos[index]; } + command.pushDescriptorSetKHR(vk::PipelineBindPoint::eCompute, m_pipeline_layout, 0, + static_cast(writes.size()), writes.data()); + command.bindPipeline(vk::PipelineBindPoint::eCompute, GetPipeline(tiling.pipeline_slot)); + const uint64_t groups = (uint64_t {tiling.num_texels} + 63u) / 64u; + const uint32_t groups_y = static_cast( + (groups + limits.maxComputeWorkGroupCount[0] - 1u) / limits.maxComputeWorkGroupCount[0]); + const uint32_t groups_x = static_cast((groups + groups_y - 1u) / groups_y); + EXIT_NOT_IMPLEMENTED(groups_y > limits.maxComputeWorkGroupCount[1]); + command.dispatch(groups_x, groups_y, 1); barriers[1].srcAccessMask = vk::AccessFlagBits::eShaderWrite; barriers[1].dstAccessMask = vk::AccessFlagBits::eTransferRead | vk::AccessFlagBits::eMemoryRead; @@ -341,51 +342,53 @@ void TileManager::Record(vk::Buffer source, uint64_t source_offset, } TileManager::Result TileManager::Detile(vk::Buffer tiled, uint64_t tiled_offset, - uint64_t tiled_capacity, uint64_t linear_capacity, - std::span infos, - ColorTransform transform) { + const ImageInfo& info) { + if (!info.IsTiled()) { + const Result source {tiled, tiled_offset, info.data.size}; + return info.GetColorTransform() == ColorTransform::None + ? source + : TransformColor(source, info.GetColorTransform(), true); + } const auto& limits = m_graphics.GetPhysicalDeviceProperties().limits; const uint64_t descriptor_alignment = std::max(limits.minStorageBufferOffsetAlignment, 4); - const uint64_t source_base = tiled_offset & (descriptor_alignment - 1); - std::vector dispatches; - Prepare(false, tiled_capacity, linear_capacity, infos, source_base, 0, dispatches, transform); + const uint64_t source_base = tiled_offset & (descriptor_alignment - 1); + const auto linear_capacity = info.LinearSize(); + const auto tiling = Prepare(false, info.data.size, linear_capacity, info, 0, source_base, 0); auto scratch = GetScratchBuffer(linear_capacity, tiled); - Record(tiled, tiled_offset, tiled_capacity, scratch.buffer, 0, scratch.size, dispatches, - true); + Record(tiled, tiled_offset, info.data.size, scratch.buffer, 0, scratch.size, tiling, true); return {scratch.buffer, 0, linear_capacity}; } void TileManager::Tile(vk::Buffer linear, uint64_t linear_offset, uint64_t linear_capacity, vk::Buffer tiled, uint64_t tiled_offset, uint64_t tiled_capacity, - std::span infos) { + const ImageInfo& info, uint32_t levels) { const auto& limits = m_graphics.GetPhysicalDeviceProperties().limits; const uint64_t descriptor_alignment = std::max(limits.minStorageBufferOffsetAlignment, 4); - const uint64_t source_base = linear_offset & (descriptor_alignment - 1); - const uint64_t target_base = tiled_offset & (descriptor_alignment - 1); - std::vector dispatches; - Prepare(true, tiled_capacity, linear_capacity, infos, source_base, target_base, dispatches); - Record(linear, linear_offset, linear_capacity, tiled, tiled_offset, tiled_capacity, - dispatches, false); + const uint64_t source_base = linear_offset & (descriptor_alignment - 1); + const uint64_t target_base = tiled_offset & (descriptor_alignment - 1); + const auto tiling = + Prepare(true, tiled_capacity, linear_capacity, info, levels, source_base, target_base); + Record(linear, linear_offset, linear_capacity, tiled, tiled_offset, tiled_capacity, tiling, + false); } void TileManager::TileImage(Image& image, std::span regions, vk::Buffer tiled, uint64_t tiled_offset, uint64_t tiled_capacity, - uint64_t linear_capacity, std::span infos, - ColorTransform transform) { + uint32_t levels) { EXIT_IF(regions.empty()); const auto& limits = m_graphics.GetPhysicalDeviceProperties().limits; const uint64_t descriptor_alignment = std::max(limits.minStorageBufferOffsetAlignment, 4); - const uint64_t target_base = tiled_offset & (descriptor_alignment - 1); - std::vector dispatches; + const uint64_t target_base = tiled_offset & (descriptor_alignment - 1); + const auto linear_capacity = image.info.LinearSize(levels); // Reserve stream parameters before recording the image download. - Prepare(true, tiled_capacity, linear_capacity, infos, 0, target_base, dispatches, transform); + const auto tiling = + Prepare(true, tiled_capacity, linear_capacity, image.info, levels, 0, target_base); auto linear = GetScratchBuffer(linear_capacity, tiled); image.Download(regions, linear.buffer, 0, linear.size); - Record(linear.buffer, 0, linear_capacity, tiled, tiled_offset, tiled_capacity, - dispatches, false); + Record(linear.buffer, 0, linear_capacity, tiled, tiled_offset, tiled_capacity, tiling, false); } TileManager::Result TileManager::GetScratchBuffer(uint64_t size, vk::Buffer input) { @@ -563,13 +566,13 @@ void TileManager::ConvertD16(Result source, Result target, D16Direction directio } command.pushDescriptorSetKHR(vk::PipelineBindPoint::eCompute, m_pipeline_layout, 0, static_cast(writes.size()), writes.data()); - Push push {}; + Conversion push {}; push.src_base = source_binding.base; push.dst_base = target_binding.base; push.width = layout.width; push.height = rows; - push.pitch_bytes = static_cast(layout.source_row_stride); - push.slice_bytes = static_cast(layout.target_row_stride); + push.source_pitch = static_cast(layout.source_row_stride); + push.target_pitch = static_cast(layout.target_row_stride); command.pushConstants(m_pipeline_layout, vk::ShaderStageFlagBits::eCompute, 0, sizeof(push), &push); command.dispatch(static_cast(groups_x), rows, 1); @@ -644,7 +647,7 @@ void TileManager::TransformColor(Result input, Result output, ColorTransform tra command.bindPipeline(vk::PipelineBindPoint::eCompute, m_color_transform); command.pushDescriptorSetKHR(vk::PipelineBindPoint::eCompute, m_pipeline_layout, 0, static_cast(writes.size()), writes.data()); - Push push {}; + Conversion push {}; push.src_base = input_binding.base; push.dst_base = output_binding.base; push.width = pixels; diff --git a/src/graphics/host_gpu/renderer/image/tiler.h b/src/graphics/host_gpu/renderer/image/tiler.h index aa0e10aaa..7716e8175 100644 --- a/src/graphics/host_gpu/renderer/image/tiler.h +++ b/src/graphics/host_gpu/renderer/image/tiler.h @@ -9,7 +9,6 @@ #include #include #include -#include namespace Libs::Graphics { @@ -20,26 +19,6 @@ class StreamBuffer; struct GraphicContext; struct TileManagerTestAccess; -struct GpuTileInfo { - TileBlockFamily family = TileBlockFamily::Count; - uint32_t bytes_per_element = 0; - uint64_t linear_offset = 0; - uint64_t linear_size = 0; - uint64_t tiled_offset = 0; - uint64_t tiled_size = 0; - uint64_t linear_slice_stride = 0; - uint32_t width = 0; - uint32_t height = 0; - uint32_t depth = 1; - uint32_t pitch = 0; - uint32_t tail_x = 0; - uint32_t tail_y = 0; - bool tail = false; - uint32_t tiled_width = 0; - uint32_t tiled_height = 0; - uint32_t surface_z = 0; -}; - class TileManager final { public: enum class D16Direction { Promote, Demote }; @@ -64,15 +43,12 @@ public: KYTY_CLASS_NO_COPY(TileManager); // Consume scratch results before the next acquisition, or pass their buffer as input. - [[nodiscard]] Result Detile(vk::Buffer tiled, uint64_t tiled_offset, uint64_t tiled_capacity, - uint64_t linear_capacity, std::span infos, - ColorTransform transform = ColorTransform::None); + [[nodiscard]] Result Detile(vk::Buffer tiled, uint64_t tiled_offset, const ImageInfo& info); void Tile(vk::Buffer linear, uint64_t linear_offset, uint64_t linear_capacity, vk::Buffer tiled, - uint64_t tiled_offset, uint64_t tiled_capacity, std::span infos); + uint64_t tiled_offset, uint64_t tiled_capacity, const ImageInfo& info, + uint32_t levels = 0); void TileImage(Image& image, std::span regions, vk::Buffer tiled, - uint64_t tiled_offset, uint64_t tiled_capacity, uint64_t linear_capacity, - std::span infos, - ColorTransform transform = ColorTransform::None); + uint64_t tiled_offset, uint64_t tiled_capacity, uint32_t levels = 0); [[nodiscard]] Result GetScratchBuffer(uint64_t size, vk::Buffer input = nullptr); void ConvertD16(Result source, Result target, D16Direction direction, bool d32, const D16Layout& layout); @@ -86,29 +62,44 @@ private: static constexpr uint32_t BytesPerElementCount = 5; static constexpr uint32_t DirectionCount = 2; static constexpr uint32_t PipelineCount = FamilyCount * BytesPerElementCount * DirectionCount; + static constexpr uint32_t MaxMipLevels = 16; - struct Push { + struct alignas(16) Mip { uint32_t src_base; uint32_t dst_base; uint32_t width; uint32_t height; - uint32_t depth; - uint32_t surface_z; uint32_t pitch_bytes; uint32_t slice_bytes; + uint32_t tiled_slice_bytes; uint32_t blocks_per_row; - uint32_t blocks_per_slice; + uint32_t surface_z; uint32_t tail_x; uint32_t tail_y; - uint32_t tail; + uint32_t num_texels; + }; + struct TilingParams { + uint32_t num_texels = 0; + uint32_t color_transform = 0; + std::array padding {}; + std::array mips; + }; + struct Tiling { + uint64_t params_offset; + uint32_t num_texels; + uint16_t params_size; + uint16_t pipeline_slot; + }; + struct Conversion { + uint32_t src_base; + uint32_t dst_base; + uint32_t width; + uint32_t height; + uint32_t source_pitch; + uint32_t target_pitch; uint32_t color_transform; uint32_t to_host; }; - struct Dispatch { - Push push {}; - uint32_t pipeline_slot = 0; - uint64_t params_offset = 0; - }; struct StorageBinding { vk::DescriptorBufferInfo info; uint32_t base = 0; @@ -119,12 +110,12 @@ private: uint64_t active, uint32_t remaining, uint64_t alignment, uint64_t max_range, uint32_t max_groups) noexcept; - void Prepare(bool tile, uint64_t tiled_capacity, uint64_t linear_capacity, - std::span infos, uint64_t source_base, uint64_t target_base, - std::vector& dispatches, ColorTransform transform = ColorTransform::None); - void Record(vk::Buffer source, uint64_t source_offset, uint64_t source_capacity, - vk::Buffer target, uint64_t target_offset, uint64_t target_capacity, - std::span dispatches, bool clear_target); + [[nodiscard]] Tiling Prepare(bool tile, uint64_t tiled_capacity, uint64_t linear_capacity, + const ImageInfo& info, uint32_t levels, uint64_t source_base, + uint64_t target_base); + void Record(vk::Buffer source, uint64_t source_offset, uint64_t source_capacity, + vk::Buffer target, uint64_t target_offset, uint64_t target_capacity, + const Tiling& tiling, bool clear_target); [[nodiscard]] vk::Pipeline GetPipeline(uint32_t slot); GraphicContext& m_graphics; diff --git a/src/graphics/host_gpu/renderer/pipeline/descriptors.cpp b/src/graphics/host_gpu/renderer/pipeline/descriptors.cpp index d5e8124df..8b36ffc34 100644 --- a/src/graphics/host_gpu/renderer/pipeline/descriptors.cpp +++ b/src/graphics/host_gpu/renderer/pipeline/descriptors.cpp @@ -378,51 +378,6 @@ static TextureCache::ImageDesc NullTextureDesc(const ShaderRecompiler::IR::Image return desc; } -static void PopulateTextureMipLayout(ImageInfo& info) { - if (info.IsVolume() && info.tile_mode != Prospero::TileMode::kLinear) { - TileSurfaceLayout surface {}; - const TileSurfaceDescription description { - info.guest_format, info.tile_mode, TileSurfaceDimension::Dim3D, info.extent.width, - info.extent.height, info.extent.depth, info.resources.levels, 1}; - if (!TileGetTiledTextureLayout(description, surface)) { - EXIT("unsupported normalized volume texture layout\n"); - } - for (uint32_t level = 0; level < info.resources.levels; level++) { - const auto& mip = surface.mips[level]; - info.mip_layout[level] = { - mip.offset, - mip.size, - mip.padded_width, - mip.padded_height, - }; - } - return; - } - - TileSizeOffset levels[16] {}; - TilePaddedSize padded[16] {}; - TileGetTextureSize(info.guest_format, info.extent.width, info.extent.height, - info.resources.levels, info.tile_mode, nullptr, levels, padded); - const auto texel_shift = info.IsBlock() ? 2u : 0u; - for (uint32_t level = 0; level < info.resources.levels; level++) { - const auto offset = - levels[level].src_size != 0 ? levels[level].src_offset : levels[level].offset; - auto size = static_cast(levels[level].src_size != 0 ? levels[level].src_size - : levels[level].size); - if (info.IsVolume()) { - size *= std::max(info.extent.depth >> level, 1u); - } else { - size *= info.resources.layers; - } - info.mip_layout[level] = { - offset, - size, - padded[level].width >> texel_shift, - padded[level].height >> texel_shift, - }; - } -} - static ImageViewInfo TextureViewInfo(const ShaderRecompiler::IR::ImageResource& resource, const ShaderTextureResource& descriptor, vk::Format format, const SurfaceFormatInfo& surface_format, bool storage, @@ -490,45 +445,54 @@ static ImageViewInfo TextureViewInfo(const ShaderRecompiler::IR::ImageResource& return view; } -static bool ResolveTextureMipView(const TileSurfaceDescription& description, bool metadata, - uint32_t view_levels, uint32_t& levels, uint32_t& base_level) { - TileSurfaceLayout physical {}; - TileSurfaceLayout view {}; - auto view_description = description; - view_description.levels = levels; - if (!TileGetTiledTextureLayout(description, physical) || - !TileGetTiledTextureLayout(view_description, view)) { +static bool ResolveTextureMipView(ImageInfo& info, uint32_t physical_levels, bool metadata, + uint32_t view_levels, uint32_t& base_level) { + if (!info.IsTiled()) { return false; } - if (physical.first_tail_level == view.first_tail_level && - physical.block_slice_size == view.block_slice_size && - physical.total_size == view.total_size && - std::equal(std::begin(physical.mips), std::begin(physical.mips) + description.levels, - std::begin(view.mips))) { - return true; + ImageInfo physical = info; + physical.resources.levels = physical_levels; + physical.UpdateSize(); + info.UpdateSize(); + const auto same_mip = [&](uint32_t target, uint32_t source) { + const auto& left = physical.mip_layout[target]; + const auto& right = info.mip_layout[source]; + return left.offset == right.offset && left.size == right.size && + left.pitch == right.pitch && left.height == right.height && + left.tail_x == right.tail_x && left.tail_y == right.tail_y && + left.surface_z == right.surface_z && + physical.MipExtent(target) == info.MipExtent(source); + }; + if (physical.first_tail_level == info.first_tail_level && + physical.tiled_slice_stride == info.tiled_slice_stride && + physical.data.size == info.data.size) { + bool matches = true; + for (uint32_t level = 0; level < physical_levels; ++level) { + matches &= same_mip(level, level); + } + if (matches) { + return true; + } } - if (metadata || ((description.layers > 1 || description.depth > 1) && - physical.block_slice_size != view.block_slice_size)) { + if (metadata || + (info.TransferLayers() > 1 && physical.tiled_slice_stride != info.tiled_slice_stride)) { return false; } // T# addresses the last mip. A view can select the same stored subresources // with different mip indices; inaccessible mips need no host representation. - for (uint32_t base = 0; base + view_levels <= description.levels; ++base) { + for (uint32_t base = 0; base + view_levels <= physical_levels; ++base) { bool matches = true; for (uint32_t i = 0; i < view_levels; ++i) { const auto source = base_level + i; const auto target = base + i; - if (physical.mips[target] != view.mips[source] || - (target >= physical.first_tail_level) != (source >= view.first_tail_level) || - std::max(description.width >> target, 1u) != std::max(description.width >> source, 1u) || - std::max(description.height >> target, 1u) != std::max(description.height >> source, 1u) || - std::max(description.depth >> target, 1u) != std::max(description.depth >> source, 1u)) { + if (!same_mip(target, source) || + (target >= physical.first_tail_level) != (source >= info.first_tail_level)) { matches = false; break; } } if (matches) { - levels = description.levels; + info = std::move(physical); base_level = base; return true; } @@ -613,40 +577,6 @@ TextureBinding RenderExecutor::ResolveTexture(const ShaderRecompiler::IR::ImageR type == Prospero::ImageType::kColor2DMsaaArray; const auto image_layers = layered ? depth : 1u; auto view_base = static_cast(base_level); - if (levels > physical_levels) { - const TileSurfaceDescription physical { - format, tile, volume ? TileSurfaceDimension::Dim3D : TileSurfaceDimension::Dim2D, - width, height, volume ? depth : 1u, physical_levels, image_layers}; - if (!ResolveTextureMipView(physical, !resource.r128 && descriptor.MetaCompress(), - view_levels, levels, view_base)) { - EXIT("unsupported texture mip view changes physical layout: base=%u last=%u max=%u " - "extent=%ux%ux%u tile=%u\n", - base_level, last_level, max_mip, width, height, depth, - static_cast(tile)); - } - } - uint32_t pitch = 0; - TileSizeAlign size {}; - if (multisampled) { - const auto bytes = Prospero::NumBytesPerElement(format); - pitch = depth_tile ? TileGetDepthPitch(width, bytes, last_level) - : TileGetRenderTargetPitch(width, bytes, last_level); - if (pitch == 0 || !TileGetRenderTargetSize(width, height, pitch, bytes, size, last_level) || - size.size > UINT32_MAX / image_layers) { - EXIT("unsupported multisample texture layout\n"); - } - size.size *= image_layers; - } else { - pitch = TileGetTexturePitch(format, width, tile); - TileGetTextureTotalSize(format, width, height, volume ? depth : image_layers, - physical_levels, tile, volume, size); - } - EXIT_NOT_IMPLEMENTED(size.size == 0 || size.align == 0 || - (address & (static_cast(size.align) - 1u)) != 0); - if (storage) { - ValidateStorageTexture(resource, descriptor, size.size); - } - auto pixel_format = surface_format.vk_format; if (resource.depth_compare) { if (const auto* depth_format = FindGuestDepthFormatPolicy(format)) { @@ -663,17 +593,35 @@ TextureBinding RenderExecutor::ResolveTexture(const ShaderRecompiler::IR::ImageR : pixel_format; const auto block_bytes = Prospero::BlockCompressedBytesPerBlock(format); TextureCache::ImageDesc desc {}; - desc.info.data = {address, size.size}; + desc.info.data = {address, 0}; desc.info.pixel_format = pixel_format; desc.info.guest_format = format; desc.info.type = TextureBaseType(type); desc.info.extent = {width, height, volume ? depth : 1u}; desc.info.resources = {levels, image_layers}; - desc.info.pitch = pitch; desc.info.bytes_per_block = block_bytes != 0 ? block_bytes : Prospero::NumBytesPerElement(format); desc.info.samples = samples; desc.info.tile_mode = tile; + if (levels > physical_levels) { + if (!ResolveTextureMipView(desc.info, physical_levels, + !resource.r128 && descriptor.MetaCompress(), view_levels, + view_base)) { + EXIT("unsupported texture mip view changes physical layout: base=%u last=%u max=%u " + "extent=%ux%ux%u tile=%u\n", + base_level, last_level, max_mip, width, height, depth, + static_cast(tile)); + } + } else { + desc.info.UpdateSize(); + } + const auto alignment = + tile == Prospero::TileMode::kLinear ? 256u : desc.info.tiling.block.block_size; + EXIT_NOT_IMPLEMENTED(desc.info.data.size == 0 || alignment == 0 || + (address & (static_cast(alignment) - 1u)) != 0); + if (storage) { + ValidateStorageTexture(resource, descriptor, desc.info.data.size); + } if (!resource.r128 && descriptor.MetaCompress() && tile != Prospero::TileMode::kDepth && !desc.info.IsDepth()) { TileSizeAlign metadata_size {}; @@ -684,11 +632,6 @@ TextureBinding RenderExecutor::ResolveTexture(const ShaderRecompiler::IR::ImageR desc.info.metadata.range = {descriptor.MetaAddr() << 8u, metadata_size.size}; desc.info.metadata.dcc_alpha_msb = descriptor.DccAlphaPos(); } - if (samples > 1) { - desc.info.mip_layout[0] = {0, size.size, pitch, height}; - } else { - PopulateTextureMipLayout(desc.info); - } desc.view_info = TextureViewInfo(resource, descriptor, view_format, surface_format, storage, view_levels, desc.info.resources.layers); desc.view_info.base_level = view_base; @@ -704,7 +647,8 @@ TextureBinding RenderExecutor::ResolveTexture(const ShaderRecompiler::IR::ImageR if (storage) { EXIT("depth target cannot be bound as a storage image\n"); } - ValidateSampledDepthBinding(resource, descriptor, *image, pixel_format, size.size); + ValidateSampledDepthBinding(resource, descriptor, *image, pixel_format, + desc.info.data.size); } else if (storage) { ValidateStorageColorView(image->info.pixel_format, view_format, descriptor.DstSelXYZW()); } else { diff --git a/src/graphics/host_gpu/shaders/gpu_tiler_color_transform.comp b/src/graphics/host_gpu/shaders/gpu_tiler_color_transform.comp index 7892b4c35..42355af6d 100644 --- a/src/graphics/host_gpu/shaders/gpu_tiler_color_transform.comp +++ b/src/graphics/host_gpu/shaders/gpu_tiler_color_transform.comp @@ -6,20 +6,13 @@ layout(local_size_x = 64) in; layout(set = 0, binding = 0, std430) readonly buffer Input { uint data[]; } input_buffer; layout(set = 0, binding = 1, std430) buffer Output { uint data[]; } output_buffer; -layout(push_constant, std430) uniform Push { +layout(push_constant, std430) uniform Conversion { uint src_base; uint dst_base; uint width; uint height; - uint depth; - uint surface_z; - uint pitch_bytes; - uint slice_bytes; - uint blocks_per_row; - uint blocks_per_slice; - uint tail_x; - uint tail_y; - uint tail; + uint source_pitch; + uint target_pitch; uint color_transform; uint to_host; } params; diff --git a/src/graphics/host_gpu/shaders/gpu_tiler_common.inc b/src/graphics/host_gpu/shaders/gpu_tiler_common.inc index 4182f9ea6..6b5683e4e 100644 --- a/src/graphics/host_gpu/shaders/gpu_tiler_common.inc +++ b/src/graphics/host_gpu/shaders/gpu_tiler_common.inc @@ -1,66 +1,84 @@ -layout(local_size_x = 8, local_size_y = 8) in; +#extension GL_EXT_shader_8bit_storage : require +#extension GL_EXT_shader_16bit_storage : require + +layout(local_size_x = 64) in; layout(set = 0, binding = 0, std430) readonly buffer Input { uint data[]; } input_buffer; layout(set = 0, binding = 1, std430) buffer Output { uint data[]; } output_buffer; +layout(set = 0, binding = 0, std430) readonly buffer Input8 { uint8_t data[]; } input_buffer8; +layout(set = 0, binding = 1, std430) buffer Output8 { uint8_t data[]; } output_buffer8; +layout(set = 0, binding = 0, std430) readonly buffer Input16 { uint16_t data[]; } input_buffer16; +layout(set = 0, binding = 1, std430) buffer Output16 { uint16_t data[]; } output_buffer16; -layout(set = 0, binding = 2, std140) uniform Push { +struct Mip { uint src_base; uint dst_base; uint width; uint height; - uint depth; - uint surface_z; uint pitch_bytes; uint slice_bytes; + uint tiled_slice_bytes; uint blocks_per_row; - uint blocks_per_slice; + uint surface_z; uint tail_x; uint tail_y; - uint tail; + uint num_texels; +}; + +layout(set = 0, binding = 2, std140) uniform TilingInfo { + uint num_texels; uint color_transform; -} params; + Mip mips[16]; +} info; #include "gpu_tiler_color.inc" void copy_element(uint src, uint dst) { - if ((ELEMENT_BYTES == 4u || ELEMENT_BYTES == 8u) && params.color_transform != 0u) { + if ((ELEMENT_BYTES == 4u || ELEMENT_BYTES == 8u) && info.color_transform != 0u) { uvec2 value = uvec2(input_buffer.data[src >> 2], 0u); if (ELEMENT_BYTES == 8u) value.y = input_buffer.data[(src >> 2) + 1u]; - value = transform_color(value, params.color_transform, TILE == 0u); + value = transform_color(value, info.color_transform, TILE == 0u); output_buffer.data[dst >> 2] = value.x; if (ELEMENT_BYTES == 8u) output_buffer.data[(dst >> 2) + 1u] = value.y; } else if (ELEMENT_BYTES >= 4) { for (uint i = 0; i < ELEMENT_BYTES; i += 4) { output_buffer.data[(dst + i) >> 2] = input_buffer.data[(src + i) >> 2]; } + } else if (ELEMENT_BYTES == 2) { + output_buffer16.data[dst >> 1] = input_buffer16.data[src >> 1]; } else { - uint mask = ELEMENT_BYTES == 1 ? 0xffu : 0xffffu; - uint value = (input_buffer.data[src >> 2] >> ((src & 3u) * 8u)) & mask; - uint shift = (dst & 3u) * 8u; - atomicAnd(output_buffer.data[dst >> 2], ~(mask << shift)); - atomicOr(output_buffer.data[dst >> 2], value << shift); + output_buffer8.data[dst] = input_buffer8.data[src]; } } void main() { - uvec3 p = gl_GlobalInvocationID; - if (p.x >= params.width || p.y >= params.height || p.z >= params.depth) { + uint group = gl_WorkGroupID.y * gl_NumWorkGroups.x + gl_WorkGroupID.x; + // Reject padding groups before multiplying: the active range may approach 4 GiB. + if (group > (info.num_texels - 1u) / 64u) { return; } + uint texel = group * 64u + gl_LocalInvocationID.x; + if (texel >= info.num_texels) { + return; + } + uint mip = 0u; + uint mip_texels = info.mips[mip].num_texels; + while (texel >= mip_texels) { + texel -= mip_texels; + mip_texels = info.mips[++mip].num_texels; + } + Mip params = info.mips[mip]; + uint row = texel / params.width; + uint z = row / params.height; + uvec3 p = uvec3(texel - row * params.width, row - z * params.height, z); uvec3 extent = block_extent(); - uvec3 swizzle = p; - uvec3 block = uvec3(0); - if (params.tail != 0) { - swizzle.xy += uvec2(params.tail_x, params.tail_y); - } else { - block = p / extent; - } + uvec3 block = p / extent; + uvec3 swizzle = p + uvec3(params.tail_x, params.tail_y, params.surface_z); - uint block_index = block.z * params.blocks_per_slice + - block.y * params.blocks_per_row + block.x; - uint tiled = block_index * BLOCK_BYTES + - block_offset(uvec3(swizzle.xy, swizzle.z + params.surface_z)); + uint block_index = block.y * params.blocks_per_row + block.x; + uint tiled = block.z * params.tiled_slice_bytes + block_index * BLOCK_BYTES + + block_offset(swizzle); uint linear = p.z * params.slice_bytes + p.y * params.pitch_bytes + p.x * ELEMENT_BYTES; if (TILE == 0) { diff --git a/src/graphics/host_gpu/shaders/gpu_tiler_demote_d16.comp b/src/graphics/host_gpu/shaders/gpu_tiler_demote_d16.comp index 66638af1c..fb9655fb3 100644 --- a/src/graphics/host_gpu/shaders/gpu_tiler_demote_d16.comp +++ b/src/graphics/host_gpu/shaders/gpu_tiler_demote_d16.comp @@ -6,20 +6,15 @@ layout(local_size_x = 64) in; layout(set = 0, binding = 0, std430) readonly buffer Input { uint data[]; } input_buffer; layout(set = 0, binding = 1, std430) buffer Output { uint data[]; } output_buffer; -layout(push_constant, std430) uniform Push { +layout(push_constant, std430) uniform Conversion { uint src_base; uint dst_base; uint width; uint height; - uint depth; - uint surface_z; - uint pitch_bytes; - uint slice_bytes; - uint blocks_per_row; - uint blocks_per_slice; - uint tail_x; - uint tail_y; - uint tail; + uint source_pitch; + uint target_pitch; + uint color_transform; + uint to_host; } params; uint decode(uint value) { @@ -35,8 +30,8 @@ void main() { if (x >= params.width || y >= params.height) { return; } - uint src = params.src_base + y * params.pitch_bytes + x * 4u; - uint dst = params.dst_base + y * params.slice_bytes + x * 2u; + uint src = params.src_base + y * params.source_pitch + x * 4u; + uint dst = params.dst_base + y * params.target_pitch + x * 2u; uint shift = (dst & 2u) * 8u; uint mask = 0xffffu << shift; uint word = dst >> 2u; diff --git a/src/graphics/host_gpu/shaders/gpu_tiler_promote_d16.comp b/src/graphics/host_gpu/shaders/gpu_tiler_promote_d16.comp index 003857a6f..4f86ac87f 100644 --- a/src/graphics/host_gpu/shaders/gpu_tiler_promote_d16.comp +++ b/src/graphics/host_gpu/shaders/gpu_tiler_promote_d16.comp @@ -6,20 +6,15 @@ layout(local_size_x = 64) in; layout(set = 0, binding = 0, std430) readonly buffer Input { uint data[]; } input_buffer; layout(set = 0, binding = 1, std430) buffer Output { uint data[]; } output_buffer; -layout(push_constant, std430) uniform Push { +layout(push_constant, std430) uniform Conversion { uint src_base; uint dst_base; uint width; uint height; - uint depth; - uint surface_z; - uint pitch_bytes; - uint slice_bytes; - uint blocks_per_row; - uint blocks_per_slice; - uint tail_x; - uint tail_y; - uint tail; + uint source_pitch; + uint target_pitch; + uint color_transform; + uint to_host; } params; void main() { @@ -28,12 +23,12 @@ void main() { if (x >= params.width || y >= params.height) { return; } - uint src = params.src_base + y * params.pitch_bytes + x * 2u; + uint src = params.src_base + y * params.source_pitch + x * 2u; uint packed = input_buffer.data[src >> 2u]; uint value = (packed >> ((src & 2u) * 8u)) & 0xffffu; uint encoded = D32 != 0u ? floatBitsToUint(float(value) / 65535.0) : value * 256u + (value + 128u) / 257u; - uint dst = params.dst_base + y * params.slice_bytes + x * 4u; + uint dst = params.dst_base + y * params.target_pitch + x * 4u; output_buffer.data[dst >> 2u] = encoded; } diff --git a/src/graphics/presentation/videoOut.cpp b/src/graphics/presentation/videoOut.cpp index 51052333e..57e25baab 100644 --- a/src/graphics/presentation/videoOut.cpp +++ b/src/graphics/presentation/videoOut.cpp @@ -566,29 +566,23 @@ Graphics::ImageInfo BufferAttributeGroup::ImageInfo(const VideoOutBuffer& buffer "pixels unchanged (format=0x{:016x}).\n", attribute.pixel_format)); } } - const auto tile_mode = Graphics::Prospero::TileMode::kRenderTarget; - const auto pitch = - Graphics::TileGetTexturePitch(pixel_format.guest_format, attribute.width, tile_mode); - Graphics::TileSizeAlign total {}; - Graphics::TileGetTextureTotalSize(pixel_format.guest_format, attribute.width, attribute.height, - 1, 1, tile_mode, false, total); - if (total.size == 0 || total.align != 65536 || - (buffer.data_address & (total.align - 1u)) != 0) { - EXIT("invalid video-out surface footprint or alignment\n"); - } + const auto tile_mode = Graphics::Prospero::TileMode::kRenderTarget; Graphics::ImageInfo info {}; - info.data = {buffer.data_address, total.size}; + info.data = {buffer.data_address, 0}; info.pixel_format = pixel_format.format; info.guest_format = pixel_format.guest_format; info.type = Graphics::Prospero::ImageType::kColor2D; info.extent = {attribute.width, attribute.height, 1}; info.resources = {1, 1}; - info.pitch = pitch; info.bytes_per_block = pixel_format.bytes_per_element; info.samples = 1; info.tile_mode = tile_mode; info.bgra16 = pixel_format.bgra16; - info.mip_layout[0] = {0, total.size, pitch, attribute.height}; + info.UpdateSize(); + if (info.data.size == 0 || info.tiling.block.block_size != 65536 || + (buffer.data_address & 65535u) != 0) { + EXIT("invalid video-out surface footprint or alignment\n"); + } if (compression != Graphics::VideoOutCompression::Uncompressed) { Graphics::TileSizeAlign dcc_size {}; if (!Graphics::TileGetDccSize(attribute.width, attribute.height, 1, diff --git a/tests/ShaderRecompilerComputeTests.cpp b/tests/ShaderRecompilerComputeTests.cpp index 725103595..3c7f8c089 100644 --- a/tests/ShaderRecompilerComputeTests.cpp +++ b/tests/ShaderRecompilerComputeTests.cpp @@ -593,6 +593,22 @@ using BindingType = TextureCache::BindingType; using ImageDesc = TextureCache::ImageDesc; using ShaderOpcode = ShaderRecompiler::Decoder::Opcode; +ImageInfo MakeTilingImage(Prospero::BufferFormat format, uint32_t width, + uint32_t height, uint32_t levels, uint32_t depth, + Prospero::TileMode tile, uint64_t guest_size, + bool volume) { + ImageInfo info{}; + info.guest_format = format; + info.type = + volume ? Prospero::ImageType::kColor3D : Prospero::ImageType::kColor2D; + info.extent = {width, height, volume ? depth : 1u}; + info.resources = {levels, volume ? 1u : depth}; + info.data.size = guest_size; + info.tile_mode = tile; + info.UpdateSize(); + return info; +} + constexpr u32 InlineU32(u32 value) { return 128u + value; } constexpr u32 Vgpr(u32 reg) { return 256u + reg; } @@ -5244,23 +5260,10 @@ public: "wide/block image readback was not aligned to its texel block"); resources.MapMemory(base, allocation_size); - ImageDesc sampled{}; - sampled.type = BindingType::Texture; - sampled.info.data = {base, sizeof(initial)}; - sampled.info.pixel_format = vk::Format::eR8G8B8A8Srgb; - sampled.info.guest_format = Prospero::BufferFormat::k8_8_8_8Srgb; - sampled.info.type = Prospero::ImageType::kColor2D; - sampled.info.extent = {1, 1, 1}; - sampled.info.resources = {1, 1}; - sampled.info.pitch = 1; - sampled.info.bytes_per_block = 4; - sampled.info.samples = 1; - sampled.info.tile_mode = Prospero::TileMode::kLinear; - sampled.info.mip_layout[0] = {0, 4, 1, 1}; - sampled.view_info.format = sampled.info.pixel_format; - sampled.view_info.type = vk::ImageViewType::e2D; - sampled.view_info.aspect = vk::ImageAspectFlagBits::eColor; - sampled.view_info.usage = vk::ImageUsageFlagBits::eSampled; + auto sampled = + MakeLinearDesc(base, sizeof(initial), vk::Format::eR8G8B8A8Srgb, + Prospero::BufferFormat::k8_8_8_8Srgb, + Prospero::ImageType::kColor2D, {1, 1, 1}, 1, 4, 1); auto &command = scheduler.Current(); auto first_desc = sampled; @@ -5435,8 +5438,9 @@ public: underspecified.info.extent = {2, 1, 1}; underspecified.info.resources = {1, 1}; underspecified.info.pitch = 2; - underspecified.info.mip_layout[0] = {0, 8, 2, 1}; - underspecified.info.mip_layout[1] = {8, 4, 1, 1}; + underspecified.info.mip_layout[0] = {0, 8, 2, 1, 0, 8, 2}; + underspecified.info.mip_layout[1] = {8, 4, 1, 1, 8, 4, 1}; + underspecified.info.linear_slice_stride = 0; underspecified.view_info.format = underspecified.info.pixel_format; underspecified.view_info.level_count = 1; const auto underspecified_id = texture_cache.FindImage(underspecified); @@ -5522,7 +5526,7 @@ public: narrow_target.info.pitch = 128; narrow_target.info.bytes_per_block = 8; narrow_target.info.tile_mode = Prospero::TileMode::kRenderTarget; - narrow_target.info.mip_layout[0] = {0, 0x10000, 128, 1}; + narrow_target.info.UpdateSize(); narrow_target.view_info.format = narrow_target.info.pixel_format; narrow_target.view_info.usage = vk::ImageUsageFlagBits::eColorAttachment; const auto narrow_target_id = texture_cache.FindImage(narrow_target); @@ -5531,6 +5535,7 @@ public: auto wide_target = narrow_target; wide_target.info.extent.width = 9; + wide_target.info.UpdateSize(); const auto wide_target_id = texture_cache.FindImage(wide_target); const auto wide_target_view = texture_cache.FindRenderTarget(wide_target_id, wide_target); @@ -5650,6 +5655,7 @@ public: } } } + desc.info.UpdateSize(); const auto id = texture_cache.FindImage(desc); Require(name, "distinct mip-layout backing", id && id != previous, "different guest mip layouts reused the same native image"); @@ -5693,6 +5699,7 @@ public: {256, 256, 1}, 1, 16, 1); raw_alias.type = BindingType::RenderTarget; raw_alias.info.tile_mode = Prospero::TileMode::kRenderTarget; + raw_alias.info.UpdateSize(); raw_alias.view_info.usage = vk::ImageUsageFlagBits::eColorAttachment; const auto raw_alias_id = texture_cache.FindImage(raw_alias); (void)texture_cache.FindRenderTarget(raw_alias_id, raw_alias); @@ -5737,6 +5744,7 @@ public: bc5_expected.insert(bc5_expected.end(), bc5_guest.begin() + word, bc5_guest.begin() + word + 4); } + bc5_alias.info.UpdateSize(); const auto bc5_alias_id = texture_cache.FindImage(bc5_alias); Require(name, "BC5 alias owner separation", bc5_alias_id != raw_alias_id && @@ -5784,6 +5792,7 @@ public: tiled_array.info.mip_layout[level] = { mip.offset, mip.size * tiled_layers, mip.padded_width, mip.padded_height}; } + tiled_array.info.UpdateSize(); std::memset(memory + tiled_array_offset, 0, surface.total_size); const auto tiled_array_id = texture_cache.FindImage(tiled_array); (void)texture_cache.FindTexture(tiled_array_id, tiled_array); @@ -5879,6 +5888,7 @@ public: mip_sizes[level].offset, mip_sizes[level].size, mip_padded[level].width, mip_padded[level].height}; } + mip_desc.info.UpdateSize(); const auto mip_image = texture_cache.FindImage(mip_desc); (void)texture_cache.FindTexture(mip_image, mip_desc); texture_cache.MarkGpuWritten(mip_image); @@ -5959,7 +5969,7 @@ public: vk::Format::eR32Uint, Prospero::BufferFormat::k32UInt, Prospero::ImageType::kColor2D, {1, 1, 1}, 2, 4, 1); array_desc.info.pitch = volume_pitch; - array_desc.info.mip_layout[0].pitch = volume_pitch; + array_desc.info.UpdateSize(); const auto array_image = texture_cache.FindImage(array_desc); (void)texture_cache.FindTexture(array_image, array_desc); texture_cache.MarkGpuWritten(array_image); @@ -5967,6 +5977,7 @@ public: volume_desc.info.type = Prospero::ImageType::kColor3D; volume_desc.info.extent = {1, 1, 2}; volume_desc.info.resources.layers = 1; + volume_desc.info.UpdateSize(); volume_desc.view_info.type = vk::ImageViewType::e3D; volume_desc.view_info.layer_count = 1; const auto volume_image = texture_cache.FindImage(volume_desc); @@ -6031,9 +6042,11 @@ public: Prospero::ImageType::kColor2D, {2, 1, 1}, 1, sizeof(float), 1); depth_containment.type = BindingType::DepthTarget; depth_containment.info.resources.levels = 2; - depth_containment.info.mip_layout[0] = {0, 2 * sizeof(float), 2, 1}; - depth_containment.info.mip_layout[1] = {2 * sizeof(float), sizeof(float), - 1, 1}; + depth_containment.info.mip_layout[0] = { + 0, 2 * sizeof(float), 2, 1, 0, 2 * sizeof(float), 2}; + depth_containment.info.mip_layout[1] = { + 2 * sizeof(float), sizeof(float), 1, 1, 2 * sizeof(float), sizeof(float), 1}; + depth_containment.info.linear_slice_stride = 0; depth_containment.view_info.format = vk::Format::eD32Sfloat; depth_containment.view_info.aspect = vk::ImageAspectFlagBits::eDepth; depth_containment.view_info.usage = @@ -6218,6 +6231,7 @@ public: Prospero::ImageType::kColor2D, {128, 128, 1}, htile_layers, 4, 1); htile_depth.type = BindingType::DepthTarget; htile_depth.info.tile_mode = Prospero::TileMode::kDepth; + htile_depth.info.UpdateSize(); htile_depth.info.metadata.kind = ImageMetadataKind::Htile; htile_depth.info.metadata.range = {base + 0x2a00000, htile_layers * 0x8000}; htile_depth.view_info.aspect = vk::ImageAspectFlagBits::eDepth; @@ -6559,7 +6573,7 @@ public: first_stencil_association && first_stencil_association != ms_depth_image && texture_cache.GetImage(first_stencil_association) - .info.pixel_format == vk::Format::eUndefined && + .info.pixel_format == vk::Format::eS8Uint && texture_cache.GetImage(first_stencil_association).backing.image == nullptr && texture_cache.GetImage(first_stencil_association).depth_id == @@ -6856,8 +6870,9 @@ public: {13, 11, 1}, 2, 4, 1); stencil_clear_desc.info.stencil = {base + 0x27e2000, 0x1000}; stencil_clear_desc.info.resources.levels = 2; - stencil_clear_desc.info.mip_layout[0] = {0, 0x800, 13, 11}; - stencil_clear_desc.info.mip_layout[1] = {0x800, 0x800, 6, 5}; + stencil_clear_desc.info.mip_layout[0] = {0, 0x800, 13, 11, 0, 0x400, 13}; + stencil_clear_desc.info.mip_layout[1] = {0x800, 0x800, 6, 5, 0x800, 0x400, 6}; + stencil_clear_desc.info.linear_slice_stride = 0x400; stencil_clear_desc.view_info.aspect = vk::ImageAspectFlagBits::eDepth | vk::ImageAspectFlagBits::eStencil; const auto stencil_clear_id = texture_cache.FindImage(stencil_clear_desc); @@ -7179,7 +7194,7 @@ public: dcc_array.type = BindingType::RenderTarget; dcc_array.info.tile_mode = Prospero::TileMode::kRenderTarget; dcc_array.info.pitch = TileGetRenderTargetPitch(128, 8); - dcc_array.info.mip_layout[0].pitch = dcc_array.info.pitch; + dcc_array.info.UpdateSize(); dcc_array.info.metadata.kind = ImageMetadataKind::Dcc; dcc_array.info.metadata.range = {alias_dcc_address, 0x2000}; dcc_array.view_info.usage = vk::ImageUsageFlagBits::eColorAttachment; @@ -7193,7 +7208,7 @@ public: auto dcc_alias = dcc_array; dcc_alias.info.data = {dcc_array.info.data.address + 0x10000, 0x10000}; dcc_alias.info.resources.layers = 1; - dcc_alias.info.mip_layout[0].size = 0x10000; + dcc_alias.info.UpdateSize(); dcc_alias.info.metadata.range = {alias_dcc_address + 0x1000, 0x1000}; dcc_alias.view_info.type = vk::ImageViewType::e2D; dcc_alias.view_info.layer_count = 1; @@ -7444,30 +7459,34 @@ public: 0.5f, 0.625f, 0.75f, 0.875f, 1.0f, 0.0625f}; std::memset(memory + layered_offset, 0, layered_guest_size); - const auto layered_layout = TextureCalcUploadLayout( + const auto layered_layout = MakeTilingImage( Prospero::BufferFormat::k32Float, 2, 2, 2, 2, - Prospero::TileMode::kLinear, layered_guest_size, false, - "UnifiedTextureCacheFlow"); - const auto layered_upload_regions = - TextureBuildImageCopies(layered_layout); - Require(name, "layered guest layout", layered_upload_regions.size() == 4, - "unexpected layered/mipped upload-region count"); + Prospero::TileMode::kLinear, layered_guest_size, false); + const auto layered_upload_regions = layered_layout.BufferCopies(); + Require(name, "layered guest layout", layered_upload_regions.size() == 2, + "linear array upload did not batch each mip"); for (const auto ®ion : layered_upload_regions) { - for (uint32_t y = 0; y < region.imageExtent.height; y++) { - for (uint32_t x = 0; x < region.imageExtent.width; x++) { - const uint32_t logical = - region.imageSubresource.mipLevel == 0 - ? region.imageSubresource.baseArrayLayer * 4 + y * 2 + x - : 8 + region.imageSubresource.baseArrayLayer; - const uint64_t byte_offset = - region.bufferOffset + - (static_cast(y) * region.bufferRowLength + x) * - sizeof(float); - Require(name, "layered guest layout bounds", - byte_offset + sizeof(float) <= layered_guest_size, - "layered guest texel lies outside its backing range"); - std::memcpy(memory + layered_offset + byte_offset, - &layered_values[logical], sizeof(float)); + for (uint32_t layer = 0; layer < region.imageSubresource.layerCount; + ++layer) { + for (uint32_t y = 0; y < region.imageExtent.height; ++y) { + for (uint32_t x = 0; x < region.imageExtent.width; ++x) { + const uint32_t absolute_layer = + region.imageSubresource.baseArrayLayer + layer; + const uint32_t logical = region.imageSubresource.mipLevel == 0 + ? absolute_layer * 4 + y * 2 + x + : 8 + absolute_layer; + const uint64_t byte_offset = + region.bufferOffset + + ((uint64_t{layer} * region.bufferImageHeight + y) * + region.bufferRowLength + + x) * + sizeof(float); + Require(name, "layered guest layout bounds", + byte_offset + sizeof(float) <= layered_guest_size, + "layered guest texel lies outside its backing range"); + std::memcpy(memory + layered_offset + byte_offset, + &layered_values[logical], sizeof(float)); + } } } } @@ -7479,6 +7498,7 @@ public: layered_color.info.mip_layout[0] = {0, 32, 2, 2}; layered_color.info.mip_layout[1] = {32, 8, 1, 1}; layered_color.view_info.level_count = 2; + layered_color.info.UpdateSize(); const auto layered_color_image = texture_cache.FindImage(layered_color); (void)texture_cache.FindTexture(layered_color_image, layered_color); texture_cache.MarkGpuWritten(layered_color_image); @@ -7494,17 +7514,19 @@ public: layered_depth.view_info.aspect = vk::ImageAspectFlagBits::eDepth; layered_depth.view_info.usage = vk::ImageUsageFlagBits::eDepthStencilAttachment; + layered_depth.info.UpdateSize(); const auto layered_depth_image = texture_cache.FindImage(layered_depth); const auto &layered_native = texture_cache.GetImage(layered_depth_image); - Require(name, "layered mipped depth alias", - layered_depth_image != layered_color_image && - layered_native.backing.layers == 4 && - layered_native.backing.mip_levels == 1 && - layered_native.info.resources == - layered_depth.info.resources && - layered_native.info.mip_layout == layered_depth.info.mip_layout && - layered_native.IsGpuModified(), - "depth/color conversion mixed the source mip count with the requested layout"); + Require( + name, "layered mipped depth alias", + layered_depth_image != layered_color_image && + layered_native.backing.layers == 4 && + layered_native.backing.mip_levels == 1 && + layered_native.info.resources == layered_depth.info.resources && + layered_native.info.mip_layout == layered_depth.info.mip_layout && + layered_native.IsGpuModified(), + "depth/color conversion mixed the source mip count with the " + "requested layout"); const volatile auto layered_guest_byte = *reinterpret_cast(memory + layered_offset); (void)layered_guest_byte; @@ -8415,8 +8437,9 @@ public: for (uint32_t index = 0; index < linear_depth_words; index++) { linear_depth_guest[index] = 0x51000000u + index; } - constexpr std::array linear_stencil_guest{ - 0x91, 0x82, 0x73, 0x64, 0x55, 0x46, 0x37, 0x28}; + constexpr std::array linear_stencil_guest{ + 0x91, 0x82, 0x73, 0x64, 0x55, 0x46, 0x37, 0x28, + 0x19, 0x2a, 0x3b, 0x4c, 0x5d, 0x6e, 0x7f, 0x80}; std::memcpy(memory + linear_depth_offset, linear_depth_guest.data(), sizeof(linear_depth_guest)); std::memcpy(memory + linear_stencil_offset, linear_stencil_guest.data(), @@ -8434,6 +8457,7 @@ public: linear_depth_desc.info.mip_layout[0] = {0, sizeof(linear_depth_guest), linear_depth_pitch, linear_depth_height}; + linear_depth_desc.info.UpdateSize(); linear_depth_desc.view_info.aspect = vk::ImageAspectFlagBits::eDepth | vk::ImageAspectFlagBits::eStencil; linear_depth_desc.view_info.usage = @@ -8535,6 +8559,7 @@ public: tiled_depth_desc.view_info.aspect = vk::ImageAspectFlagBits::eDepth; tiled_depth_desc.view_info.usage = vk::ImageUsageFlagBits::eDepthStencilAttachment; + tiled_depth_desc.info.UpdateSize(); const auto tiled_depth_image = texture_cache.FindImage(tiled_depth_desc); Require(name, "tiled depth native clear", texture_cache.ClearImageFromBuffer( @@ -8584,28 +8609,18 @@ public: TileGetBlockLayout(TileBlockFamily::Depth64KB, sizeof(uint32_t), depth_block), "failed to describe the tiled depth test surface"); - std::array tiled_depth_tiles{}; - for (uint32_t layer = 0; layer < tiled_depth_layers; layer++) { - const uint64_t offset = tiled_depth_slice * layer; - tiled_depth_tiles[layer] = {depth_block.family, - depth_block.bytes_per_element, - offset, - tiled_depth_slice, - offset, - tiled_depth_slice, - 0, - tiled_depth_width, - tiled_depth_height, - 1, - tiled_depth_pitch}; - tiled_depth_tiles[layer].surface_z = layer; - } + auto tiled_depth_info = + MakeTilingImage(Prospero::BufferFormat::k32Float, tiled_depth_width, + tiled_depth_height, 1, tiled_depth_layers, + Prospero::TileMode::kDepth, tiled_depth_size, false); + tiled_depth_info.linear_slice_stride = tiled_depth_slice; + tiled_depth_info.mip_layout[0].linear_size = tiled_depth_slice; + tiled_depth_info.mip_layout[0].linear_pitch = tiled_depth_pitch; auto tiled_depth_input = CreateHostBuffer(name, tiled_depth_size, AllFlags, tiled_depth_after); auto tiled_depth_linear = TextureCacheTestAccess::Tiler(texture_cache) - .Detile(tiled_depth_input.buffer, 0, tiled_depth_size, - tiled_depth_size, tiled_depth_tiles); + .Detile(tiled_depth_input.buffer, 0, tiled_depth_info); auto tiled_depth_output = CreateHostBuffer(name, tiled_depth_size, AllFlags, std::vector(tiled_depth_guest.size(), 0)); @@ -8669,6 +8684,7 @@ public: vk::ImageAspectFlagBits::eDepth | vk::ImageAspectFlagBits::eStencil; tiled_d16_desc.view_info.usage = vk::ImageUsageFlagBits::eDepthStencilAttachment; + tiled_d16_desc.info.UpdateSize(); const auto tiled_d16_image = texture_cache.FindImage(tiled_d16_desc); Require(name, "tiled D16 native clear", texture_cache.ClearImageFromBuffer( @@ -8701,32 +8717,22 @@ public: sizeof(uint32_t)); std::memcpy(tiled_d16_words.data(), tiled_d16_after.data(), tiled_depth_size); - std::array tiled_d16_tiles{}; Require(name, "tiled D16 block layout", TileGetBlockLayout(TileBlockFamily::Depth64KB, sizeof(uint16_t), depth_block), "failed to describe the tiled D16 test surface"); - for (uint32_t layer = 0; layer < tiled_depth_layers; layer++) { - const uint64_t offset = tiled_depth_slice * layer; - tiled_d16_tiles[layer] = {depth_block.family, - depth_block.bytes_per_element, - offset, - tiled_depth_slice, - offset, - tiled_depth_slice, - 0, - tiled_depth_width, - tiled_depth_height, - 1, - tiled_depth_pitch}; - tiled_d16_tiles[layer].surface_z = layer; - } + auto tiled_d16_info = + MakeTilingImage(Prospero::BufferFormat::k16UNorm, tiled_depth_width, + tiled_depth_height, 1, tiled_depth_layers, + Prospero::TileMode::kDepth, tiled_depth_size, false); + tiled_d16_info.linear_slice_stride = tiled_depth_slice; + tiled_d16_info.mip_layout[0].linear_size = tiled_depth_slice; + tiled_d16_info.mip_layout[0].linear_pitch = tiled_depth_pitch; auto tiled_d16_input = CreateHostBuffer(name, tiled_depth_size, AllFlags, tiled_d16_words); auto tiled_d16_linear = TextureCacheTestAccess::Tiler(texture_cache) - .Detile(tiled_d16_input.buffer, 0, tiled_depth_size, - tiled_depth_size, tiled_d16_tiles); + .Detile(tiled_d16_input.buffer, 0, tiled_d16_info); auto tiled_d16_output = CreateHostBuffer(name, tiled_depth_size, AllFlags, std::vector(tiled_d16_words.size(), 0)); @@ -8985,6 +8991,7 @@ public: {tile_alias_extent, tile_alias_extent, 1}, 1, 4, 1); render_target_alias.type = BindingType::Storage; render_target_alias.info.tile_mode = Prospero::TileMode::kRenderTarget; + render_target_alias.info.UpdateSize(); render_target_alias.view_info.usage = vk::ImageUsageFlagBits::eStorage; const auto render_target_alias_image = texture_cache.FindImage(render_target_alias); @@ -8992,6 +8999,7 @@ public: auto standard_4kb_alias = render_target_alias; standard_4kb_alias.type = BindingType::Texture; standard_4kb_alias.info.tile_mode = Prospero::TileMode::kStandard4KB; + standard_4kb_alias.info.UpdateSize(); standard_4kb_alias.view_info.usage = vk::ImageUsageFlagBits::eSampled; const auto standard_4kb_alias_image = texture_cache.FindImage(standard_4kb_alias); @@ -9145,7 +9153,7 @@ public: desc.info.samples = 1; desc.info.tile_mode = tile; desc.info.bgra16 = true; - desc.info.mip_layout[0] = {mip.offset, mip.size, pitch, 1}; + desc.info.UpdateSize(); desc.view_info.format = desc.info.pixel_format; desc.view_info.type = vk::ImageViewType::e2D; desc.view_info.aspect = vk::ImageAspectFlagBits::eColor; @@ -9347,7 +9355,10 @@ public: if (width == 1 && height == 1) { Require(name, "single-texel tiled layout", - color.desc.info.mip_layout[0] == ImageMipInfo{0, 0x10000, 128, 64}, + color.desc.info.mip_layout[0].offset == 0 && + color.desc.info.mip_layout[0].size == 0x10000 && + color.desc.info.mip_layout[0].pitch == 128 && + color.desc.info.mip_layout[0].height == 64, "the render target lost the expected padded block height"); } @@ -9844,6 +9855,7 @@ public: auto expanded = storage_desc; expanded.info.extent.depth = 56; expanded.info.data.size = allocation_size; + expanded.info.UpdateSize(); const auto expanded_id = texture_cache.FindImage(expanded); Require(name, "3D storage depth expansion", expanded_id != color.image_id && @@ -9902,9 +9914,24 @@ public: desc.info.samples = samples; desc.info.tile_mode = Prospero::TileMode::kLinear; desc.info.mip_layout[0] = {0, size, extent.width, extent.height}; + desc.info.tiling.texel_width = desc.info.IsBlock() ? 4u : 1u; + desc.info.tiling.texel_height = desc.info.IsBlock() ? 4u : 1u; + desc.info.tiling.block.bytes_per_element = bytes_per_block; + desc.info.linear_slice_stride = + size / (type == Prospero::ImageType::kColor3D ? extent.depth : layers); + desc.info.mip_layout[0].pitch = + (extent.width + desc.info.tiling.texel_width - 1u) / + desc.info.tiling.texel_width; + desc.info.mip_layout[0].height = + (extent.height + desc.info.tiling.texel_height - 1u) / + desc.info.tiling.texel_height; + desc.info.mip_layout[0].linear_pitch = desc.info.mip_layout[0].pitch; + desc.info.mip_layout[0].linear_size = desc.info.linear_slice_stride; desc.view_info.format = format; - desc.view_info.type = type == Prospero::ImageType::kColor3D ? vk::ImageViewType::e3D - : layers > 1 ? vk::ImageViewType::e2DArray : vk::ImageViewType::e2D; + desc.view_info.type = type == Prospero::ImageType::kColor3D + ? vk::ImageViewType::e3D + : layers > 1 ? vk::ImageViewType::e2DArray + : vk::ImageViewType::e2D; desc.view_info.aspect = vk::ImageAspectFlagBits::eColor; desc.view_info.layer_count = layers; desc.view_info.usage = vk::ImageUsageFlagBits::eSampled; @@ -10859,7 +10886,7 @@ public: metadata_alias.info.extent = {4096, 1, 1}; metadata_alias.info.pitch = 4096; metadata_alias.info.bytes_per_block = sizeof(uint32_t); - metadata_alias.info.mip_layout[0] = {0, 0x4000, 4096, 1}; + metadata_alias.info.UpdateSize(); metadata_alias.view_info.format = vk::Format::eR32Uint; const auto metadata_alias_id = texture_cache.FindImage(metadata_alias); vk::ClearValue alias_paint{}; @@ -10917,7 +10944,7 @@ public: previous_depth.info.bytes_per_block = 4; previous_depth.info.samples = 1; previous_depth.info.tile_mode = Prospero::TileMode::kLinear; - previous_depth.info.mip_layout[0] = {0, 0x10000, 128, 128}; + previous_depth.info.UpdateSize(); previous_depth.info.metadata.kind = ImageMetadataKind::Htile; previous_depth.info.metadata.range = {dcc_address, metadata_size}; previous_depth.info.htile_clear_mask = 0; @@ -11103,13 +11130,17 @@ public: target.export_mapping == Prospero::ColorMappingAbgr, "shared format resolution changed the existing 1555 render target"); for (const auto native : {target, TextureGetRenderTargetFormat( - Prospero::ChannelLayout::k8, Prospero::ChannelType::kSNorm, - Prospero::ChannelOrder::kStandard)}) { - const auto upload = TextureCalcUploadLayout(native.guest_format, 4, 1, 1, 1, - Prospero::TileMode::kLinear, 256, false, name); - Require(name, "render-target guest upload layout", - upload.pitch * native.bytes_per_element == 256 && upload.mips[0].size == 256, - "retaining the render-target format lost its physical byte width"); + Prospero::ChannelLayout::k8, + Prospero::ChannelType::kSNorm, + Prospero::ChannelOrder::kStandard)}) { + const auto upload = + MakeTilingImage(native.guest_format, 4, 1, 1, 1, + Prospero::TileMode::kLinear, 256, false); + Require( + name, "render-target guest upload layout", + upload.pitch * native.bytes_per_element == 256 && + upload.mip_layout[0].linear_size == 256, + "retaining the render-target format lost its physical byte width"); } EnsureRuntimeContext(); int64_t direct_offset = -1; @@ -12232,7 +12263,7 @@ public: auto depth_desc = MakeLinearDesc(alias_address, 256, vk::Format::eD16Unorm, format, Prospero::ImageType::kColor2D, {1, 1, 1}, 1, 2, 1); depth_desc.info.pitch = 128; - depth_desc.info.mip_layout[0] = {0, 256, 128, 1}; + depth_desc.info.UpdateSize(); depth_desc.type = BindingType::DepthTarget; depth_desc.view_info.aspect = vk::ImageAspectFlagBits::eDepth; depth_desc.view_info.usage = vk::ImageUsageFlagBits::eDepthStencilAttachment; @@ -12312,7 +12343,7 @@ public: Prospero::BufferFormat::k8UNorm, Prospero::ImageType::kColor2D, {1, 1, 1}, 6, 1, 1); array_desc.info.pitch = 256; - array_desc.info.mip_layout[0] = {0, 1536, 256, 1}; + array_desc.info.UpdateSize(); auto volume_desc = array_desc; volume_desc.info.type = Prospero::ImageType::kColor3D; volume_desc.info.data.size = 256; @@ -12448,7 +12479,10 @@ public: desc.info.bytes_per_block = 4; desc.info.samples = 1; desc.info.tile_mode = linear; - desc.info.mip_layout[0] = {0, size, extent.width, extent.height}; + desc.info.mip_layout[0] = { + 0, size, extent.width, extent.height, 0, size, extent.width}; + desc.info.tiling.block.bytes_per_element = 4; + desc.info.linear_slice_stride = size; desc.view_info.format = desc.info.pixel_format; desc.view_info.type = vk::ImageViewType::e2D; desc.view_info.aspect = vk::ImageAspectFlagBits::eColor; @@ -13165,8 +13199,9 @@ public: split_desc.info.bytes_per_block = 4; split_desc.info.samples = 1; split_desc.info.tile_mode = linear; - split_desc.info.mip_layout[0] = {0, 8, 2, 1}; - split_desc.info.mip_layout[1] = {8, 4, 1, 1}; + split_desc.info.mip_layout[0] = {0, 8, 2, 1, 0, 8, 2}; + split_desc.info.mip_layout[1] = {8, 4, 1, 1, 8, 4, 1}; + split_desc.info.tiling.block.bytes_per_element = 4; split_desc.view_info.format = split_desc.info.pixel_format; split_desc.view_info.type = vk::ImageViewType::e2D; split_desc.view_info.aspect = vk::ImageAspectFlagBits::eColor; @@ -14106,9 +14141,10 @@ public: video_parent.info.pixel_format = vk::Format::eR8G8B8A8Srgb; video_parent.info.guest_format = Prospero::BufferFormat::k8_8_8_8Srgb; video_parent.info.resources.levels = 2; - video_parent.info.mip_layout[0] = {0, target_mip_size, 2, 2}; - video_parent.info.mip_layout[1] = {target_mip_size, target_mip_size, 1, - 1}; + video_parent.info.mip_layout[0] = {0, target_mip_size, 2, 2, + 0, target_mip_size, 2}; + video_parent.info.mip_layout[1] = {target_mip_size, target_mip_size, 1, 1, + target_mip_size, target_mip_size, 1}; video_parent.view_info.format = video_parent.info.pixel_format; video_parent.view_info.usage = vk::ImageUsageFlagBits::eSampled; const auto video_parent_id = texture_cache.FindImage(video_parent); @@ -14482,7 +14518,7 @@ public: depth.info.bytes_per_block = 4; depth.info.samples = 1; depth.info.tile_mode = linear; - depth.info.mip_layout[0] = {0, depth.info.data.size, 1, 1}; + depth.info.UpdateSize(); depth.view_info.format = depth.info.pixel_format; depth.view_info.type = vk::ImageViewType::e2D; depth.view_info.aspect = vk::ImageAspectFlagBits::eDepth; @@ -14508,18 +14544,17 @@ public: target_parent.info.resources.levels = 2; // PS5 linear rows align to 256 bytes; smaller mips precede larger ones. target_parent.info.pitch = 64; - target_parent.info.mip_layout[0] = {512, 1024, 64, 4}; - target_parent.info.mip_layout[1] = {0, 512, 64, 2}; + target_parent.info.UpdateSize(); target_parent.view_info.usage = vk::ImageUsageFlagBits::eColorAttachment; auto target_base_subresource = make_target_desc(base + 512, 1024, {4, 4, 1}); target_base_subresource.info.pitch = 64; - target_base_subresource.info.mip_layout[0] = {0, 1024, 64, 4}; + target_base_subresource.info.UpdateSize(); const auto target_base_subresource_id = texture_cache.FindImage(target_base_subresource); auto target_subresource = make_target_desc(base, 512, {2, 2, 1}); target_subresource.info.pitch = 64; - target_subresource.info.mip_layout[0] = {0, 512, 64, 2}; + target_subresource.info.UpdateSize(); const auto target_subresource_id = texture_cache.FindImage(target_subresource); const auto *target_subresource_owner = @@ -14716,13 +14751,13 @@ public: const auto sampled_stencil_id = prepared.images[0].image_id; Require(name, "first stencil discovery", prepared.images.size() == 1 && - sampled_stencil_id != depth_id && + sampled_stencil_id == depth_id && prepared.images[0].image_view == nullptr && texture_cache.GetImage(sampled_stencil_id).binding.is_bound && !texture_cache.GetImage(stencil_proxy_id).binding.is_bound && !texture_cache.GetImage(stencil_proxy_id).binding.is_target, - "the first sampled-stencil lookup did not remain an ordinary " - "discovery before final depth acquisition"); + "the first sampled-stencil lookup did not use the canonical " + "depth owner without acquiring the stencil proxy"); context.GetRenderExecutor().RebindImages(prepared); Require(name, "first stencil acquisition", @@ -14738,7 +14773,7 @@ public: executor, scheduler.Current(), &no_reassociated_color, 0, reassociated_depth); Require(name, "existing stencil association selection", - texture_cache.GetImage(sampled_stencil_id).depth_id == + texture_cache.GetImage(stencil_proxy_id).depth_id == reassociated_depth.image_id, "final depth acquisition did not associate the existing image " "at the stencil guest address"); @@ -14751,7 +14786,7 @@ public: redirected.images[0].image_id == depth_id && redirected.images[0].image_view == nullptr && texture_cache.GetImage(depth_id).binding.is_bound && - !texture_cache.GetImage(sampled_stencil_id).binding.is_bound, + !texture_cache.GetImage(stencil_proxy_id).binding.is_bound, "the established stencil association did not redirect the next " "discovery to the depth owner"); context.GetRenderExecutor().RebindImages(redirected); @@ -14872,7 +14907,7 @@ public: plane_depth.info.extent = {3, 2, 1}; plane_depth.info.pitch = TileGetDepthPitch(3, sizeof(uint32_t)); plane_depth.info.tile_mode = Prospero::TileMode::kDepth; - plane_depth.info.mip_layout[0] = {0, plane_size, plane_depth.info.pitch, 2}; + plane_depth.info.UpdateSize(); const auto plane_depth_id = texture_cache.FindImage(plane_depth); auto sampled_plane = plane_depth; sampled_plane.type = BindingType::Texture; @@ -14883,7 +14918,7 @@ public: sampled_plane.info.guest_format = Prospero::BufferFormat::k8UInt; sampled_plane.info.bytes_per_block = 1; sampled_plane.info.pitch = TileGetDepthPitch(3, 1); - sampled_plane.info.mip_layout[0] = {0, plane_size, sampled_plane.info.pitch, 2}; + sampled_plane.info.UpdateSize(); sampled_plane.view_info.format = vk::Format::eR8Uint; sampled_plane.view_info.aspect = vk::ImageAspectFlagBits::eColor; sampled_plane.view_info.usage = vk::ImageUsageFlagBits::eSampled; @@ -15884,7 +15919,7 @@ public: depth.desc.info.pitch = 64; depth.desc.info.bytes_per_block = 4; depth.desc.info.tile_mode = Prospero::TileMode::kLinear; - depth.desc.info.mip_layout[0] = {0, 256 * extent, 64, extent}; + depth.desc.info.UpdateSize(); depth.desc.view_info.format = depth.desc.info.pixel_format; depth.desc.view_info.aspect = vk::ImageAspectFlagBits::eDepth; depth.desc.view_info.usage = vk::ImageUsageFlagBits::eDepthStencilAttachment; @@ -15962,7 +15997,10 @@ public: RenderColorInfo color{}; color.desc.type = BindingType::RenderTarget; color.desc.info.data = {depth_address + 0x10000, extent * extent * 16}; - color.desc.info.mip_layout[0] = {0, color.desc.info.data.size, extent, extent}; + color.desc.info.mip_layout[0] = { + 0, color.desc.info.data.size, extent, extent, 0, color.desc.info.data.size, extent}; + color.desc.info.tiling.block.bytes_per_element = 16; + color.desc.info.linear_slice_stride = color.desc.info.data.size; color.desc.info.pixel_format = vk::Format::eR32G32B32A32Sfloat; color.desc.info.guest_format = Prospero::BufferFormat::k32_32_32_32Float; color.desc.info.extent = {extent, extent, 1}; @@ -17176,18 +17214,23 @@ public: const auto gpu_detile = [&](const std::vector &tiled, std::vector *linear, uint64_t tiled_capacity, uint64_t linear_capacity, - std::span infos) { + const ImageInfo &info, TileManager *manager = nullptr, + uint64_t source_offset = 0) { const uint64_t padded_tiled = align_dword(tiled_capacity); const uint64_t padded_linear = align_dword(linear_capacity); - auto input = CreateHostBuffer(name, padded_tiled, AllFlags, - to_dwords(tiled, padded_tiled)); + std::vector source(source_offset + padded_tiled, 0x7b); + std::copy(tiled.begin(), tiled.end(), source.begin() + source_offset); + auto input = CreateHostBuffer(name, source.size(), AllFlags, + to_dwords(source, source.size())); auto output = CreateHostBuffer( name, padded_linear, AllFlags, std::vector(static_cast(padded_linear / sizeof(u32)), - 0xabababab)); - const auto result = tile_manager.Detile(input.buffer, 0, padded_tiled, - padded_linear, infos); - const vk::BufferCopy copy{result.offset, 0, padded_linear}; + 0)); + const auto result = (manager ? *manager : tile_manager) + .Detile(input.buffer, source_offset, info); + Require(name, "detile capacity", result.size <= padded_linear, + "canonical transfer size exceeded the fixture output"); + const vk::BufferCopy copy{result.offset, 0, align_dword(result.size)}; scheduler.Current().Handle().copyBuffer(result.buffer, output.buffer, 1, ©); host_barrier(output.buffer, padded_linear, @@ -17201,22 +17244,34 @@ public: const auto gpu_tile = [&](const std::vector &linear, std::vector *tiled, uint64_t tiled_capacity, uint64_t linear_capacity, - std::span infos) { + const ImageInfo &info, + TileManager *manager = nullptr, + uint32_t levels = 0, uint64_t source_offset = 0, + uint64_t target_offset = 0) { const uint64_t padded_tiled = align_dword(tiled_capacity); const uint64_t padded_linear = align_dword(linear_capacity); - auto input = CreateHostBuffer(name, padded_linear, AllFlags, - to_dwords(linear, padded_linear)); + std::vector source(source_offset + padded_linear, 0x7b); + std::copy(linear.begin(), linear.end(), source.begin() + source_offset); + auto input = CreateHostBuffer(name, source.size(), AllFlags, + to_dwords(source, source.size())); auto output = CreateHostBuffer( - name, padded_tiled, AllFlags, - std::vector(static_cast(padded_tiled / sizeof(u32)), - 0xabababab)); - tile_manager.Tile(input.buffer, 0, padded_linear, output.buffer, 0, - padded_tiled, infos); - host_barrier(output.buffer, padded_tiled, + name, target_offset + padded_tiled, AllFlags, + std::vector( + static_cast((target_offset + padded_tiled) / sizeof(u32)), + 0xabababab)); + (manager ? *manager : tile_manager) + .Tile(input.buffer, source_offset, padded_linear, output.buffer, + target_offset, padded_tiled, info, levels); + host_barrier(output.buffer, target_offset + padded_tiled, vk::PipelineStageFlagBits::eComputeShader, vk::AccessFlagBits::eShaderWrite); scheduler.Finish(); - *tiled = read_bytes(output, tiled_capacity); + const auto observed = read_bytes(output, target_offset + tiled_capacity); + Require(name, "nonzero transfer target base", + std::all_of(observed.begin(), observed.begin() + target_offset, + [](uint8_t value) { return value == 0xab; }), + "tiling changed bytes preceding its target offset"); + tiled->assign(observed.begin() + target_offset, observed.end()); DestroyBuffer(&output); DestroyBuffer(&input); }; @@ -17337,59 +17392,105 @@ public: }; auto convert_reference = [&](bool to_tiled, std::vector *dst, const std::vector &src, - const GpuTileInfo &info) { - TileBlockLayout block{}; - Require(name, "reference layout", - TileGetBlockLayout(info.family, info.bytes_per_element, block), - "CPU reference rejected GPU tile info"); - const u32 tiled_width = - info.tiled_width != 0 ? info.tiled_width : info.pitch; - const u32 tiled_height = - info.tiled_height != 0 ? info.tiled_height : info.height; + const ImageInfo &info, uint32_t level = 0) { + const auto &block = info.tiling.block; + const auto &mip = info.mip_layout[level]; + const auto active = info.MipExtent(level); + const uint32_t width = (active.width + info.tiling.texel_width - 1u) / + info.tiling.texel_width; + const uint32_t height = (active.height + info.tiling.texel_height - 1u) / + info.tiling.texel_height; const uint64_t columns = - (tiled_width + block.block_width - 1u) / block.block_width; - const uint64_t rows = - (tiled_height + block.block_height - 1u) / block.block_height; + (mip.pitch + block.block_width - 1u) / block.block_width; const uint64_t slice = info.linear_slice_stride != 0 ? info.linear_slice_stride - : static_cast(info.pitch) * - info.height * info.bytes_per_element; - for (u32 z = 0; z < info.depth; ++z) { - for (u32 y = 0; y < info.height; ++y) { - for (u32 x = 0; x < info.width; ++x) { - const u32 bx = info.tail ? 0 : x / block.block_width; - const u32 by = info.tail ? 0 : y / block.block_height; - const u32 bz = info.tail ? 0 : z / block.block_depth; - const u32 lx = info.tail ? x + info.tail_x : x % block.block_width; - const u32 ly = info.tail ? y + info.tail_y : y % block.block_height; + : mip.linear_size; + const bool tail = level >= info.first_tail_level; + for (u32 z = 0; z < active.depth; ++z) { + for (u32 y = 0; y < height; ++y) { + for (u32 x = 0; x < width; ++x) { + const u32 bx = tail ? 0 : x / block.block_width; + const u32 by = tail ? 0 : y / block.block_height; + const u32 bz = z / block.block_depth; + const u32 lx = tail ? x + mip.tail_x : x % block.block_width; + const u32 ly = tail ? y + mip.tail_y : y % block.block_height; const u32 lz = z % block.block_depth; u32 local = 0, block_xor = 0; Require(name, "reference offset", TileGetBlockOffset(block, lx, ly, lz, local) && - TileGetBlockXor(block, bx, by, info.surface_z + bz, + TileGetBlockXor(block, bx, by, mip.surface_z + bz, block_xor), "CPU reference address lookup failed"); - const uint64_t block_index = - static_cast(bz) * columns * rows + by * columns + bx; - const uint64_t tiled = info.tiled_offset + - block_index * block.block_size + - (local ^ block_xor); + const uint64_t tiled = + mip.offset + uint64_t{bz} * info.tiled_slice_stride + + (by * columns + bx) * block.block_size + (local ^ block_xor); const uint64_t linear = - info.linear_offset + static_cast(z) * slice + - static_cast(y) * info.pitch * info.bytes_per_element + - static_cast(x) * info.bytes_per_element; + mip.linear_offset + uint64_t{z} * slice + + (uint64_t{y} * mip.linear_pitch + x) * block.bytes_per_element; const uint64_t dst_offset = to_tiled ? tiled : linear; const uint64_t src_offset = to_tiled ? linear : tiled; Require(name, "reference range", - dst_offset + info.bytes_per_element <= dst->size() && - src_offset + info.bytes_per_element <= src.size(), + dst_offset + block.bytes_per_element <= dst->size() && + src_offset + block.bytes_per_element <= src.size(), "CPU reference address escaped storage"); - std::memcpy(dst->data() + dst_offset, src.data() + src_offset, - info.bytes_per_element); + if (info.GetColorTransform() == ColorTransform::Reverse10_11_11) { + uint32_t packed = 0; + std::memcpy(&packed, src.data() + src_offset, sizeof(packed)); + const std::array widths = + to_tiled ? std::array{11, 11, 10} + : std::array{10, 11, 11}; + std::array channels{}; + for (uint32_t channel = 0; channel < channels.size(); ++channel) { + channels[channel] = packed & ((1u << widths[channel]) - 1u); + packed >>= widths[channel]; + } + uint32_t shift = 0; + for (int channel = 2; channel >= 0; --channel) { + packed |= channels[channel] << shift; + shift += widths[channel]; + } + std::memcpy(dst->data() + dst_offset, &packed, sizeof(packed)); + } else if (info.GetColorTransform() == ColorTransform::SwapBgra16) { + std::array channels{}; + std::memcpy(channels.data(), src.data() + src_offset, + sizeof(channels)); + std::swap(channels[0], channels[2]); + std::memcpy(dst->data() + dst_offset, channels.data(), + sizeof(channels)); + } else { + std::memcpy(dst->data() + dst_offset, src.data() + src_offset, + block.bytes_per_element); + } } } } }; + const auto block_image = [](const TileBlockLayout &block, u32 width, + u32 height, u32 depth, u32 pitch, + uint64_t guest_size) { + ImageInfo info{}; + info.extent = {width, height, block.block_depth > 1 ? depth : 1u}; + info.type = block.block_depth > 1 ? Prospero::ImageType::kColor3D + : Prospero::ImageType::kColor2D; + info.resources = {1, block.block_depth > 1 ? 1u : depth}; + info.tile_mode = Prospero::TileMode::kStandard4KB; + info.bytes_per_block = block.bytes_per_element; + info.tiling = {block, 1, 1}; + info.data.size = guest_size; + info.first_tail_level = 16; + const uint64_t columns = + (pitch + block.block_width - 1u) / block.block_width; + const uint64_t rows = + (height + block.block_height - 1u) / block.block_height; + info.tiled_slice_stride = columns * rows * block.block_size; + info.mip_layout[0] = { + 0, info.IsVolume() ? info.tiled_slice_stride : guest_size, pitch, + static_cast(rows * block.block_height)}; + info.mip_layout[0].linear_pitch = pitch; + info.mip_layout[0].linear_size = + uint64_t{pitch} * height * block.bytes_per_element; + return info; + }; struct FamilyCase { TileBlockFamily family; u32 max_bpe; @@ -17457,33 +17558,136 @@ public: u32 case_index = 0; auto check_round_trip = [&](const char *stage, uint64_t tiled_size, - std::span infos) { - uint64_t linear_size = 0; - for (const auto &info : infos) { - linear_size = - std::max(linear_size, info.linear_offset + info.linear_size); - } + const ImageInfo &info, + TileManager *manager = nullptr) { + const uint64_t linear_size = info.LinearSize(); std::vector tiled(tiled_size); std::vector cpu(linear_size, 0); std::vector gpu(linear_size, 0xab); fill(&tiled, ++case_index); - for (const auto &info : infos) { - convert_reference(false, &cpu, tiled, info); + for (uint32_t level = 0; level < info.resources.levels; ++level) { + convert_reference(false, &cpu, tiled, info, level); } - gpu_detile(tiled, &gpu, tiled_size, linear_size, infos); + gpu_detile(tiled, &gpu, tiled_size, linear_size, info, manager); compare((std::string(stage) + " detile bytes").c_str(), cpu, gpu); - std::vector linear(linear_size); std::vector cpu_tiled(tiled_size, 0xab); std::vector gpu_tiled(tiled_size, 0xab); fill(&linear, 0x280u + case_index); - for (const auto &info : infos) { - convert_reference(true, &cpu_tiled, linear, info); + for (uint32_t level = 0; level < info.resources.levels; ++level) { + convert_reference(true, &cpu_tiled, linear, info, level); } - gpu_tile(linear, &gpu_tiled, tiled_size, linear_size, infos); + gpu_tile(linear, &gpu_tiled, tiled_size, linear_size, info, manager); compare((std::string(stage) + " tile bytes").c_str(), cpu_tiled, gpu_tiled); }; + + { + constexpr auto format = Prospero::BufferFormat::k8UNorm; + constexpr auto tile = Prospero::TileMode::kStandard256B; + constexpr u32 layers = 261; + const auto info = + MakeTilingImage(format, 1, 1, 1, layers, tile, 0, false); + const auto regions = info.BufferCopies(); + Require(name, "large array mip record", + regions.size() == 1 && + regions[0].imageSubresource.layerCount == layers && + info.MipExtent(0).depth == layers && + info.tiled_slice_stride == 256 && + info.data.size == uint64_t{layers} * 256, + "array layers expanded the mip table or lost their guest stride"); + check_round_trip("large array", info.data.size, info); + } + { + // Adjacent subword mips share output dwords and a guest tail block. + TileBlockLayout block{}; + Require(name, "packed mip block", + TileGetBlockLayout(TileBlockFamily::Standard4KB, 1, block), + "packed mip block is unavailable"); + auto info = block_image(block, 1, 1, 1, 1, 4096); + info.resources.levels = 16; + info.first_tail_level = 0; + for (u32 mip = 0; mip < info.resources.levels; ++mip) { + info.mip_layout[mip] = {0, 4096, block.block_width, block.block_height}; + info.mip_layout[mip].linear_offset = mip; + info.mip_layout[mip].linear_size = 1; + info.mip_layout[mip].linear_pitch = 1; + info.mip_layout[mip].tail_x = mip; + } + check_round_trip("16 packed subword mip records", 4096, info); + const auto atom = + std::max(m_runtime_context.GetPhysicalDeviceProperties() + .limits.nonCoherentAtomSize, + 1); + for (const auto [mips, bytes] : + std::array{std::pair{1u, 64u}, std::pair{2u, 112u}}) { + StreamBuffer compact_parameters(m_runtime_context, scheduler, + MemoryUsage::Stream, + Common::AlignUp(bytes, atom)); + TileManager compact_tiler(m_runtime_context, scheduler, + compact_parameters); + auto compact_info = info; + compact_info.resources.levels = mips; + check_round_trip("compact parameter buffer", 4096, compact_info, + &compact_tiler); + } + } + { + constexpr auto format = Prospero::BufferFormat::k32Float; + constexpr auto tile = Prospero::TileMode::kStandard256B; + constexpr u32 levels = 7, layers = 3; + constexpr uint64_t guest_stride = 0x4800, + guest_size = guest_stride * layers; + constexpr uint64_t offsets[levels] = {0x1a00, 0xb00, 0x500, 0x300, + 0x200, 0x100, 0}; + const auto info = MakeTilingImage(format, 65, 33, levels, layers, tile, + guest_size, false); + const auto regions = info.BufferCopies(); + bool valid = + info.tiled_slice_stride == guest_stride && regions.size() == levels; + for (u32 mip = 0; mip < levels && valid; ++mip) { + valid &= info.mip_layout[mip].offset == offsets[mip] && + info.MipExtent(mip).depth == layers && + regions[mip].imageSubresource.layerCount == layers; + } + Require(name, "reverse mip array addresses", valid, + "mip order or padded guest layer stride changed"); + check_round_trip("reverse mip array", guest_size, info); + std::vector tiled(guest_size), + expected_linear(info.LinearSize(), 0), observed_linear; + fill(&tiled, ++case_index); + for (u32 mip = 0; mip < levels; ++mip) + convert_reference(false, &expected_linear, tiled, info, mip); + gpu_detile(tiled, &observed_linear, guest_size, info.LinearSize(), info, + nullptr, 260); + compare("detile nonzero source base", expected_linear, observed_linear); + const auto saved_mips = info.mip_layout; + const auto prefix = info.BufferCopies(256, 2); + Require(name, "partial mip array addresses", + prefix.size() == 2 && prefix[0].bufferOffset == 256 && + prefix[1].bufferOffset == + info.mip_layout[1].linear_offset + 256 && + info.mip_layout[0].offset == 0x1a00 && + info.mip_layout[1].offset == 0xb00, + "partial mip transfer recomputed the full-chain guest stride"); + std::vector linear(info.LinearSize()); + fill(&linear, ++case_index); + for (const u32 count : {2u, levels, 1u, levels}) { + std::vector expected(guest_size, 0xab), + observed(guest_size, 0xab); + for (u32 mip = 0; mip < count; ++mip) + convert_reference(true, &expected, linear, info, mip); + gpu_tile(linear, &observed, guest_size, info.LinearSize(), info, + nullptr, count, count == levels ? 516 : 260, + count == levels ? 520 : 264); + compare("repeated partial/full transfer", expected, observed); + Require( + name, "immutable transfer metadata", + info.mip_layout == saved_mips && info.resources.levels == levels && + info.tiled_slice_stride == guest_stride, + "partial or repeated transfer changed canonical image metadata"); + } + } for (const auto family : families) { for (u32 bpe = 1; bpe <= family.max_bpe; bpe <<= 1u) { TileBlockLayout block{}; @@ -17505,31 +17709,20 @@ public: (depth + block.block_depth - 1u) / block.block_depth; const uint64_t storage_size = block_columns * block_rows * block_slices * block.block_size; - const uint64_t slice_stride = - static_cast(pitch) * height * bpe; - std::vector tiled(storage_size); std::vector cpu(storage_size, 0); std::vector gpu(storage_size, 0xab); fill(&tiled, ++case_index); - GpuTileInfo info{}; - info.family = block.family; - info.bytes_per_element = block.bytes_per_element; - info.linear_size = storage_size; - info.tiled_size = storage_size; - info.linear_slice_stride = volume ? slice_stride : 0; - info.width = width; - info.height = height; - info.depth = depth; - info.pitch = pitch; - info.surface_z = family.family == TileBlockFamily::RenderTarget64KB || - family.family == TileBlockFamily::Depth64KB - ? 3 - : 0; + auto info = + block_image(block, width, height, depth, pitch, storage_size); + info.mip_layout[0].surface_z = + family.family == TileBlockFamily::RenderTarget64KB || + family.family == TileBlockFamily::Depth64KB + ? 3 + : 0; convert_reference(false, &cpu, tiled, info); - gpu_detile(tiled, &gpu, storage_size, storage_size, - std::span(&info, 1)); + gpu_detile(tiled, &gpu, storage_size, storage_size, info); const auto family_label = [&](const char *operation) { std::ostringstream out; out << operation << " family=" << static_cast(family.family) @@ -17544,8 +17737,7 @@ public: std::vector gpu_tiled(storage_size, 0xab); fill(&linear, 0x80u + case_index); convert_reference(true, &cpu_tiled, linear, info); - gpu_tile(linear, &gpu_tiled, storage_size, storage_size, - std::span(&info, 1)); + gpu_tile(linear, &gpu_tiled, storage_size, storage_size, info); compare(family_label("tile bytes").c_str(), cpu_tiled, gpu_tiled); } } @@ -17603,19 +17795,11 @@ public: !TileGetTiledTextureLayout(description, invalid), "Standard256B volume silently fell back to a 2D array layout"); } - { - constexpr auto format = Prospero::BufferFormat::kFmask8_S4_F4; - constexpr auto tile = Prospero::TileMode::kStandard64KB; - TileSizeAlign total{}; - TileGetTextureSize(format, 128, 128, 1, tile, &total, nullptr, nullptr); - const auto layout = TextureCalcUploadLayout( - format, 128, 128, 1, 1, tile, total.size, false, name); - const auto regions = TextureBuildImageCopies(layout); - std::vector infos; - Require(name, "FMASK policy", - !TextureBuildGpuTileInfos(total.size, regions, layout, 1, infos), - "FMASK entered the texel tiler through a non-depth family"); - } + Require( + name, "FMASK policy", + Prospero::IsFmaskTextureFormat(Prospero::BufferFormat::kFmask8_S4_F4), + "FMASK sample metadata lost its explicit tiler exclusion " + "classification"); u32 format_cases = 0; for (u32 raw_format = 1; raw_format <= static_cast(Prospero::BufferFormat::kBc7Srgb); @@ -17636,36 +17820,27 @@ public: constexpr u32 width = 67; constexpr u32 height = 51; - const u32 pitch = TileGetTexturePitch(format, width, mode.tile); TileSizeAlign total{}; TileGetTextureSize(format, width, height, 1, mode.tile, &total, nullptr, nullptr); Require(name, "format size", total.size != 0, "supported format has an empty layout"); - const auto layout = - TextureCalcUploadLayout(format, width, height, 1, 1, mode.tile, - total.size, false, name); - const auto regions = TextureBuildImageCopies(layout); - std::vector infos; - if (!TextureBuildGpuTileInfos(total.size, regions, layout, 1, infos)) { - std::ostringstream out; - out << "format=" << raw_format - << " tile=" << static_cast(mode.tile) - << " size=" << total.size << " pitch=" << pitch; - Fail(name, "format infos", out.str()); - } + const auto layout = MakeTilingImage(format, width, height, 1, 1, + mode.tile, total.size, false); + const auto regions = layout.BufferCopies(); Require(name, "format family", - infos.size() == 1 && infos[0].family == mode.family && - infos[0].bytes_per_element == bpe, + layout.resources.levels == 1 && + layout.tiling.block.family == mode.family && + layout.tiling.block.bytes_per_element == bpe, "texture info selected the wrong shader family"); std::vector tiled(total.size); std::vector cpu(total.size, 0); std::vector gpu(total.size, 0xab); fill(&tiled, ++case_index); - convert_reference(false, &cpu, tiled, infos[0]); - gpu_detile(tiled, &gpu, total.size, total.size, infos); + convert_reference(false, &cpu, tiled, layout); + gpu_detile(tiled, &gpu, total.size, total.size, layout); compare("format bytes", cpu, gpu); ++format_cases; } @@ -17740,24 +17915,16 @@ public: TileSizeAlign total{}; TileGetTextureSize(format, width, height, levels, tile, &total, nullptr, nullptr); - const auto layout = - TextureCalcUploadLayout(format, width, height, levels, 1, tile, - total.size, false, name); - const auto regions = TextureBuildImageCopies(layout); - std::vector infos; - const bool built = - TextureBuildGpuTileInfos(total.size, regions, layout, levels, infos); - uint64_t linear_size = 0; - for (const auto &info : infos) { - linear_size = - std::max(linear_size, info.linear_offset + info.linear_size); - } + const auto layout = MakeTilingImage(format, width, height, levels, 1, + tile, total.size, false); + const auto regions = layout.BufferCopies(); + + const auto linear_size = layout.LinearSize(); Require(name, "BC1 mip-tail capacities", - built && total.size == 0x10000 && - layout.surface.first_tail_level == 0 && + total.size == 0x10000 && layout.first_tail_level == 0 && linear_size == 0x15560 && linear_size > total.size, "BC1 mip tail conflated tiled and linear capacities"); - check_round_trip("BC1 mip tail", total.size, infos); + check_round_trip("BC1 mip tail", total.size, layout); } { @@ -17767,18 +17934,19 @@ public: TileSizeAlign total{}; TileGetTextureTotalSize(format, width, height, layers, 1, tile, false, total); - const auto layout = - TextureCalcUploadLayout(format, width, height, 1, layers, tile, - total.size, false, name); - const auto regions = TextureBuildImageCopies(layout); - std::vector infos; - const bool built = - TextureBuildGpuTileInfos(total.size, regions, layout, 1, infos); + const auto layout = MakeTilingImage(format, width, height, 1, layers, + tile, total.size, false); + const auto regions = layout.BufferCopies(); + Require(name, "array infos", - built && infos.size() == layers && infos[0].surface_z == 0 && - infos[1].surface_z == 1 && infos[2].surface_z == 2, - "array slices lost their absolute surface Z"); - check_round_trip("array", total.size, infos); + layout.resources.levels == 1 && + layout.MipExtent(0).depth == layers && + layout.mip_layout[0].surface_z == 0 && + layout.tiled_slice_stride == total.size / layers && + regions.size() == 1 && + regions[0].imageSubresource.layerCount == layers, + "array mip lost its layer count or absolute surface Z origin"); + check_round_trip("array", total.size, layout); } for (const auto &mode : standard_modes) { @@ -17792,21 +17960,25 @@ public: TileSizeAlign total{}; TileGetTextureSize(format, width, height, levels, mode.tile, &total, nullptr, nullptr); - const auto layout = - TextureCalcUploadLayout(format, width, height, levels, 1, mode.tile, - total.size, false, name); - const auto regions = TextureBuildImageCopies(layout); - std::vector infos; + const auto layout = MakeTilingImage(format, width, height, levels, 1, + mode.tile, total.size, false); + const auto regions = layout.BufferCopies(); Require(name, "odd mip infos", - TextureBuildGpuTileInfos(total.size, regions, layout, levels, - infos) && - infos.size() == 2 && infos[1].tiled_width >= infos[1].pitch && - infos[1].tiled_height >= infos[1].height && + layout.resources.levels == 2 && + layout.mip_layout[1].pitch >= + layout.mip_layout[1].linear_pitch && + layout.mip_layout[1].height >= + ((layout.MipExtent(1).height + + layout.tiling.texel_height - 1u) / + layout.tiling.texel_height) && (mode.family == TileBlockFamily::Standard256B || - infos[1].tiled_height > infos[1].height), + layout.mip_layout[1].height > + ((layout.MipExtent(1).height + + layout.tiling.texel_height - 1u) / + layout.tiling.texel_height)), "odd multi-mip physical stride collapsed to the active linear " "extent"); - check_round_trip("odd multi-mip", total.size, infos); + check_round_trip("odd multi-mip", total.size, layout); } struct TailMode { @@ -17831,20 +18003,13 @@ public: TileSizeAlign total{}; TileGetTextureSize(format, width, height, levels, mode.tile, &total, nullptr, nullptr); - const auto layout = - TextureCalcUploadLayout(format, width, height, levels, 1, mode.tile, - total.size, false, name); - const auto regions = TextureBuildImageCopies(layout); - std::vector infos; + const auto layout = MakeTilingImage(format, width, height, levels, 1, + mode.tile, total.size, false); + const auto regions = layout.BufferCopies(); Require(name, "2D mip tail seam", - layout.surface.first_tail_level == 2 && - TextureBuildGpuTileInfos(total.size, regions, layout, levels, - infos) && - infos.size() == levels && !infos[0].tail && !infos[1].tail && - std::all_of(infos.begin() + 2, infos.end(), - [](const auto &info) { return info.tail; }), + layout.first_tail_level == 2 && layout.resources.levels == levels, "2D mip chain lost its linear/tiled tail boundary"); - check_round_trip("2D mip tail seam", total.size, infos); + check_round_trip("2D mip tail seam", total.size, layout); } { @@ -17857,25 +18022,31 @@ public: TileSizeAlign total{}; TileGetTextureTotalSize(format, width, height, depth, levels, tile, true, total); - const auto layout = - TextureCalcUploadLayout(format, width, height, levels, depth, tile, - total.size, true, name); - const auto regions = TextureBuildImageCopies(layout); - std::vector infos; - const bool built = - TextureBuildGpuTileInfos(total.size, regions, layout, levels, infos); - Require(name, "3D mip infos", - regions.size() == depth + (depth >> 1u) && built && - infos.size() == 4 && infos[0].depth == 8 && - infos[1].depth == 8 && infos[2].depth == 1 && - infos[3].depth == 8 && infos[0].tiled_offset == 0x19000 && - infos[1].tiled_offset == 0x83000 && - infos[2].tiled_offset == 0xed000 && - infos[3].tiled_offset == 0 && infos[3].pitch == 32 && - infos[3].height == 64 && infos[3].tiled_width == 40 && - infos[3].tiled_height == 80, - "Standard4KB3D mip depth, packing, or physical stride changed"); - check_round_trip("3D mip", total.size, infos); + const auto layout = MakeTilingImage(format, width, height, levels, depth, + tile, total.size, true); + const auto regions = layout.BufferCopies(); + + Require( + name, "3D mip infos", + regions.size() == levels && layout.resources.levels == levels && + regions[0].imageExtent.depth == depth && + regions[1].imageExtent.depth == (depth >> 1u) && + layout.MipExtent(0).depth == 17 && + layout.MipExtent(1).depth == 8 && + layout.mip_layout[0].offset == 0x19000 && + layout.tiled_slice_stride == 0x6a000 && + layout.mip_layout[0].offset + layout.tiled_slice_stride == + 0x83000 && + layout.mip_layout[0].offset + 2 * layout.tiled_slice_stride == + 0xed000 && + layout.mip_layout[1].offset == 0 && + layout.mip_layout[1].linear_pitch == 32 && + ((layout.MipExtent(1).height + layout.tiling.texel_height - 1u) / + layout.tiling.texel_height) == 64 && + layout.mip_layout[1].pitch == 40 && + layout.mip_layout[1].height == 80, + "Standard4KB3D mip depth, packing, or physical stride changed"); + check_round_trip("3D mip", total.size, layout); } { @@ -17888,19 +18059,21 @@ public: TileSizeAlign total{}; TileGetTextureTotalSize(format, width, height, depth, levels, tile, true, total); - const auto layout = - TextureCalcUploadLayout(format, width, height, levels, depth, tile, - total.size, true, name); - const auto regions = TextureBuildImageCopies(layout); - std::vector infos; - Require(name, "3D BC mip infos", - TextureBuildGpuTileInfos(total.size, regions, layout, levels, - infos) && - infos.size() == 4 && infos[3].pitch == 8 && - infos[3].height == 16 && infos[3].tiled_width == 16 && - infos[3].tiled_height == 24, - "block-compressed 3D mip lost its physical row or slice stride"); - check_round_trip("3D BC mip", total.size, infos); + const auto layout = MakeTilingImage(format, width, height, levels, depth, + tile, total.size, true); + const auto regions = layout.BufferCopies(); + Require( + name, "3D BC mip infos", + layout.resources.levels == levels && + layout.MipExtent(0).depth == depth && + layout.MipExtent(1).depth == (depth >> 1u) && + layout.mip_layout[1].linear_pitch == 8 && + ((layout.MipExtent(1).height + layout.tiling.texel_height - 1u) / + layout.tiling.texel_height) == 16 && + layout.mip_layout[1].pitch == 16 && + layout.mip_layout[1].height == 24, + "block-compressed 3D mip lost its physical row or slice stride"); + check_round_trip("3D BC mip", total.size, layout); } { @@ -17910,10 +18083,9 @@ public: TileSizeAlign total{}; TileGetTextureSize(format, width, height, levels, tile, &total, nullptr, nullptr); - const auto layout = - TextureCalcUploadLayout(format, width, height, levels, 1, tile, - total.size, false, name); - const auto regions = TextureBuildImageCopies(layout); + const auto layout = MakeTilingImage(format, width, height, levels, 1, + tile, total.size, false); + const auto regions = layout.BufferCopies(); Require(name, "linear BC native regions", regions.size() == levels && std::all_of(regions.begin(), regions.end(), @@ -17952,27 +18124,21 @@ public: TileSizeAlign total{}; TileGetTextureTotalSize(test.format, width, height, depth, levels, test.tile, true, total); - const auto layout = - TextureCalcUploadLayout(test.format, width, height, levels, depth, - test.tile, total.size, true, name); - const auto regions = TextureBuildImageCopies(layout); - std::vector infos; - const bool built = - TextureBuildGpuTileInfos(total.size, regions, layout, levels, infos); + const auto layout = MakeTilingImage(test.format, width, height, levels, + depth, test.tile, total.size, true); + const auto regions = layout.BufferCopies(); + const bool uses_z = test.family == TileBlockFamily::RenderTarget64KB || test.family == TileBlockFamily::Depth64KB; Require(name, "volume family infos", - built && !infos.empty() && - std::all_of(infos.begin(), infos.end(), - [&](const auto &info) { - return info.family == test.family; - }) && - (!uses_z || std::any_of(infos.begin(), infos.end(), - [](const auto &info) { - return info.surface_z != 0; - })), + layout.resources.levels != 0 && + layout.tiling.block.family == test.family && + layout.resources.levels == levels && + layout.MipExtent(0).depth == depth && + (!uses_z || (layout.mip_layout[0].surface_z == 0 && + layout.tiled_slice_stride != 0)), "volume mode selected the wrong family or lost its surface Z"); - check_round_trip("volume family", total.size, infos); + check_round_trip("volume family", total.size, layout); } struct VolumeTailCase { @@ -18011,36 +18177,29 @@ public: TileSizeAlign total{}; TileGetTextureTotalSize(tail.format, width, height, depth, levels, tile, true, total); - const auto layout = - TextureCalcUploadLayout(tail.format, width, height, levels, depth, - tile, total.size, true, name); - const auto regions = TextureBuildImageCopies(layout); - std::vector infos; - bool valid = total.size == 8192 && - regions.size() == depth + std::max(depth >> 1u, 1u) + - std::max(depth >> 2u, 1u) + - std::max(depth >> 3u, 1u) + - std::max(depth >> 4u, 1u) && - TextureBuildGpuTileInfos(total.size, regions, layout, levels, - infos) && - infos.size() == 6; + const auto layout = MakeTilingImage(tail.format, width, height, levels, + depth, tile, total.size, true); + const auto regions = layout.BufferCopies(); + bool valid = total.size == 8192 && regions.size() == levels && + layout.resources.levels == levels; const u32 table = std::countr_zero(tail.bytes); for (u32 level = 0; level < levels && valid; level++) { - const auto &info = infos[level == 0 ? 0 : level + 1]; - valid &= info.tail && info.tail_x == volume_tail_xy[table][level][0] && - info.tail_y == volume_tail_xy[table][level][1] && - info.tiled_offset == 0; + const auto &mip = layout.mip_layout[level]; + valid &= + level >= layout.first_tail_level && + mip.tail_x == volume_tail_xy[table][level][0] && + mip.tail_y == volume_tail_xy[table][level][1] && mip.offset == 0 && + layout.MipExtent(level).depth == std::max(depth >> level, 1u) && + layout.tiled_slice_stride == 4096; } if (valid) { - valid = infos[1].tail && - infos[1].tail_x == volume_tail_xy[table][0][0] && - infos[1].tail_y == volume_tail_xy[table][0][1] && - infos[1].tiled_offset == 4096; + valid = layout.MipExtent(0).depth == block.block_depth + 1u && + layout.mip_layout[0].offset + layout.tiled_slice_stride == 4096; } Require( name, "3D mip tail infos", valid, "Standard4KB3D mip tail coordinates or block-slice packing changed"); - check_round_trip("3D mip tail", total.size, infos); + check_round_trip("3D mip tail", total.size, layout); } for (const auto family : @@ -18064,24 +18223,18 @@ public: std::vector cpu(linear_size, 0xcd); std::vector gpu(linear_size, 0xab); fill(&tiled, ++case_index); - GpuTileInfo info{}; - info.family = block.family; - info.bytes_per_element = block.bytes_per_element; - info.linear_size = linear_size; - info.tiled_size = block.block_size; - info.width = width; - info.height = height; - info.pitch = pitch; - info.tail = true; - info.tail_x = x; - info.tail_y = y; - info.surface_z = family == TileBlockFamily::RenderTarget64KB || - family == TileBlockFamily::Depth64KB - ? 2 - : 0; + auto info = + block_image(block, width, height, 1, pitch, block.block_size); + info.first_tail_level = 0; + info.mip_layout[0].tail_x = x; + info.mip_layout[0].tail_y = y; + info.mip_layout[0].surface_z = + family == TileBlockFamily::RenderTarget64KB || + family == TileBlockFamily::Depth64KB + ? 2 + : 0; convert_reference(false, &cpu, tiled, info); - gpu_detile(tiled, &gpu, block.block_size, linear_size, - std::span(&info, 1)); + gpu_detile(tiled, &gpu, block.block_size, linear_size, info); compare("tail bytes", cpu, gpu); std::vector linear(linear_size); @@ -18089,8 +18242,7 @@ public: std::vector gpu_tiled(block.block_size, 0xab); fill(&linear, 0x400u + case_index); convert_reference(true, &cpu_tiled, linear, info); - gpu_tile(linear, &gpu_tiled, block.block_size, linear_size, - std::span(&info, 1)); + gpu_tile(linear, &gpu_tiled, block.block_size, linear_size, info); compare("tail tile bytes", cpu_tiled, gpu_tiled); } } @@ -18103,58 +18255,50 @@ public: std::vector small_expected(small_block.block_size, 0); std::vector small_output(small_block.block_size, 0xab); fill(&small_input, 0xee); - GpuTileInfo small_info{}; - small_info.family = small_block.family; - small_info.bytes_per_element = small_block.bytes_per_element; - small_info.linear_size = small_output.size(); - small_info.tiled_size = small_input.size(); - small_info.width = 1; - small_info.height = 1; - small_info.pitch = small_block.block_width; + auto small_info = block_image(small_block, 1, 1, 1, small_block.block_width, + small_input.size()); convert_reference(false, &small_expected, small_input, small_info); gpu_detile(small_input, &small_output, small_input.size(), - small_output.size(), - std::span(&small_info, 1)); + small_output.size(), small_info); compare("small detile", small_expected, small_output); std::fill(small_output.begin(), small_output.end(), 0xab); gpu_detile(small_input, &small_output, small_input.size(), - small_output.size(), - std::span(&small_info, 1)); + small_output.size(), small_info); compare("scheduler-owned reuse", small_expected, small_output); constexpr auto volume_format = Prospero::BufferFormat::k32UInt; constexpr auto volume_tile = Prospero::TileMode::kLinear; constexpr u32 volume_width = 8, volume_height = 4, volume_depth = 5; constexpr u32 volume_levels = 3; - const u32 volume_pitch = - TileGetTexturePitch(volume_format, volume_width, volume_tile); TileSizeAlign volume_size{}; TileGetTextureTotalSize(volume_format, volume_width, volume_height, volume_depth, volume_levels, volume_tile, true, volume_size); - const auto volume_layout = TextureCalcUploadLayout( + const auto volume_layout = MakeTilingImage( volume_format, volume_width, volume_height, volume_levels, volume_depth, - volume_tile, volume_size.size, true, name); - const auto volume_copies = TextureBuildImageCopies(volume_layout); + volume_tile, volume_size.size, true); + const auto volume_copies = volume_layout.BufferCopies(); std::vector volume_source(volume_size.size / sizeof(u32), 0); std::vector> volume_probes; for (const auto © : volume_copies) { - for (u32 y = 0; y < copy.imageExtent.height; y++) { - for (u32 x = 0; x < copy.imageExtent.width; x++) { - const auto index = - (copy.bufferOffset + - (static_cast(y) * copy.bufferRowLength + x) * - sizeof(u32)) / - sizeof(u32); - const u32 value = - 0xa0000000u | (copy.imageSubresource.mipLevel << 24u) | - (static_cast(copy.imageOffset.z) << 16u) | (y << 8u) | x; - Require(name, "linear volume native bounds", - index < volume_source.size(), - "linear volume copy escaped its padded guest layout"); - volume_source[index] = value; - volume_probes.emplace_back(static_cast(index), value); + for (u32 z = 0; z < copy.imageExtent.depth; ++z) { + for (u32 y = 0; y < copy.imageExtent.height; ++y) { + for (u32 x = 0; x < copy.imageExtent.width; ++x) { + const auto index = copy.bufferOffset / sizeof(u32) + + (uint64_t{z} * copy.bufferImageHeight + y) * + copy.bufferRowLength + + x; + const u32 value = + 0xa0000000u | (copy.imageSubresource.mipLevel << 24u) | + ((static_cast(copy.imageOffset.z) + z) << 16u) | + (y << 8u) | x; + Require(name, "linear volume native bounds", + index < volume_source.size(), + "linear volume copy escaped its padded guest layout"); + volume_source[index] = value; + volume_probes.emplace_back(static_cast(index), value); + } } } } @@ -18164,22 +18308,9 @@ public: auto volume_download = CreateHostBuffer( name, volume_size.size, vk::BufferUsageFlagBits::eTransferDst, std::vector(volume_source.size(), 0)); - ImageInfo volume_info{}; - volume_info.data = {0x10000, volume_size.size}; + auto volume_info = volume_layout; + volume_info.data.address = 0x10000; volume_info.pixel_format = vk::Format::eR32Uint; - volume_info.guest_format = volume_format; - volume_info.type = Prospero::ImageType::kColor3D; - volume_info.extent = {volume_width, volume_height, volume_depth}; - volume_info.resources = {volume_levels, 1}; - volume_info.pitch = volume_pitch; - volume_info.bytes_per_block = sizeof(u32); - volume_info.tile_mode = volume_tile; - for (u32 level = 0; level < volume_levels; level++) { - volume_info.mip_layout[level] = {volume_layout.mips[level].offset, - volume_layout.mips[level].size, - volume_layout.mips[level].row_length, - volume_layout.mips[level].image_height}; - } Libs::Graphics::Image volume_image(m_runtime_context, scheduler, volume_info); volume_image.Upload(volume_copies, volume_upload.buffer, 0, @@ -39528,14 +39659,13 @@ void CheckStorageTextureLinearUploadLayout() { TileSizeAlign total{}; TileGetTextureTotalSize(format, width, height, depth, 1, tile, false, total); const auto layout = - TextureCalcUploadLayout(format, width, height, 1, depth, tile, total.size, - false, "StorageTextureLinearTest"); - const auto regions = TextureBuildImageCopies(layout); + MakeTilingImage(format, width, height, 1, depth, tile, total.size, false); + const auto regions = layout.BufferCopies(); Require("StorageTextureLinearUpload", "layout", pitch == width && total.size == 0x1fa4000 && total.align == 256 && - layout.surface.description.tile_mode == tile && - layout.pitch == width && layout.slice_stride == total.size && - regions.size() == 1 && regions[0].bufferOffset == 0 && + layout.tile_mode == tile && layout.pitch == width && + layout.linear_slice_stride == total.size && regions.size() == 1 && + regions[0].bufferOffset == 0 && regions[0].imageExtent.width == width && regions[0].imageExtent.height == height && regions[0].bufferRowLength == width, @@ -39557,22 +39687,20 @@ void CheckStorageTextureDepthTileUploadLayout() { TileGetTextureSize(format, width, height, 1, tile, &slice, &level, &padded); TileGetTextureTotalSize(format, width, height, depth, 1, tile, false, total); const auto layout = - TextureCalcUploadLayout(format, width, height, 1, depth, tile, total.size, - false, "StorageTextureDepthTileTest"); - const auto regions = TextureBuildImageCopies(layout); + MakeTilingImage(format, width, height, 1, depth, tile, total.size, false); + const auto regions = layout.BufferCopies(); Require("StorageTextureDepthTileUpload", "PPSA14053 layout", pitch == 256 && padded.width == 256 && padded.height == 256 && slice.size == 0x10000 && slice.align == 0x10000 && level.size == slice.size && level.offset == 0 && total.size == slice.size && total.align == slice.align && - layout.surface.description.tile_mode == tile && - layout.surface.texture.block.family == - TileBlockFamily::Depth64KB && - layout.pitch == pitch && layout.slice_stride == pitch && - layout.source_slice_stride == total.size && - layout.mips[0].size == pitch && - layout.surface.mips[0].size == total.size && - regions.size() == 1 && regions[0].bufferOffset == 0 && + layout.tile_mode == tile && + layout.tiling.block.family == TileBlockFamily::Depth64KB && + layout.pitch == pitch && layout.linear_slice_stride == 0 && + layout.tiled_slice_stride == total.size && + layout.mip_layout[0].linear_size == pitch && + layout.mip_layout[0].size == total.size && regions.size() == 1 && + regions[0].bufferOffset == 0 && regions[0].imageExtent.width == width && regions[0].imageExtent.height == height && regions[0].bufferRowLength == pitch, @@ -40055,27 +40183,29 @@ void CheckStorageTextureVolumeUploadLayout() { TileSizeAlign total{}; TileGetTextureTotalSize(format, width, height, depth, 1, Prospero::TileMode::kRenderTarget, true, total); - const auto layout = TextureCalcUploadLayout( - format, width, height, 1, depth, Prospero::TileMode::kRenderTarget, - total.size, true, "StorageTextureVolumeTest"); - const auto regions = TextureBuildImageCopies(layout); - Require( - "StorageTextureVolumeUpload", "layout", - pitch == 128 && total.size == 0x210000 && layout.slice_stride == 0x2208 && - layout.source_slice_stride == 0 && layout.mips[0].size == 0x2208 && - layout.surface.mips[0].size == 0x10000 && regions.size() == depth, - "3D render-target upload did not preserve its compact linear layout"); + const auto layout = + MakeTilingImage(format, width, height, 1, depth, + Prospero::TileMode::kRenderTarget, total.size, true); + const auto regions = layout.BufferCopies(); + Require("StorageTextureVolumeUpload", "layout", + pitch == 128 && total.size == 0x210000 && + layout.linear_slice_stride == 0 && + layout.tiled_slice_stride == 0x10000 && + layout.mip_layout[0].linear_size == 0x2208 && + layout.mip_layout[0].size == 0x10000 && regions.size() == 1 && + regions[0].imageExtent.depth == depth, + "3D render-target upload did not preserve its compact linear layout"); - std::vector infos; Require("StorageTextureVolumeUpload", "GPU records", - TextureBuildGpuTileInfos(total.size, regions, layout, 1, infos) && - infos.size() == depth, - "3D render-target GPU records were not built"); + layout.resources.levels == 1 && layout.MipExtent(0).depth == depth, + "3D render-target canonical mip record lost its depth"); for (const uint32_t z : {0u, 1u, depth - 1u}) { + const auto &mip = layout.mip_layout[0]; Require("StorageTextureVolumeUpload", "slice offsets", - infos[z].linear_offset == static_cast(z) * 0x2208 && - infos[z].tiled_offset == static_cast(z) * 0x10000 && - infos[z].surface_z == z && infos[z].pitch == width, + mip.linear_offset + z * mip.linear_size == uint64_t{z} * 0x2208 && + mip.offset + z * layout.tiled_slice_stride == + uint64_t{z} * 0x10000 && + mip.surface_z + z == z && mip.linear_pitch == width, "volume slice lost its linear stride, block slice, or Z swizzle"); } std::printf("[host] %-32s ok\n", "StorageTextureVolumeUpload"); @@ -40091,30 +40221,34 @@ void CheckStorageTextureVolumeMipRegions() { TileSizeAlign total{}; TileGetTextureTotalSize(format, width, height, depth, levels, tile, true, total); - const auto layout = TextureCalcUploadLayout( - format, width, height, levels, depth, tile, total.size, true, - "StorageTextureVolumeMipTest"); - const auto copies = TextureBuildImageCopies(layout); + const auto layout = MakeTilingImage(format, width, height, levels, depth, + tile, total.size, true); + const auto copies = layout.BufferCopies(); - bool valid = copies.size() == 8; - size_t index = 0; - for (uint32_t level = 0; level < levels && valid; level++) { - const uint32_t mip_depth = std::max(depth >> level, 1u); - const uint32_t mip_width = std::max(width >> level, 1u); - const uint32_t mip_height = std::max(height >> level, 1u); - for (uint32_t z = 0; z < mip_depth; z++, index++) { - const auto © = copies[index]; - valid &= copy.imageSubresource.mipLevel == level && - copy.imageOffset.z == static_cast(z) && - copy.imageExtent.width == mip_width && - copy.imageExtent.height == mip_height && - copy.bufferOffset == - layout.mips[level].offset + z * layout.slice_stride && - copy.bufferRowLength == layout.mips[level].row_length && - copy.bufferImageHeight == layout.mips[level].image_height; - } + bool valid = copies.size() == levels; + for (uint32_t level = 0; level < levels && valid; ++level) { + const auto © = copies[level]; + const auto &mip = layout.mip_layout[level]; + const auto active = layout.MipExtent(level); + valid &= copy.imageSubresource.mipLevel == level && + copy.imageOffset.z == 0 && + copy.imageExtent.width == active.width && + copy.imageExtent.height == active.height && + copy.imageExtent.depth == active.depth && + copy.bufferOffset == mip.linear_offset && + copy.bufferRowLength == mip.linear_pitch && + uint64_t{copy.bufferRowLength} * copy.bufferImageHeight * + sizeof(uint32_t) == + layout.linear_slice_stride; } - valid &= index == copies.size(); + const auto saved_mips = layout.mip_layout; + const auto prefix = layout.BufferCopies(1024, 1); + const auto again = layout.BufferCopies(); + valid &= prefix.size() == 1 && + prefix[0].bufferOffset == 1024 + layout.mip_layout[0].linear_offset && + copies[0].bufferOffset == 768 && copies[1].bufferOffset == 256 && + copies[2].bufferOffset == 0 && again == copies && + layout.mip_layout == saved_mips; Require("StorageTextureVolumeMipRegions", "per-mip depth", valid, "direction-neutral image copies did not shrink depth or preserve " "Vulkan Z coordinates"); @@ -43013,6 +43147,13 @@ int main(int argc, char **argv) { vulkan.CheckStreamBufferRing(); return 0; } + if (argc == 2 && std::strcmp(argv[1], "--image-layout-only") == 0) { + CheckStorageTextureLinearUploadLayout(); + CheckStorageTextureDepthTileUploadLayout(); + CheckStorageTextureVolumeUploadLayout(); + CheckStorageTextureVolumeMipRegions(); + return 0; + } if (argc == 2 && std::strcmp(argv[1], "--gpu-tiler-only") == 0) { VulkanHarness vulkan; vulkan.CheckGpuTilerCpuParity();