graphics: add RDNA2 FLAT BDA buffer cache and faults

src/graphics/shader/recompiler/ir/ShaderIR.h

@@ -176,20 +176,6 @@ struct SampledResourcePair {
 	bool operator==(const SampledResourcePair& other) const = default;
 };
 
-struct AddressResource {
-	uint32_t     source           = UINT32_MAX;
-	uint32_t     first_use_pc     = 0;
-	ResourceKind kind             = ResourceKind::Flat;
-	int32_t      min_offset       = 0;
-	uint64_t     specialized_base = 0;
-	bool         unbased          = true;
-	bool         read             = false;
-	bool         written          = false;
-	bool         atomic           = false;
-
-	bool operator==(const AddressResource& other) const = default;
-};
-
 enum class StageInputKind {
 	VertexIndex,
 	InstanceIndex,
@@ -321,7 +307,8 @@ enum class DescriptorBindingKind {
 	StorageAtomic3D,
 	Samplers,
 	Gds,
-	AddressMemory,
+	BdaPagetable,
+	FaultBuffer,
 	FlattenedSrt,
 	UserData,
 	Count,
@@ -431,13 +418,11 @@ struct BindingLayout {
 
 struct ShaderInfo {
 	static constexpr uint32_t MaxBuffers      = 32;
-	static constexpr uint32_t MaxAddresses    = 32;
 	static constexpr uint32_t MaxImages       = 32;
 	static constexpr uint32_t MaxSamplers     = 32;
 	static constexpr uint32_t MaxSampledPairs = 64;
 
 	std::vector<BufferResource>      buffers;
-	std::vector<AddressResource>     addresses;
 	std::vector<ImageResource>       images;
 	std::vector<SamplerResource>     samplers;
 	std::vector<SampledResourcePair> sampled_pairs;
@@ -446,6 +431,7 @@ struct ShaderInfo {
 	std::array<uint8_t, 32>          vertex_fetch_components {};
 	int32_t                          vertex_offset_sgpr = -1;
 	bool                             has_bitwise_xor    = false;
+	bool                             uses_dma           = false;
 
 	bool operator==(const ShaderInfo& other) const = default;
 };

src/graphics/shader/recompiler/ir/passes/BindingLayout.cpp

@@ -93,8 +93,7 @@ bool AllocateBindings(Program& program, uint32_t push_constant_offset, std::stri
 		return false;
 	}
 	next.memory_offset_dword = static_cast<uint32_t>(next.user_data_registers.size());
-	next.memory_offset_count =
-	    static_cast<uint32_t>(program.info.buffers.size() + program.info.addresses.size());
+	next.memory_offset_count = static_cast<uint32_t>(program.info.buffers.size());
 
 	if (!program.info.buffers.empty()) {
 		std::vector<uint32_t> resources(program.info.buffers.size());
@@ -157,12 +156,9 @@ bool AllocateBindings(Program& program, uint32_t push_constant_offset, std::stri
 	if (uses_gds) {
 		AddBinding(next, DescriptorBindingKind::Gds);
 	}
-	if (!program.info.addresses.empty()) {
-		std::vector<uint32_t> resources(program.info.addresses.size());
-		for (uint32_t i = 0; i < resources.size(); i++) {
-			resources[i] = i;
-		}
-		AddBinding(next, DescriptorBindingKind::AddressMemory, std::move(resources));
+	if (program.info.uses_dma) {
+		AddBinding(next, DescriptorBindingKind::BdaPagetable);
+		AddBinding(next, DescriptorBindingKind::FaultBuffer);
 	}
 	const bool uses_flattened_runtime =
 	    !program.srt_reads.empty() ||

src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.cpp

@@ -267,13 +267,6 @@ bool MaterializeIndirectImage(const DescriptorSource::IndirectImage& indirect,
 	return true;
 }
 
-uint64_t AddressSpecialization(const AddressResource&           resource,
-                               const ResourceSnapshot::Address& snapshot) {
-	return resource.kind == ResourceKind::Flat || resource.unbased
-	           ? snapshot.binding_base
-	           : snapshot.guest_base - snapshot.binding_base;
-}
-
 size_t FlattenedRuntimeDwords(const Program& program) {
 	size_t size = program.srt_reads.size();
 	for (uint32_t resource = 0; resource < program.info.images.size(); resource++) {
@@ -298,8 +291,7 @@ bool ValidateResourceSnapshot(const Program& program, const ResourceSnapshot& sn
 	}
 	if (snapshot.buffers.size() != program.info.buffers.size() ||
 	    snapshot.images.size() != program.info.images.size() ||
-	    snapshot.samplers.size() != program.info.samplers.size() ||
-	    snapshot.addresses.size() != program.info.addresses.size()) {
+	    snapshot.samplers.size() != program.info.samplers.size()) {
 		if (error != nullptr) {
 			*error = "resource snapshot does not match dense shader topology";
 		}
@@ -343,14 +335,6 @@ bool ValidateResourceSnapshot(const Program& program, const ResourceSnapshot& sn
 		}
 		return true;
 	};
-	for (uint32_t i = 0; i < snapshot.addresses.size(); i++) {
-		if (snapshot.addresses[i].binding_base > snapshot.addresses[i].guest_base) {
-			if (error != nullptr) {
-				*error = fmt::format("address resource {} binds above its guest base", i);
-			}
-			return false;
-		}
-	}
 	std::unordered_set<uint32_t> indirect_resources;
 	for (const auto& table: snapshot.indirect_images) {
 		const auto* source = table.resource < program.info.images.size()
@@ -526,15 +510,6 @@ bool ValidateResourceSpecialization(const Program& program, const ResourceSnapsh
 			}
 		}
 	}
-	for (uint32_t i = 0; i < program.info.addresses.size(); i++) {
-		if (program.info.addresses[i].specialized_base !=
-		    AddressSpecialization(program.info.addresses[i], snapshot.addresses[i])) {
-			if (error != nullptr) {
-				*error = fmt::format("address resource {} no longer matches specialization", i);
-			}
-			return false;
-		}
-	}
 	return true;
 }
 
@@ -550,7 +525,7 @@ bool MaterializeResources(const Program& program, const SrtRuntime& runtime,
 	std::vector<DescriptorSourceRequest> requests;
 	std::vector<uint8_t>                 clean_flat_slots(program.srt_reads.size());
 	requests.reserve(program.info.buffers.size() + program.info.images.size() * 2u +
-	                 program.info.samplers.size() + program.info.addresses.size());
+	                 program.info.samplers.size());
 	for (const auto& buffer: program.info.buffers) {
 		requests.push_back({buffer.source, buffer.first_use_pc});
 	}
@@ -576,12 +551,6 @@ bool MaterializeResources(const Program& program, const SrtRuntime& runtime,
 	for (const auto& sampler: program.info.samplers) {
 		requests.push_back({sampler.source, sampler.first_use_pc});
 	}
-	for (const auto& address: program.info.addresses) {
-		if (!address.unbased) {
-			requests.push_back({address.source, address.first_use_pc});
-		}
-	}
-
 	std::vector<DescriptorValue> values;
 	std::vector<uint32_t>        flattened_srt;
 	if (!EvaluateRuntimeSources(program, requests, runtime, values, flattened_srt, clean_flat_slots,
@@ -686,37 +655,6 @@ bool MaterializeResources(const Program& program, const SrtRuntime& runtime,
 		return false;
 	}
 	next.samplers.assign(cursor, cursor + program.info.samplers.size());
-	cursor += program.info.samplers.size();
-	for (const auto& address: program.info.addresses) {
-		if (!address.unbased) {
-			const auto value = *cursor++;
-			auto       base  = (static_cast<uint64_t>(value.dwords[0]) |
-			                    static_cast<uint64_t>(value.dwords[1]) << 32u) &
-			                   AddressMask;
-			if (address.kind == ResourceKind::ScalarAddress) {
-				base &= ~uint64_t {3};
-			}
-			const auto before = static_cast<uint64_t>(-static_cast<int64_t>(address.min_offset));
-			uint64_t   binding_base = 0;
-			if (address.kind == ResourceKind::Flat) {
-				binding_base = base & ~(FlatAddressWindowSize - 1u);
-			} else if (base >= before) {
-				binding_base = base - before;
-			}
-			next.addresses.push_back({base, binding_base});
-		} else {
-			if (!runtime.flat_memory_base.has_value()) {
-				if (error != nullptr) {
-					*error = fmt::format("unbased {} address at pc 0x{:08x} requires runtime "
-					                     "guest-address translation",
-					                     address.kind == ResourceKind::Flat ? "FLAT" : "global",
-					                     address.first_use_pc);
-				}
-				return false;
-			}
-			next.addresses.push_back({*runtime.flat_memory_base, *runtime.flat_memory_base});
-		}
-	}
 	next.user_data.assign(runtime.user_data.begin(), runtime.user_data.end());
 	if (!ValidateResourceSnapshot(program, next, error)) {
 		return false;
@@ -791,10 +729,6 @@ bool SpecializeResources(Program& program, ResourceSnapshot& snapshot, std::stri
 		next.buffers[i].descriptor_format  = descriptor.Format();
 		next.buffers[i].descriptor_swizzle = descriptor.DstSelXYZW();
 	}
-	for (uint32_t i = 0; i < next.addresses.size(); i++) {
-		next.addresses[i].specialized_base =
-		    AddressSpecialization(next.addresses[i], next_snapshot.addresses[i]);
-	}
 	for (uint32_t i = 0; i < next.images.size(); i++) {
 		const auto& descriptor = next_snapshot.images[i];
 		auto&       image      = next.images[i];

src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.h

@@ -5,15 +5,7 @@
 
 namespace Libs::Graphics::ShaderRecompiler::IR {
 
-constexpr uint64_t FlatAddressWindowSize = 0x10000;
-
 struct ResourceSnapshot {
-	struct Address {
-		uint64_t guest_base   = 0;
-		uint64_t binding_base = 0;
-
-		bool operator==(const Address& other) const = default;
-	};
 	struct IndirectImage {
 		uint32_t                     resource = 0;
 		uint32_t                     capacity = 0;
@@ -25,7 +17,6 @@ struct ResourceSnapshot {
 	std::vector<DescriptorValue> buffers;
 	std::vector<DescriptorValue> images;
 	std::vector<DescriptorValue> samplers;
-	std::vector<Address>         addresses;
 	std::vector<uint32_t>        flattened_srt;
 	std::vector<uint32_t>        user_data;
 	std::vector<IndirectImage>   indirect_images;

src/graphics/shader/recompiler/ir/passes/ResourceTracking.cpp

@@ -7,7 +7,6 @@
 #include <algorithm>
 #include <fmt/format.h>
 #include <span>
-#include <unordered_set>
 #include <utility>
 
 namespace Libs::Graphics::ShaderRecompiler::IR {
@@ -96,10 +95,10 @@ class Tracker {
 public:
 	explicit Tracker(Program& program): m_program(program), m_info(program.info) {
 		m_info.buffers.clear();
-		m_info.addresses.clear();
 		m_info.images.clear();
 		m_info.samplers.clear();
 		m_info.sampled_pairs.clear();
+		m_info.uses_dma = false;
 	}
 
 	bool Run(std::string* error) {
@@ -180,90 +179,6 @@ private:
 		                       m_program.shader_hash, StageName(m_program.stage), pc, reason));
 	}
 
-	struct AddressPart {
-		Value value;
-		bool  rooted = false;
-	};
-
-	AddressPart FindAddressPart(Value value, Value active,
-	                            std::unordered_set<const Inst*>& visiting) const {
-		value = value.Resolve();
-		if (value.IsImmediate()) {
-			return {value, false};
-		}
-		const auto* inst = value.TryInstruction();
-		if (inst == nullptr || !visiting.insert(inst).second) {
-			return {};
-		}
-		const auto finish = [&](AddressPart part) {
-			visiting.erase(inst);
-			return part;
-		};
-		std::string reason;
-		if (ValidateRuntimeValue(m_program, value, reason)) {
-			return finish({value, true});
-		}
-		const auto merge = [&](AddressPart left, AddressPart right, bool require_both) {
-			if (left.value.IsEmpty() || right.value.IsEmpty()) {
-				return require_both ? AddressPart {} : (left.value.IsEmpty() ? right : left);
-			}
-			return EquivalentValue(m_program, left.value, right.value)
-			           ? AddressPart {left.value, left.rooted || right.rooted}
-			           : AddressPart {};
-		};
-		switch (inst->GetOpcode()) {
-			case ValueOpcode::SelectU32:
-				if (EquivalentValue(m_program, inst->Arg(0), active)) {
-					return finish(FindAddressPart(inst->Arg(1), active, visiting));
-				}
-				return finish(merge(FindAddressPart(inst->Arg(1), active, visiting),
-				                    FindAddressPart(inst->Arg(2), active, visiting), true));
-			case ValueOpcode::Phi: {
-				const auto invariant = ResolveInvariantPhi(m_program, value);
-				return finish(invariant.IsEmpty() ? AddressPart {}
-				                                  : FindAddressPart(invariant, active, visiting));
-			}
-			case ValueOpcode::IAdd32:
-			case ValueOpcode::ISub32: {
-				const auto left  = FindAddressPart(inst->Arg(0), active, visiting);
-				const auto right = FindAddressPart(inst->Arg(1), active, visiting);
-				if (left.rooted == right.rooted ||
-				    (inst->GetOpcode() == ValueOpcode::ISub32 && right.rooted)) {
-					return finish({});
-				}
-				return finish(left.rooted ? left : right);
-			}
-			case ValueOpcode::CompositeExtractU32x2: {
-				const auto* source = inst->Arg(0).ResolveInstruction();
-				if (source == nullptr || source->GetOpcode() != ValueOpcode::IAddCarry32) {
-					return finish({});
-				}
-				const auto left  = FindAddressPart(source->Arg(0), active, visiting);
-				const auto right = FindAddressPart(source->Arg(1), active, visiting);
-				if (left.rooted == right.rooted) {
-					return finish({});
-				}
-				return finish(left.rooted ? left : right);
-			}
-			default: return finish({});
-		}
-	}
-
-	bool MakeFlatAddressSource(const Inst& handle, Value active,
-	                           DescriptorSource& descriptor) const {
-		std::unordered_set<const Inst*> visiting;
-		auto                            low = FindAddressPart(handle.Arg(0), active, visiting);
-		visiting.clear();
-		auto high = FindAddressPart(handle.Arg(1), active, visiting);
-		if ((!low.rooted && !high.rooted) || low.value.IsEmpty() || high.value.IsEmpty()) {
-			return false;
-		}
-		descriptor.dword_count = 2;
-		descriptor.dwords[0]   = low.value;
-		descriptor.dwords[1]   = high.value;
-		return true;
-	}
-
 	bool MakeSource(const Inst& handle, uint32_t width, bool sampler, bool sample_adjust,
 	                DescriptorSource& descriptor, uint32_t pc, std::string* error) const {
 		if (handle.NumArgs() != width) {
@@ -577,42 +492,14 @@ private:
 		return true;
 	}
 
-	bool GetAddressHandle(const Inst& memory_inst, Value value, uint32_t pc, Inst*& handle,
-	                      uint32_t& source, bool& unbased, std::string* error) {
-		handle = value.Resolve().TryInstruction();
+	bool ValidateAddressHandle(Value value, uint32_t pc, std::string* error) const {
+		const auto* handle = value.Resolve().TryInstruction();
 		if (handle == nullptr || handle->GetOpcode() != ValueOpcode::GetAddressResource) {
 			return Fail(pc, error, "address operation requires GetAddressResource");
 		}
 		if (handle->NumArgs() != 2) {
 			return Fail(pc, error, "GetAddressResource must have two address dwords");
 		}
-		DescriptorSource descriptor;
-		const auto&      memory = m_program.memory_info[memory_inst.Flags<MemoryFlags>().index];
-		if (memory.address_is_full) {
-			const auto active = memory_inst.Arg(memory_inst.NumArgs() - 1u);
-			if (memory.kind != ResourceKind::Flat ||
-			    !MakeFlatAddressSource(*handle, active, descriptor)) {
-				unbased = true;
-				source  = UINT32_MAX;
-				return true;
-			}
-		} else if (!MakeSource(*handle, 2, false, false, descriptor, pc, error)) {
-			return false;
-		}
-		std::string reason;
-		uint32_t    bad_dword = 0;
-		if (!ValidateSource(descriptor, reason, bad_dword)) {
-			if (memory.address_is_full) {
-				unbased = true;
-				source  = UINT32_MAX;
-				return true;
-			}
-			return Fail(
-			    pc, error,
-			    fmt::format("scalar memory base dword {} is unresolved: {}", bad_dword, reason));
-		}
-		unbased = false;
-		source  = InternSource(descriptor);
 		return true;
 	}
 
@@ -706,39 +593,6 @@ private:
 		return static_cast<uint32_t>(m_info.samplers.size() - 1);
 	}
 
-	uint32_t AddAddress(uint32_t source, bool unbased, const MemoryInfo& memory,
-	                    AddressAccess access, uint32_t pc) {
-		auto immediate = static_cast<int32_t>(memory.offset);
-		if (memory.kind == ResourceKind::ScalarAddress) {
-			immediate = static_cast<int32_t>(static_cast<uint32_t>(immediate) & ~3u);
-		}
-		const auto min_offset = unbased ? 0 : std::min(immediate, 0);
-		for (uint32_t i = 0; i < m_info.addresses.size(); i++) {
-			auto& address = m_info.addresses[i];
-			if (address.source == source && address.unbased == unbased &&
-			    address.kind == memory.kind) {
-				address.first_use_pc = std::min(address.first_use_pc, pc);
-				address.min_offset   = std::min(address.min_offset, min_offset);
-				address.read         = address.read || access == AddressAccess::Read;
-				address.written      = address.written || access == AddressAccess::Write;
-				return i;
-			}
-		}
-		if (m_info.addresses.size() >= ShaderInfo::MaxAddresses) {
-			return UINT32_MAX;
-		}
-		AddressResource address;
-		address.source       = source;
-		address.first_use_pc = pc;
-		address.kind         = memory.kind;
-		address.min_offset   = min_offset;
-		address.unbased      = unbased;
-		address.read         = access == AddressAccess::Read;
-		address.written      = access == AddressAccess::Write;
-		m_info.addresses.push_back(address);
-		return static_cast<uint32_t>(m_info.addresses.size() - 1);
-	}
-
 	bool AddSampledPair(uint32_t image, uint32_t sampler, uint32_t pc, std::string* error) {
 		for (auto& pair: m_info.sampled_pairs) {
 			if (pair.image == image && pair.sampler == sampler) {
@@ -839,16 +693,11 @@ private:
 				}
 				return true;
 			}
-			bool unbased = false;
-			if (!GetAddressHandle(inst, inst.Arg(0), flags.pc, handle, source, unbased, error)) {
+			if (!ValidateAddressHandle(inst.Arg(0), flags.pc, error)) {
 				return false;
 			}
-			resource = AddAddress(source, unbased, memory, address_info.access, flags.pc);
-			if (resource == UINT32_MAX) {
-				return Fail(flags.pc, error, "address resource limit exceeded");
-			}
-			return AddHandlePatch(handle, resource, flags.pc, error) &&
-			       AddMemoryPatch(flags.index, resource, 0, false, flags.pc, error);
+			m_info.uses_dma = true;
+			return true;
 		}
 
 		if (!ImageResourceKindMatches(memory.kind, image_info.resource_class)) {

src/graphics/shader/recompiler/ir/passes/SrtWalker.h

@@ -3,7 +3,6 @@
 
 #include "graphics/shader/recompiler/ir/ShaderIR.h"
 
-#include <optional>
 #include <span>
 
 namespace Libs::Graphics::ShaderRecompiler::IR {
@@ -17,7 +16,6 @@ struct SrtRuntime {
 	uint64_t                  shader_base = 0;
 	SrtMemoryReader           read_memory = nullptr;
 	void*                     userdata    = nullptr;
-	std::optional<uint64_t>   flat_memory_base;
 	SrtMemoryReader           read_specialization_memory = nullptr;
 };