diff --git a/src/core/memory.cpp b/src/core/memory.cpp index 82fac1059c..22590443fe 100644 --- a/src/core/memory.cpp +++ b/src/core/memory.cpp @@ -6,6 +6,8 @@ // SPDX-License-Identifier: GPL-2.0-or-later #include +#include +#include #include #include #include @@ -494,10 +496,58 @@ struct Memory::Impl { } } + // EDEN_NX2_MME_DMA_STATS: how often pages are protected again after an upload and + // unprotected by a CPU write, and how many distinct pages that is. + struct ProtectStats { + std::mutex mutex; + std::array calls{}; + std::array pages{}; + std::unordered_map protected_pages; + std::chrono::steady_clock::time_point last_log{std::chrono::steady_clock::now()}; + + static ProtectStats* Get() { + static const bool enabled = std::getenv("EDEN_NX2_MME_DMA_STATS") != nullptr; + static ProtectStats stats; + return enabled ? &stats : nullptr; + } + + void Note(u64 vaddr, u64 size, bool cached) { + std::scoped_lock lk{mutex}; + const u64 first = vaddr >> YUZU_PAGEBITS; + const u64 last = (vaddr + size - 1) >> YUZU_PAGEBITS; + ++calls[cached]; + pages[cached] += last - first + 1; + if (cached) { + for (u64 page = first; page <= last; ++page) { + ++protected_pages[page]; + } + } + const auto now = std::chrono::steady_clock::now(); + if (now - last_log < std::chrono::seconds{5}) { + return; + } + last_log = now; + size_t hot = 0; + for (const auto& [page, count] : protected_pages) { + hot += count >= 100 ? 1 : 0; + } + LOG_INFO(HW_Memory, + "Page protection (5 s): protect {} calls {} pages ({} distinct, {} protected " + "100+ times), unprotect {} calls {} pages", + calls[1], pages[1], protected_pages.size(), hot, calls[0], pages[0]); + calls = {}; + pages = {}; + protected_pages.clear(); + } + }; + void RasterizerMarkRegionCached(u64 vaddr, u64 size, bool cached) { if (vaddr == 0 || !AddressSpaceContains(*current_page_table, vaddr, size)) { return; } + if (auto* stats = ProtectStats::Get()) { + stats->Note(vaddr, size, cached); + } if (current_page_table->fastmem_arena) { Common::MemoryPermission perm{}; diff --git a/src/tests/video_core/buffer_cache_ports.cpp b/src/tests/video_core/buffer_cache_ports.cpp index 034167cd9f..4f1a5d0ce0 100644 --- a/src/tests/video_core/buffer_cache_ports.cpp +++ b/src/tests/video_core/buffer_cache_ports.cpp @@ -229,9 +229,6 @@ struct BufferCacheTestAccess { channel.uniform_buffers[0][0] = {.device_addr = address, .size = size, .buffer_id = id}; cache.BindHostGraphicsUniformBuffer(0, 0, 0, true); } - static void NextFrame(Cache& cache) { - cache.native_indirect_uniforms_this_frame.clear(); - } static u32 CpuMarks(Cache& cache) { return cache.memory_tracker.cpu_marks; } @@ -544,7 +541,7 @@ TEST_CASE("Command page readers remain ordered during batch completion", "[buffe REQUIRE(f.runtime.downloads == 1); } -TEST_CASE("Native uniform banks upload directly without overwriting GPU-owned bytes", +TEST_CASE("Native uniform banks mark only changed pages and keep GPU-owned bytes", "[buffer_ports]") { Fixture f; f.memory->EnableNativeTiling(); @@ -559,20 +556,26 @@ TEST_CASE("Native uniform banks upload directly without overwriting GPU-owned by std::memset(f.bytes + 0x10000, 0x5a, 0x10000); Access::Uniform(*f.cache, channel, id, 0x20000, 0x10000); REQUIRE(f.runtime.upload_bytes == 0x10000); - REQUIRE(Access::CpuMarks(*f.cache) == 0); + REQUIRE(Access::CpuMarks(*f.cache) == 16); REQUIRE(Access::Get(*f.cache, id).bytes[0xffff] == 0x5a); + const u32 uploads = f.runtime.uploads; + Access::Uniform(*f.cache, channel, id, 0x20000, 0x10000); + REQUIRE(f.runtime.uploads == uploads); + REQUIRE(Access::CpuMarks(*f.cache) == 16); + // A write the tracker missed is found on the next bind, and only its page is marked. + std::memset(f.bytes + 0x13000, 0x77, 0x1000); Access::Uniform(*f.cache, channel, id, 0x20000, 0x10000); - REQUIRE(f.runtime.uploads == 1); - Access::NextFrame(*f.cache); + REQUIRE(Access::CpuMarks(*f.cache) == 17); + REQUIRE(f.runtime.upload_bytes == 0x11000); + REQUIRE(Access::Get(*f.cache, id).bytes[0x3000] == 0x77); Access::Dirty(*f.cache, 0x21000, 0x1000); std::memset(f.bytes + 0x10000, 0xa5, 0x10000); Access::Uniform(*f.cache, channel, id, 0x20000, 0x10000); - REQUIRE(f.runtime.uploads == 1); + REQUIRE(f.runtime.upload_bytes == 0x11000); REQUIRE(Access::Get(*f.cache, id).bytes[0x1000] == 0x5a); } -TEST_CASE("Native uniform uploads stop at holes and keep unaligned bank boundaries", - "[buffer_ports]") { +TEST_CASE("Native uniform bank checks stop at holes", "[buffer_ports]") { Fixture f; f.memory->EnableNativeTiling(); Tegra::Control::ChannelState state{0}; @@ -586,20 +589,18 @@ TEST_CASE("Native uniform uploads stop at holes and keep unaligned bank boundari std::memset(f.bytes + 0x10000, 0x6b, 0x20000); f.device.Unmap(0x22000, 0x1000); Access::Uniform(*f.cache, channel, id, 0x20123, 0x10000); - REQUIRE(f.runtime.upload_bytes == 0x1edd); + REQUIRE(Access::CpuMarks(*f.cache) == 2); + REQUIRE(f.runtime.upload_bytes == 0x2000); const auto& data = Access::Get(*f.cache, id).bytes; - REQUIRE(data[0x122] == 0); REQUIRE(data[0x123] == 0x6b); REQUIRE(data[0x1fff] == 0x6b); REQUIRE(data[0x2000] == 0); REQUIRE(data[0x3000] == 0); - REQUIRE(Access::CpuMarks(*f.cache) == 0); f.device.Map(0x22000, 0x22000, 0x1000, f.asid); - Access::NextFrame(*f.cache); Access::Uniform(*f.cache, channel, id, 0x20123, 0x10000); - REQUIRE(f.runtime.upload_bytes == 0x11edd); + REQUIRE(Access::CpuMarks(*f.cache) == 17); + REQUIRE(data[0x2000] == 0x6b); REQUIRE(data[0x10122] == 0x6b); - REQUIRE(data[0x10123] == 0); } TEST_CASE("Downloads keep CPU writes collected and not yet applied", "[buffer_ports]") { diff --git a/src/tests/video_core/maxwell_fast_method.cpp b/src/tests/video_core/maxwell_fast_method.cpp index 71f69ae48f..07a30d6c5c 100644 --- a/src/tests/video_core/maxwell_fast_method.cpp +++ b/src/tests/video_core/maxwell_fast_method.cpp @@ -3,6 +3,7 @@ #include #include +#include #include #include #include @@ -12,6 +13,7 @@ #include #include +#include #include "common/page_table.h" #include "common/scope_exit.h" @@ -837,3 +839,85 @@ TEST_CASE("Maxwell3D's split front uploads macro code sent as one multi-method r REQUIRE(macros.native_write_offset == pointer * 3 + code.size()); CHECK(std::equal(code.begin(), code.end(), macros.native_code.begin() + pointer * 3)); } + +TEST_CASE("Split front orders queued draws before reads at the next wait for idle", + "[video_core][gpu_split_dependencies]") { + using Access = VideoCommon::GpuSplitTestAccess; + constexpr u32 WaitForIdle = MAXWELL3D_REG_INDEX(wait_for_idle); + constexpr GPUVAddr address = 0x100000; + // Instance id in draw.begin bits 26-27: a plain draw, or the first draw of an instanced + // loop that the back defers until the wait for idle flushes it. + const u32 begin = GENERATE(0u, 1u << 26); + CAPTURE(begin); + Fixture f; + VideoCommon::GpuSplit split; + Access::Initialize(split, f.system); + auto& engine = f.fast; + const auto ready = [&] { return split.ReadReadyPosition(&f.memory, address, address + 4); }; + + // A wait for idle without draws orders nothing. + engine.FrontCallMethod(f.system, split, WaitForIdle, 0, true); + REQUIRE(ready() == 0); + // The back has executed nothing, so nothing has told the front this draw writes memory. + engine.FrontCallMethod(f.system, split, DrawBegin, begin, true); + engine.FrontCallMethod(f.system, split, DrawEnd, 0, true); + // Hardware orders macro reads after draws only across a wait for idle. + REQUIRE(ready() == 0); + engine.FrontCallMethod(f.system, split, WaitForIdle, 0, true); + REQUIRE(ready() == Access::End(split)); + REQUIRE(split.WriterPosition() == Access::End(split)); + // Recorded once: a second wait for idle does not move it. + const u64 after_draw = Access::End(split); + engine.FrontCallMethod(f.system, split, WaitForIdle, 0, true); + REQUIRE(ready() == after_draw); +} + +TEST_CASE("Split host syncs order queued draws as waits for idle do", + "[video_core][gpu_split_dependencies]") { + using Access = VideoCommon::GpuSplitTestAccess; + Fixture f; + VideoCommon::GpuSplit split; + Access::Initialize(split, f.system); + split.NoteSync(); + REQUIRE(split.WriterPosition() == 0); + f.fast.FrontCallMethod(f.system, split, DrawBegin, 0, true); + f.fast.FrontCallMethod(f.system, split, DrawEnd, 0, true); + REQUIRE(split.WriterPosition() == 0); + split.NoteSync(); + REQUIRE(split.WriterPosition() == Access::End(split)); +} + +TEST_CASE("Split front read after a draw and a wait for idle waits for a delayed back", + "[video_core][gpu_split_dependencies]") { + using Access = VideoCommon::GpuSplitTestAccess; + constexpr GPUVAddr address = 0x100000; + Fixture f; + f.system.Initialize(); + VideoCommon::GpuSplit split; + Access::Start(split, f.system); + SCOPE_EXIT { split.Stop(); }; + auto& engine = f.fast; + // The front drains once as it starts: start it before the back stalls. + engine.FrontCallMethod(f.system, split, f.state_methods.front(), 0x1234, true); + std::atomic release{false}; + std::atomic drawn{false}; + // The back stalls before the first draw it executes, as it can behind real work. + split.Defer([&release] { + while (!release.load(std::memory_order_acquire)) { + std::this_thread::yield(); + } + }); + engine.FrontCallMethod(f.system, split, DrawBegin, 0, true); + engine.FrontCallMethod(f.system, split, DrawEnd, 0, true); + // Stands for the draw's shader stores. + split.Defer([&drawn] { drawn.store(true, std::memory_order_release); }); + engine.FrontCallMethod(f.system, split, MAXWELL3D_REG_INDEX(wait_for_idle), 0, true); + const u64 target = split.ReadReadyPosition(&f.memory, address, address + 4); + REQUIRE(target > split.Consumed()); + std::jthread releaser([&release] { + std::this_thread::sleep_for(std::chrono::milliseconds{20}); + release.store(true, std::memory_order_release); + }); + split.WaitForPosition(target, VideoCommon::DrainSite::MacroDmaRead); + REQUIRE(drawn.load(std::memory_order_acquire)); +} diff --git a/src/tests/video_core/uniform_stream_reuse.cpp b/src/tests/video_core/uniform_stream_reuse.cpp index 6059b70eeb..bcfee7e35c 100644 --- a/src/tests/video_core/uniform_stream_reuse.cpp +++ b/src/tests/video_core/uniform_stream_reuse.cpp @@ -23,6 +23,10 @@ struct Memory { Check(address + size <= bytes.size(), "read outside guest memory"); std::memcpy(destination, bytes.data() + address, size); } + template + T* GetPointer(u64 address) { + return address < bytes.size() ? reinterpret_cast(bytes.data() + address) : nullptr; + } }; struct Runtime { diff --git a/src/video_core/buffer_cache/buffer_cache.h b/src/video_core/buffer_cache/buffer_cache.h index bf5aaabbba..74219b6e6d 100644 --- a/src/video_core/buffer_cache/buffer_cache.h +++ b/src/video_core/buffer_cache/buffer_cache.h @@ -112,7 +112,6 @@ void BufferCache

::TickFrame() { return; } runtime.TickFrame(slot_buffers); - native_indirect_uniforms_this_frame.clear(); if (frame_tick % 600 == 0 && !uploaded_buffer_ranges.empty()) { for (const auto& [address, size] : uploaded_buffer_ranges) { device_memory.HintCold(address, size); @@ -1397,26 +1396,62 @@ void BufferCache

::BindHostGraphicsUniformBuffer(size_t stage, u32 index, u32 }(); // Native programs keep tables next to their code in a 64 KiB bank read with indirect offsets // (Splatoon Raiders: the dialogue box corners in c1). Writes to that memory can go unnoticed by - // the memory tracker, leaving stale host contents, so upload such a bank again once per frame. + // the memory tracker, leaving stale host contents, so check such a bank on every bind: games + // rewrite per-draw data between draws, and checking once per frame left it stale for a frame + // now and then (white flashes in Ocarina of Time, Eglantine 031e07436c). if (has_host_buffer && (*channel_state->uniform_buffer_sizes)[stage][index] == 0x10000 && size != 0 && gpu_memory != nullptr && gpu_memory->UsesNativeTiling() && - (Nx2Perf::Verify() || !native_indirect_uniforms_this_frame.contains(device_addr)) && - !memory_tracker.IsRegionGpuModified(device_addr, size) && - native_indirect_uniforms_this_frame.insert(device_addr).second) { + !memory_tracker.IsRegionGpuModified(device_addr, size)) { // The bank size is only an upper limit: stop at the end of the mapped memory. u64 mapped_size = 0; while (mapped_size < size && device_memory.template GetPointer(device_addr + mapped_size) != nullptr) { mapped_size += Core::DEVICE_PAGESIZE - ((device_addr + mapped_size) & Core::DEVICE_PAGEMASK); } - const u64 upload_size = (std::min)(mapped_size, size); - if (upload_size != 0) { - std::array copies{BufferCopy{ - .src_offset = 0, - .dst_offset = buffer.Offset(device_addr), - .size = upload_size, - }}; - UploadMemory(buffer, upload_size, upload_size, copies); + const u64 check_size = (std::min)(mapped_size, size); + // Banks slide through larger buffers, a new address per draw: compare each page with + // its last contents and mark only the pages that changed. + if (native_indirect_page_snapshots.size() >= 16384) { + native_indirect_page_snapshots.clear(); + } + const DAddr first_page = Common::AlignDown(device_addr, Core::DEVICE_PAGESIZE); + const DAddr end = device_addr + check_size; + for (DAddr page = first_page; page < end; page += Core::DEVICE_PAGESIZE) { + const u8* const guest = device_memory.template GetPointer(page); + if (guest == nullptr) { + continue; + } + auto [it, inserted] = native_indirect_page_snapshots.try_emplace(page); + const bool same = + !inserted && std::memcmp(it->second.data(), guest, Core::DEVICE_PAGESIZE) == 0; + // EDEN_NX2_MME_DMA_STATS: how many checked pages had changed. + if (static const bool stats = std::getenv("EDEN_NX2_MME_DMA_STATS") != nullptr; + stats) { + static std::array counts{}; + static std::array changed_state{}; + ++counts[same ? 0 : (inserted ? 1 : 2)]; + if (!same && !inserted) { + // Did the tracker already know (CPU-modified), or was the page clean, and + // was it GPU-modified? + const bool known = + memory_tracker.IsRegionCpuModified(page, Core::DEVICE_PAGESIZE); + const bool gpu = + memory_tracker.IsRegionGpuModified(page, Core::DEVICE_PAGESIZE); + ++changed_state[(known ? 1 : 0) | (gpu ? 2 : 0)]; + } + if (((counts[0] + counts[1] + counts[2]) & 0x3ffff) == 0) { + LOG_INFO(HW_GPU, + "Native bank pages checked: same {}, new {}, changed {} (tracker " + "clean {}, CPU-modified {}, GPU {}, both {})", + counts[0], counts[1], counts[2], changed_state[0], + changed_state[1], changed_state[2], changed_state[3]); + } + } + if (same) { + continue; + } + std::memcpy(it->second.data(), guest, Core::DEVICE_PAGESIZE); + memory_tracker.MarkRegionAsCpuModified(page, Core::DEVICE_PAGESIZE); } } const bool use_fast_buffer = needs_alignment_stream diff --git a/src/video_core/buffer_cache/buffer_cache_base.h b/src/video_core/buffer_cache/buffer_cache_base.h index 2afb361c40..b340f746f5 100644 --- a/src/video_core/buffer_cache/buffer_cache_base.h +++ b/src/video_core/buffer_cache/buffer_cache_base.h @@ -463,8 +463,10 @@ class BufferCache : public VideoCommon::ChannelSetupCaches native_indirect_uniforms_this_frame; + /// The guest words of each page of native programs' indirectly read banks when last checked: + /// unchanged pages keep their tracking (marking them CPU-modified costs two mprotect calls + /// per page). + std::unordered_map> native_indirect_page_snapshots; private: template @@ -513,6 +515,7 @@ class BufferCache : public VideoCommon::ChannelSetupCaches } Reuse& reuse = UniformStreamReuseState(); typename Reuse::Slot& slot = reuse.slots[stage][index]; - u8* const scratch = reuse.scratch.data(); - device_memory.ReadBlockUnsafe(device_addr, scratch, size); + // Within one device page the guest bytes are contiguous: use them in place instead of + // copying them to the scratch first. + constexpr u64 PageMask = (u64{1} << 12) - 1; + const u8* scratch = nullptr; + if ((device_addr & PageMask) + size <= PageMask + 1) { + scratch = device_memory.template GetPointer(device_addr); + } + if (scratch == nullptr) { + device_memory.ReadBlockUnsafe(device_addr, reuse.scratch.data(), size); + scratch = reuse.scratch.data(); + } const bool same_bytes = slot.size == size && std::memcmp(slot.bytes.data(), scratch, size) == 0; @@ -106,15 +115,17 @@ template runtime.RebindStreamUniform(slot.ref, size); return true; } - const std::span span = runtime.BindMappedUniformBuffer(stage, binding_index, size); - std::memcpy(span.data(), scratch, size); - slot.ref = runtime.LastStreamUniformRef(); - // After the allocation: the tick the stream region is tagged with is the one current now. - slot.tick = runtime.CurrentTick(); + // Upload the slot's copy: guest memory may change while this runs, and a later reuse must + // find exactly the bytes that were uploaded. if (!same_bytes) { slot.size = size; std::memcpy(slot.bytes.data(), scratch, size); } + const std::span span = runtime.BindMappedUniformBuffer(stage, binding_index, size); + std::memcpy(span.data(), slot.bytes.data(), size); + slot.ref = runtime.LastStreamUniformRef(); + // After the allocation: the tick the stream region is tagged with is the one current now. + slot.tick = runtime.CurrentTick(); return true; } diff --git a/src/video_core/dma_pusher.cpp b/src/video_core/dma_pusher.cpp index af8fca936e..a5299d9909 100644 --- a/src/video_core/dma_pusher.cpp +++ b/src/video_core/dma_pusher.cpp @@ -44,6 +44,35 @@ bool DmaPusher::IsMemoryWriter(const Engines::EngineInterface& engine, Engines:: } return false; } +namespace { +// Host methods that can write guest memory without noting it themselves. +bool PullerMethodWritesMemory(u32 method) { + switch (static_cast(method)) { + case BufferMethods::BindObject: + case BufferMethods::Nop: + case BufferMethods::SemaphoreAddressHigh: + case BufferMethods::SemaphoreAddressLow: + case BufferMethods::SemaphoreSequencePayload: + case BufferMethods::SemaphoreOperation: // Releases note their own writes (Puller) + case BufferMethods::SemaphoreRelease: + case BufferMethods::NonStallInterrupt: + case BufferMethods::WrcacheFlush: + case BufferMethods::MemOpA: + case BufferMethods::MemOpB: + case BufferMethods::MemOpC: + case BufferMethods::MemOpD: + case BufferMethods::RefCnt: + case BufferMethods::SemaphoreAcquire: // Ampere: SEM_PAYLOAD_HI, like 0x17..0x19 + case BufferMethods::SyncpointPayload: + case BufferMethods::SyncpointOperation: + case BufferMethods::WaitForIdle: + case BufferMethods::Yield: + return false; + default: + return !(method >= 0x17 && method <= 0x19); + } +} +} // namespace DmaPusher::DmaPusher(Core::System& system_, MemoryManager& memory_manager_, Control::ChannelState& channel_state_) : system{system_} @@ -198,8 +227,10 @@ bool DmaPusher::Step() { // (the split only starts with unsafe reads; the settings can change mid-game). const bool use_safe = !VideoCommon::GpuSplit::Get().Active() && (Settings::IsDMALevelDefault() ? Settings::IsGPULevelHigh() : Settings::IsDMALevelSafe()); - if (!gpu_written && VideoCommon::GpuSplit::Get().Active() && - VideoCommon::GpuSplit::Get().WaitForWriters()) { + // Recheck even when the wait returns at once: a writer may have finished (and marked the + // pages) since the check above. + if (!gpu_written && VideoCommon::GpuSplit::Get().Active()) { + VideoCommon::GpuSplit::Get().WaitForWriters(); gpu_written = IsGpuWrittenCommands(dma_state.dma_get, size_t{header.size} * sizeof(u32)); } @@ -463,6 +494,9 @@ void DmaPusher::CallMethod(u32 argument) { dma_state.subchannel, dma_state.method_count, }); + if (PullerMethodWritesMemory(dma_state.method)) { + split.NoteMemoryBarrier(0x10000 | dma_state.method); + } } else { auto subchannel = subchannels[dma_state.subchannel]; if (subchannel == nullptr) { @@ -472,9 +506,16 @@ void DmaPusher::CallMethod(u32 argument) { const auto engine_type = subchannel_type[dma_state.subchannel]; const bool writer = split.Active() && IsMemoryWriter(*subchannel, engine_type, dma_state.method); + // Other engines' memory writes are not tracked by address (Maxwell3D::FrontDmaRead). + const bool untracked_writes = engine_type != Engines::EngineTypes::Maxwell3D && + engine_type != Engines::EngineTypes::KeplerCompute && + engine_type != Engines::EngineTypes::MaxwellDMA; SCOPE_EXIT { if (writer) { split.NoteWriter(); + } else if (untracked_writes) { + split.NoteMemoryBarrier(0x20000 | (static_cast(engine_type) << 12) | + (dma_state.method & 0xfff)); } }; if (split.Active()) { @@ -521,6 +562,12 @@ void DmaPusher::CallMultiMethod(const u32* base_start, u32 num_methods) { auto& split = VideoCommon::GpuSplit::Get(); if (dma_state.method < non_puller_methods) { puller.CallMultiMethod(*this, dma_state.method, dma_state.subchannel, base_start, num_methods, dma_state.method_count); + for (u32 i = 0; i < num_methods; ++i) { + if (PullerMethodWritesMemory(dma_state.method + (dma_state.non_incrementing ? 0 : i))) { + split.NoteMemoryBarrier(0x10000 | dma_state.method); + break; + } + } } else { auto subchannel = subchannels[dma_state.subchannel]; if (subchannel == nullptr) { @@ -530,9 +577,16 @@ void DmaPusher::CallMultiMethod(const u32* base_start, u32 num_methods) { const auto engine_type = subchannel_type[dma_state.subchannel]; const bool writer = split.Active() && IsMemoryWriter(*subchannel, engine_type, dma_state.method); + // Other engines' memory writes are not tracked by address (Maxwell3D::FrontDmaRead). + const bool untracked_writes = engine_type != Engines::EngineTypes::Maxwell3D && + engine_type != Engines::EngineTypes::KeplerCompute && + engine_type != Engines::EngineTypes::MaxwellDMA; SCOPE_EXIT { if (writer) { split.NoteWriter(); + } else if (untracked_writes) { + split.NoteMemoryBarrier(0x20000 | (static_cast(engine_type) << 12) | + (dma_state.method & 0xfff)); } }; if (split.Active()) { diff --git a/src/video_core/engines/maxwell_3d.cpp b/src/video_core/engines/maxwell_3d.cpp index 29a9ceab3d..99b3a15987 100644 --- a/src/video_core/engines/maxwell_3d.cpp +++ b/src/video_core/engines/maxwell_3d.cpp @@ -11,6 +11,8 @@ #include #include #include +#include +#include #include "common/assert.h" #include "common/bit_util.h" #include "common/scope_exit.h" @@ -1043,6 +1045,11 @@ struct Maxwell3D::FrontState { /// then can it have a deferred instanced draw to issue at the end of a macro. bool draw_pending{true}; bool skip_same{}; ///< SkipSameValueWrites() && FrontSkipSame(), read once + u32 dma_reads{}; + /// The inline-to-memory upload being sent writes [upload_begin, upload_end). + bool upload_tracked{}; + GPUVAddr upload_begin{}; + GPUVAddr upload_end{}; /// The back's draw manager is in an instance loop (DrawMode::Instance): draw.end does nothing. bool instance_loop{}; }; @@ -1056,6 +1063,33 @@ bool FrontVerifyEnabled() { return verify; } +// Draws and waits for idle in [method, method + amount), for the split's draw write tracking. +// draw.begin counts too: in an instance loop the front skips draw.end and the back defers the +// draw until something flushes it. +void FrontNoteDraws(VideoCommon::GpuSplit& split, u32 method, u32 amount) { + const auto covers = [method, amount](u32 index) { + return method <= index && index < method + amount; + }; + if (covers(MAXWELL3D_REG_INDEX(draw.begin)) || covers(MAXWELL3D_REG_INDEX(draw.end)) || covers(MAXWELL3D_REG_INDEX(draw_inline_index)) || + covers(MAXWELL3D_REG_INDEX(inline_index_2x16.even)) || + covers(MAXWELL3D_REG_INDEX(inline_index_4x8.index0)) || + covers(MAXWELL3D_REG_INDEX(vertex_array_instance_first)) || + covers(MAXWELL3D_REG_INDEX(vertex_array_instance_subsequent)) || + covers(MAXWELL3D_REG_INDEX(draw_texture.src_y0)) || + covers(MAXWELL3D_REG_INDEX(index_buffer32_first)) || + covers(MAXWELL3D_REG_INDEX(index_buffer16_first)) || + covers(MAXWELL3D_REG_INDEX(index_buffer8_first)) || + covers(MAXWELL3D_REG_INDEX(index_buffer32_subsequent)) || + covers(MAXWELL3D_REG_INDEX(index_buffer16_subsequent)) || + covers(MAXWELL3D_REG_INDEX(index_buffer8_subsequent))) { + split.NoteDraw(); + } + // Sent after the draws, so a deferred draw it flushes on the back counts too. + if (covers(MAXWELL3D_REG_INDEX(wait_for_idle))) { + split.NoteIdle(); + } +} + // Executable methods ProcessMethodCall handles itself; the rest go to the draw manager. bool FrontHandledOutsideDrawManager(u32 method) { constexpr u32 cb_data = MAXWELL3D_REG_INDEX(const_buffer.buffer); @@ -1213,6 +1247,113 @@ void Maxwell3D::FrontWrite(VideoCommon::GpuSplit& split, FrontState& fs, u32 met } else { split.Method(this, method, argument, is_last_call, nullptr); } + if (execution_mask[method]) { + FrontNoteWrites(split, fs, method, 1, &argument); + FrontNoteDraws(split, method, 1); + } +} + +template +bool Maxwell3D::DeviceWords(GPUVAddr begin, u32 count, Func&& func) { + GPUVAddr page = ~GPUVAddr{}; + std::optional page_device; + for (u32 i = 0; i < count; ++i) { + const GPUVAddr address = begin + u64{i} * sizeof(u32); + if ((address & ~GPUVAddr{0xfff}) != page) { + page = address & ~GPUVAddr{0xfff}; + page_device = memory_manager.GpuToCpuAddress(page); + } + const std::optional device = + page_device ? std::optional{*page_device + (address & 0xfff)} : std::nullopt; + if (!func(i, device)) { + return false; + } + } + return true; +} + +void Maxwell3D::FrontNoteWrites(VideoCommon::GpuSplit& split, FrontState& fs, u32 method, + u32 amount, const u32* values) { + constexpr u32 cb_data = MAXWELL3D_REG_INDEX(const_buffer.buffer); + const auto covers = [method, amount](u32 index) { + return method <= index && index < method + amount; + }; + const auto address = [&fs](u32 high, u32 low) { + return (GPUVAddr{fs.regs[high]} << 32) | fs.regs[low]; + }; + const auto note = [&split, this](GPUVAddr begin, GPUVAddr end) { + split.NoteMemoryWrite(&memory_manager, begin, end, true); + }; + if (method >= cb_data && method < cb_data + 16) { + const GPUVAddr base = address(MAXWELL3D_REG_INDEX(const_buffer.address_high), + MAXWELL3D_REG_INDEX(const_buffer.address_low)); + const u64 offset_end = fs.regs[MAXWELL3D_REG_INDEX(const_buffer.offset)]; + const u64 bytes = u64{amount} * sizeof(u32); + if (offset_end < bytes) { + split.NoteMemoryBarrier(0x30001); + return; + } + const GPUVAddr begin = base + offset_end - bytes; + // In Replay the back writes the shadow RAM values, not these words: reads wait for it. + const bool replay = fs.control == Regs::ShadowRamControl::Replay; + split.NoteMemoryWrite(&memory_manager, begin, base + offset_end, replay); + if (replay) { + return; + } + // The words themselves, for MME DMA reads of them while the back has not written them. + auto& front_words = split.FrontWords(); + if (front_words.size() >= 16384) { + const u64 done = split.Consumed(); + std::erase_if(front_words, [done](const auto& entry) { return entry.second.position <= done; }); + if (front_words.size() >= 16384) { + front_words.clear(); + } + } + // Only the words ProcessCBMultiData writes: it drops those past the buffer's end. + const u64 offset = offset_end - bytes; + const u64 size = fs.regs[MAXWELL3D_REG_INDEX(const_buffer.size)]; + const u64 room = offset < size ? size - offset : 0; + const u32 count = static_cast(std::min(amount, room / sizeof(u32))); + const u64 position = split.Position(); + DeviceWords(begin, count, [&](u32 i, std::optional device) { + if (device) { + front_words[*device] = {values[i], position}; + } + return true; + }); + return; + } + if (covers(MAXWELL3D_REG_INDEX(launch_dma))) { + // Inline-to-memory: a pitch-linear upload writes its lines from the destination address; + // a block-linear one or a release on completion is not tracked. + const u32 launch = fs.regs[MAXWELL3D_REG_INDEX(launch_dma)]; + const bool pitch_layout = (launch & 1) != 0; // memory_layout: Pitch + const bool release = ((launch >> 4) & 3) == 2; // completion_type: Release + const u64 line_length = fs.regs[MAXWELL3D_REG_INDEX(upload.line_length_in)]; + const u64 lines = fs.regs[MAXWELL3D_REG_INDEX(upload.line_count)]; + const u64 pitch = fs.regs[MAXWELL3D_REG_INDEX(upload.dest.pitch)]; + fs.upload_tracked = pitch_layout && !release && lines != 0; + if (fs.upload_tracked) { + fs.upload_begin = address(MAXWELL3D_REG_INDEX(upload.dest.address_high), + MAXWELL3D_REG_INDEX(upload.dest.address_low)); + fs.upload_end = fs.upload_begin + (std::max)(pitch, line_length) * (lines - 1) + line_length; + } else { + split.NoteMemoryBarrier(0x30002); + } + } + if (covers(MAXWELL3D_REG_INDEX(inline_data))) { + if (fs.upload_tracked) { + note(fs.upload_begin, fs.upload_end); + } else { + split.NoteMemoryBarrier(0x30003); + } + } + if (covers(MAXWELL3D_REG_INDEX(report_semaphore.query))) { + // A release writes its payload, or a 16-byte report, at the semaphore address. + const GPUVAddr begin = address(MAXWELL3D_REG_INDEX(report_semaphore.address_high), + MAXWELL3D_REG_INDEX(report_semaphore.address_low)); + note(begin, begin + 16); + } } void Maxwell3D::FrontCallMethod(Core::System& system, VideoCommon::GpuSplit& split, u32 method, @@ -1320,6 +1461,8 @@ void Maxwell3D::FrontCallMultiMethod(Core::System& system, VideoCommon::GpuSplit } } split.MultiMethod(this, method, base_start, amount, methods_pending, nullptr); + FrontNoteWrites(split, fs, method, amount, base_start); + FrontNoteDraws(split, method, amount); } void Maxwell3D::FrontRunMacro(Core::System& system, VideoCommon::GpuSplit& split, @@ -1375,7 +1518,9 @@ void Maxwell3D::FrontRunMacro(Core::System& system, VideoCommon::GpuSplit& split } } -void Maxwell3D::FrontTrackInstanceLoop(FrontState& fs, u32 method, u32 value) { +// Not inlined: GCC 13 -O3 unswitches FrontCallMultiMethod's loop on this switch and drops the +// macro upload cases from the loop the other methods take (an empty macro RAM on the front). +[[gnu::noinline]] void Maxwell3D::FrontTrackInstanceLoop(FrontState& fs, u32 method, u32 value) { switch (method) { case MAXWELL3D_REG_INDEX(draw.begin): // Instance id, bits 27:26: First leaves an instance loop, Subsequent enters one. @@ -1428,8 +1573,100 @@ void Maxwell3D::FrontDmaRead(VideoCommon::GpuSplit& split, FrontState& fs, u32 m memory_manager.ReadBlockUnsafe(source, diag_before.data(), read_bytes); } VideoCommon::Nx2Profile::CountDmaRead(macro); - split.Drain(VideoCommon::DrainSite::MacroDmaRead); - if (VideoCommon::QueryDma::TraceEnabled()) { + // Wait only for queued work that may write the words: compute, copies, untracked + // writes and overlapping inline constant buffer data. Ocarina of Time reads ~20,000 times a + // second; a full drain each time kept the two GPU threads waiting on each other. Every 4096th + // read still drains and checks for GPU-written data, which turns the precise reads off. + static bool precise_disabled{}; + const bool precise = !precise_disabled && + !VideoCommon::QueryDma::TraceEnabled() && + !Tegra::Nx2CmdPageDiag::g_enabled && + fs.regs[MAXWELL3D_REG_INDEX(transform_feedback_enabled)] == 0 && + (++fs.dma_reads & 4095) != 0; + static thread_local std::vector shadow_words; + shadow_words.clear(); + if (precise) { + const u64 target = split.ReadReadyPosition(&memory_manager, source, source + read_bytes); + const u64 consumed = split.Consumed(); + // Constant buffer words sent inline after any other write to them are known here: + // read them without waiting for the back to store them. + if (target > consumed) { + const u64 opaque = + split.OpaqueReadyPosition(&memory_manager, source, source + read_bytes); + shadow_words.clear(); + const bool complete = DeviceWords(source, words, [&](u32, std::optional device) { + const auto& front_words = split.FrontWords(); + const auto it = device ? front_words.find(*device) : front_words.end(); + // Only words still queued: memory has the others, maybe rewritten by the CPU. + if (it == front_words.end() || it->second.position < opaque || + it->second.position <= consumed) { + return false; + } + shadow_words.push_back(it->second.value); + return true; + }); + if (!complete) { + shadow_words.clear(); + } + } + const bool from_shadow = !shadow_words.empty(); + // EDEN_NX2_MME_DMA_STATS: why precise reads wait (writer = compute or copy work). + if (static const bool stats = std::getenv("EDEN_NX2_MME_DMA_STATS") != nullptr; stats) { + static std::array causes{}; + const u64 writer = split.WriterPosition(); + const u64 barrier = split.MemoryBarrierPosition(); + const size_t cause = target <= consumed || from_shadow ? 0 + : target == writer ? 1 + : target == barrier ? 2 + : 3; + ++causes[cause]; + static std::unordered_map sources; + if (cause == 2) { + ++sources[split.MemoryBarrierSource()]; + } + if (((causes[0] + causes[1] + causes[2] + causes[3]) & 0xffff) == 0) { + LOG_INFO(HW_GPU, "MME DMA reads: no wait {}, writer {}, barrier {}, constant buffer {}", + causes[0], causes[1], causes[2], causes[3]); + for (const auto& [barrier_source, barrier_count] : sources) { + LOG_INFO(HW_GPU, "MME DMA read barrier source {:#x}: {}", barrier_source, + barrier_count); + } + } + } + if (target > consumed && !from_shadow) { + split.WaitForPosition(target, VideoCommon::DrainSite::MacroDmaRead); + } + } else { + split.Drain(VideoCommon::DrainSite::MacroDmaRead); + } + // GPU-written words may still be only in a host buffer: the back checks and downloads them. + if (precise && shadow_words.empty() && words != 0) { + bool gpu_written = false; + for (GPUVAddr page = source & ~GPUVAddr{0xfff}; page < source + read_bytes && !gpu_written; + page += 0x1000) { + const auto device = memory_manager.GpuToCpuAddress(page); + gpu_written = device && Tegra::Nx2IsGpuWritten(*device); + } + if (gpu_written) { + if (static const bool stats = std::getenv("EDEN_NX2_MME_DMA_STATS") != nullptr; stats) { + static u64 checks{}; + if ((++checks & 0x3ff) == 0) { + LOG_INFO(HW_GPU, "MME DMA reads of GPU-written pages checked on the back: {}", + checks); + } + } + split.Run( + [memory = &memory_manager, source, read_bytes] { + if (memory->IsMemoryDirty(source, read_bytes, + VideoCommon::CacheType::BufferCache)) { + memory->FlushRegion(source, read_bytes, + VideoCommon::CacheType::BufferCache); + } + }, + VideoCommon::DrainSite::MacroDmaFlush); + } + } + if (!precise && VideoCommon::QueryDma::TraceEnabled()) { split.Run( [this, macro, method, source, read_bytes] { if (VideoCommon::QueryDma::HasPendingQueries(memory_manager, *rasterizer, source, @@ -1443,22 +1680,56 @@ void Maxwell3D::FrontDmaRead(VideoCommon::GpuSplit& split, FrontState& fs, u32 m if (VideoCommon::Nx2Profile::Enabled()) { VideoCommon::Nx2Profile::Add(VideoCommon::Nx2Profile::Counter::DmaReadBytes, read_bytes); } - if (memory_manager.IsMemoryDirty(source, read_bytes, VideoCommon::CacheType::BufferCache)) { + // The caches belong to the back, which may still be running after a precise wait. + if (!precise && + memory_manager.IsMemoryDirty(source, read_bytes, VideoCommon::CacheType::BufferCache)) { + if (!precise_disabled) { + precise_disabled = true; + LOG_WARNING(HW_GPU, "Native MME DMA read of GPU-written memory at {:#x}: precise reads off", + source); + } split.Run( [memory = &memory_manager, source, read_bytes] { memory->FlushRegion(source, read_bytes, VideoCommon::CacheType::BufferCache); }, VideoCommon::DrainSite::MacroDmaFlush); } - // Write out the reports held in the query cache, or the read returns stale occlusion counts. - split.Run( - [this, source, read_bytes] { - VideoCommon::QueryDma::FlushPending(memory_manager, *rasterizer, source, read_bytes); - }, - VideoCommon::DrainSite::MacroDmaFlush); + // A read of reports the query cache still holds writes them out first, waiting for their + // host results (Lichii #215), or the guest reads stale occlusion counts. The wait above ran + // the queued reports the read may see, so the back has registered them. + const bool reads_reports = [&] { + if (!shadow_words.empty() || words == 0) { + return false; + } + for (GPUVAddr begin = source; begin < source + read_bytes;) { + const GPUVAddr end = std::min((begin | 0xfff) + 1, source + read_bytes); + const auto device = memory_manager.GpuToCpuAddress(begin); + if (device && Tegra::Nx2ReportsPending(*device, end - begin)) { + return true; + } + begin = end; + } + return false; + }(); + if (static const bool stats = std::getenv("EDEN_NX2_MME_DMA_STATS") != nullptr; stats) { + static u64 reads{}, flushes{}; + flushes += reads_reports ? 1 : 0; + if ((++reads & 0xffff) == 0) { + LOG_INFO(HW_GPU, "MME DMA reads: {} of {} flushed pending reports", flushes, reads); + } + } + if (reads_reports) { + split.Run( + [this, source, read_bytes] { + VideoCommon::QueryDma::FlushPending(memory_manager, *rasterizer, source, read_bytes); + }, + VideoCommon::DrainSite::MacroDmaFlush); + } u32 changed{}; for (u32 i = 0; i < words; ++i) { - const u32 word = memory_manager.Read(source + u64{i} * sizeof(u32)); + const u32 word = !shadow_words.empty() + ? shadow_words[i] + : memory_manager.Read(source + u64{i} * sizeof(u32)); if (method == 0x158) { fs.parameters.push_back(word); } else if (ram + i < native_data_ram.size()) { diff --git a/src/video_core/engines/maxwell_3d.h b/src/video_core/engines/maxwell_3d.h index 97790bbcbd..8fb844bb56 100644 --- a/src/video_core/engines/maxwell_3d.h +++ b/src/video_core/engines/maxwell_3d.h @@ -3405,6 +3405,11 @@ class Maxwell3D final : public EngineInterface { bool FrontSkipsLoopWrite(FrontState& state, u32 method, u32 value, bool shadow_kept); void FrontDmaRead(VideoCommon::GpuSplit& split, FrontState& state, u32 macro, u32 method, u32 count); + void FrontNoteWrites(VideoCommon::GpuSplit& split, FrontState& state, u32 method, u32 amount, + const u32* values); + /// Calls func(index, device address) for `count` words from `begin`, while it returns true. + template + bool DeviceWords(GPUVAddr begin, u32 count, Func&& func); void FrontVerify(VideoCommon::GpuSplit& split, FrontState& state); void FinishFrontMacro(Core::System& system); std::unique_ptr front; diff --git a/src/video_core/engines/puller.cpp b/src/video_core/engines/puller.cpp index 8fb6a43f47..a701cccb1c 100644 --- a/src/video_core/engines/puller.cpp +++ b/src/video_core/engines/puller.cpp @@ -58,6 +58,43 @@ void OnBack(const Func& func) { } } +// EDEN_NX2_MME_DMA_STATS: how semaphore acquires end and how long they hold the front. +struct AcquireStats { + enum Kind : size_t { Fast, Own, AfterWait, AfterDrain, Suspended, Count }; + std::array counts{}; + u64 wait_ns{}; + u64 drain_ns{}; + u64 suspended_ns{}; + std::chrono::steady_clock::time_point last_log{std::chrono::steady_clock::now()}; + + static AcquireStats* Get() { + static const bool enabled = std::getenv("EDEN_NX2_MME_DMA_STATS") != nullptr; + static AcquireStats stats; + return enabled ? &stats : nullptr; + } + void Note(Kind kind) { + ++counts[kind]; + const auto now = std::chrono::steady_clock::now(); + if (now - last_log < std::chrono::seconds{5}) { + return; + } + last_log = now; + LOG_INFO(HW_GPU, + "Acquires (5 s): fast {}, own release {}, met after wait {} ({} ms), " + "met after drain {} ({} ms), suspended {} ({} ms)", + counts[Fast], counts[Own], counts[AfterWait], wait_ns / 1000000, + counts[AfterDrain], drain_ns / 1000000, counts[Suspended], suspended_ns / 1000000); + counts = {}; + wait_ns = drain_ns = suspended_ns = 0; + } +}; + +u64 NsSince(std::chrono::steady_clock::time_point start) { + return static_cast(std::chrono::duration_cast( + std::chrono::steady_clock::now() - start) + .count()); +} + void DrainBack() { VideoCommon::GpuSplit::Get().Drain(VideoCommon::DrainSite::Acquire); } @@ -130,6 +167,7 @@ bool Puller::ResumeAcquire(DmaPusher& dma_pusher) { // counting those made the count odd ("suspended") at random, so submit-time snapshots were // thrown away for about half of all entries (CommandList::snapshot_suspensions). if (pending_acquire) { + if (auto* stats = AcquireStats::Get()) stats->suspended_ns += NsSince(pending_acquire->since); pending_acquire.reset(); dma_pusher.suspensions.fetch_add(1, std::memory_order_release); Nx2SemaphoreWaiters::Get().Remove(this); @@ -211,6 +249,59 @@ void Puller::ProcessFenceActionMethod(DmaPusher& dma_pusher) { } } +bool Puller::StableSemaphore(DmaPusher& dma_pusher, GPUVAddr address) { + auto& split = VideoCommon::GpuSplit::Get(); + // Queued compute and copy work may write it too, untracked. + if (std::max(split.TrackedWritePosition(&dma_pusher.memory_manager, address, address + 4), + split.MemoryBarrierPosition()) > split.Consumed()) { + return false; + } + const auto device = dma_pusher.memory_manager.GpuToCpuAddress(address); + return device && !Nx2IsGpuWritten(*device); +} + +void Puller::CheckReleaseEpoch() { + if (const u64 epoch = VideoCommon::GpuSplit::Get().Epoch(); epoch != queued_releases_epoch) { + queued_releases.clear(); + queued_releases_epoch = epoch; + } +} + +void Puller::NoteRelease(GPUVAddr address, u32 payload) { + auto& split = VideoCommon::GpuSplit::Get(); + if (!split.Active()) { + return; + } + CheckReleaseEpoch(); + if (queued_releases.size() >= 1024) { + const u64 done = split.Consumed(); + std::erase_if(queued_releases, [done](const auto& entry) { return entry.second.position <= done; }); + if (queued_releases.size() >= 1024) { + queued_releases.clear(); + } + } + queued_releases[address] = {payload, split.Position()}; +} + +std::optional Puller::OwnQueuedRelease(DmaPusher& dma_pusher, GPUVAddr address) { + auto& split = VideoCommon::GpuSplit::Get(); + // At High GPU accuracy the release is written when its fence signals, after the back ran it. + if (!split.Active() || Settings::IsGPULevelHigh()) { + return std::nullopt; + } + CheckReleaseEpoch(); + const auto it = queued_releases.find(address); + if (it == queued_releases.end() || it->second.position <= split.Consumed()) { + return std::nullopt; + } + // Another write queued after it (a reset by compute, copy or another release) wins. + if (split.ReadReadyPosition(&dma_pusher.memory_manager, address, address + 4) > + it->second.position) { + return std::nullopt; + } + return it->second.payload; +} + void Puller::ProcessSemaphoreTriggerMethod(DmaPusher& dma_pusher) { const auto semaphoreOperationMask = 0xF; const auto op = GpuSemaphoreOperation(regs.semaphore_trigger & semaphoreOperationMask); @@ -238,6 +329,10 @@ void Puller::ProcessSemaphoreTriggerMethod(DmaPusher& dma_pusher) { OnBack([rasterizer = dma_pusher.rasterizer, sequence_address, properties, payload] { rasterizer->Query(sequence_address, VideoCommon::QueryType::Payload, properties, payload, 0); }); + const u64 size = properties == VideoCommon::QueryPropertiesFlags::IsAFence ? 4 : 16; + VideoCommon::GpuSplit::Get().NoteMemoryWrite(&dma_pusher.memory_manager, sequence_address, + sequence_address + size); + NoteRelease(sequence_address, payload); } else if (g_nx2_ampere) { // Switch 2 waits on semaphores the CPU (or another channel) releases, e.g. once it has // written the commands this channel runs next: suspend the channel until the value is @@ -255,8 +350,69 @@ void Puller::ProcessSemaphoreTriggerMethod(DmaPusher& dma_pusher) { return; } VideoCommon::Nx2Profile::Add(VideoCommon::Nx2Profile::Counter::AcquireChecks, 1); + // A greater-or-equal acquire whose value is already in guest memory is met, unless a + // queued write (a release that lowers it) or GPU-written data not yet downloaded may change + // it. Skip draining the back for it (~8% of the front in Ocarina of Time); later command + // fetches still wait for the work before it (NoteSync). + auto& split = VideoCommon::GpuSplit::Get(); + if (op == GpuSemaphoreOperation::AcquireGequal && split.Active() && + StableSemaphore(dma_pusher, acquire.address) && + acquire.Matches(dma_pusher.memory_manager.Read(acquire.address))) { + if (auto* stats = AcquireStats::Get()) stats->Note(AcquireStats::Fast); + split.NoteSync(); + return; + } + // This channel's own release, still queued, meets it once the back runs it, which is + // before anything sent after the acquire. + if (const auto released = OwnQueuedRelease(dma_pusher, acquire.address); + released && acquire.Matches(*released)) { + if (auto* stats = AcquireStats::Get()) stats->Note(AcquireStats::Own); + split.NoteSync(); + return; + } + if (split.Active()) { + // Only queued work that writes the semaphore (or untracked writes) can change it. + const u64 target = split.ReadReadyPosition(&dma_pusher.memory_manager, acquire.address, + acquire.address + 4); + if (static const bool stats = std::getenv("EDEN_NX2_MME_DMA_STATS") != nullptr; stats) { + static std::array causes{}; + const auto own = queued_releases.find(acquire.address); + const size_t cause = target <= split.Consumed() ? 0 + : target == split.WriterPosition() ? 1 + : target == split.MemoryBarrierPosition() ? 2 + : own != queued_releases.end() ? 3 + : 4; + ++causes[cause]; + if (cause == 3 && causes[3] < 16) { + LOG_INFO(HW_GPU, "Acquire op {} {:#x} at {:#x}: own release {:#x}", + static_cast(op), acquire.value, acquire.address, + own->second.payload); + } + if (((causes[0] + causes[1] + causes[2] + causes[3] + causes[4]) & 0x3fff) == 0) { + LOG_INFO(HW_GPU, + "Acquire waits: none {}, writer {}, barrier {} (source {:#x}), own " + "release {}, other write {}", + causes[0], causes[1], causes[2], split.MemoryBarrierSource(), + causes[3], causes[4]); + } + } + const auto wait_start = std::chrono::steady_clock::now(); + split.WaitForPosition(target, VideoCommon::DrainSite::Acquire); + if (auto* stats = AcquireStats::Get()) stats->wait_ns += NsSince(wait_start); + // A GPU-written word may be newer in a host buffer: the drain below flushes it. + const auto device = dma_pusher.memory_manager.GpuToCpuAddress(acquire.address); + if (device && !Nx2IsGpuWritten(*device) && + acquire.Matches(dma_pusher.memory_manager.Read(acquire.address))) { + if (auto* stats = AcquireStats::Get()) stats->Note(AcquireStats::AfterWait); + split.NoteSync(); + return; + } + } + const auto drain_start = std::chrono::steady_clock::now(); DrainBack(); + if (auto* stats = AcquireStats::Get()) stats->drain_ns += NsSince(drain_start); if (acquire.Matches(ReadSemaphoreWord(dma_pusher, acquire.address))) { + if (auto* stats = AcquireStats::Get()) stats->Note(AcquireStats::AfterDrain); return; } // Registered before the fences run, so a release among them that meets the value counts @@ -268,10 +424,13 @@ void Puller::ProcessSemaphoreTriggerMethod(DmaPusher& dma_pusher) { ReleaseFences(dma_pusher); if (!waiters.Latched(this) && !acquire.Matches(ReadSemaphoreWord(dma_pusher, acquire.address))) { pending_acquire = acquire; + pending_acquire->since = std::chrono::steady_clock::now(); + if (auto* stats = AcquireStats::Get()) stats->Note(AcquireStats::Suspended); dma_pusher.suspensions.fetch_add(1, std::memory_order_release); VideoCommon::Nx2Profile::Add(VideoCommon::Nx2Profile::Counter::AcquireWaits, 1); } else { waiters.Remove(this); + if (auto* stats = AcquireStats::Get()) stats->Note(AcquireStats::AfterDrain); } } else { DrainBack(); @@ -312,6 +471,8 @@ void Puller::ProcessSemaphoreRelease(DmaPusher& dma_pusher) { rasterizer->Query(sequence_address, VideoCommon::QueryType::Payload, VideoCommon::QueryPropertiesFlags::IsAFence, payload, 0); }); + VideoCommon::GpuSplit::Get().NoteMemoryWrite(&dma_pusher.memory_manager, sequence_address, + sequence_address + 4); } void Puller::ProcessSemaphoreAcquire(DmaPusher& dma_pusher) { @@ -341,8 +502,31 @@ void Puller::ProcessSemaphoreExecute(DmaPusher& dma_pusher, u32 argument) { rasterizer->Query(address, VideoCommon::QueryType::Payload, VideoCommon::QueryPropertiesFlags::IsAFence, payload, 0); }); + VideoCommon::GpuSplit::Get().NoteMemoryWrite(&dma_pusher.memory_manager, address, + address + 4); + NoteRelease(address, payload); return; } + auto& split = VideoCommon::GpuSplit::Get(); + if (operation <= 3 && operation != 1) { + // This channel's own queued release meets it before anything sent after the acquire. + if (const auto released = OwnQueuedRelease(dma_pusher, address)) { + if (operation == 0 ? *released == payload + : operation == 2 ? *released >= payload + : static_cast(*released - payload) >= 0) { + split.NoteSync(); + return; + } + } + } + if ((operation == 2 || operation == 3) && split.Active() && StableSemaphore(dma_pusher, address)) { + // A greater-or-equal value already in memory with no queued write to it: skip the drain. + const u32 current = dma_pusher.memory_manager.Read(address); + if (operation == 2 ? current >= payload : static_cast(current - payload) >= 0) { + split.NoteSync(); + return; + } + } // Acquires: check once after flushing pending fences; never block the GPU thread on them. DrainBack(); ReleaseFences(dma_pusher); diff --git a/src/video_core/engines/puller.h b/src/video_core/engines/puller.h index 2b9e2c5df3..8dba782f02 100644 --- a/src/video_core/engines/puller.h +++ b/src/video_core/engines/puller.h @@ -7,9 +7,12 @@ #pragma once #include +#include +#include #include #include #include +#include #include #include "common/bit_field.h" #include "common/common_funcs.h" @@ -149,6 +152,21 @@ class Puller final { [[nodiscard]] bool ExecuteMethodOnEngine(DmaPusher& dma_pusher, u32 method); std::optional pending_acquire; + /// Semaphore releases this channel queued for the split's back: payload and ring position. + struct QueuedRelease { + u32 payload; + u64 position; + }; + std::unordered_map queued_releases; + /// GpuSplit::Epoch when queued_releases was filled: a restarted split voids its positions. + u64 queued_releases_epoch{}; + void CheckReleaseEpoch(); + /// Whether no queued write or undownloaded GPU write can change the semaphore at `address`. + bool StableSemaphore(DmaPusher& dma_pusher, GPUVAddr address); + void NoteRelease(GPUVAddr address, u32 payload); + /// The payload of this channel's queued release at `address`, if it is still queued and is + /// the last write to it before what is sent now. + std::optional OwnQueuedRelease(DmaPusher& dma_pusher, GPUVAddr address); /// Mapping of command subchannels to their bound engine ids std::array bound_engines{}; diff --git a/src/video_core/gpu_split.cpp b/src/video_core/gpu_split.cpp index c13781131a..7aef1d5ca0 100644 --- a/src/video_core/gpu_split.cpp +++ b/src/video_core/gpu_split.cpp @@ -124,6 +124,57 @@ void GpuSplit::MaybeStart(Core::System& system_) { front_macros ? ", macros on the first" : ""); } +namespace { +constexpr u64 PageShift = 12; + +// The key of a 64-byte block: its device address, or its GPU address in `space` while unmapped. +template +void ForEachBlockKey(const void* space, u64 begin, u64 end, u64 block_shift, Func&& func) { + const auto* memory_manager = static_cast(space); + const u64 per_page = u64{1} << (PageShift - block_shift); + u64 block = begin >> block_shift; + const u64 last = (end - 1) >> block_shift; + while (block <= last) { + const u64 page = block >> (PageShift - block_shift); + const u64 page_last = std::min(last, (page + 1) * per_page - 1); + const auto device = memory_manager->GpuToCpuAddress(page << PageShift); + for (; block <= page_last; ++block) { + const u64 offset = block & (per_page - 1); + func(device ? ((*device >> block_shift) | offset) & ~(u64{1} << 63) + : (block ^ (static_cast(reinterpret_cast(space)) << 40)) | + (u64{1} << 63)); + } + } +} +} // Anonymous namespace + +void GpuSplit::NoteBlocks(std::unordered_map& blocks, const void* space, u64 begin, + u64 end) { + ForEachBlockKey(space, begin, end, WriteBlockShift, + [&](u64 key) { blocks[key] = write_position; }); +} + +u64 GpuSplit::BlocksPosition(const std::unordered_map& blocks, const void* space, + u64 begin, u64 end) const { + u64 position = 0; + ForEachBlockKey(space, begin, end, WriteBlockShift, [&](u64 key) { + if (const auto it = blocks.find(key); it != blocks.end()) { + position = std::max(position, it->second); + } + }); + return position; +} + +u64 GpuSplit::TrackedWritePosition(const void* space, u64 begin, u64 end) { + if (end <= begin || write_blocks.empty() || latest_block_write <= Consumed()) { + return 0; + } + if (((end - 1) >> WriteBlockShift) - (begin >> WriteBlockShift) >= MaxWriteBlocks) { + return latest_block_write; + } + return BlocksPosition(write_blocks, space, begin, end); +} + void GpuSplit::Stop() { if (!back_thread.joinable()) { return; @@ -140,6 +191,14 @@ void GpuSplit::Stop() { ring.reset(); write_position = 0; last_writer_position = 0; + last_barrier_position = 0; + draws_queued = false; + ++epoch; + write_blocks.clear(); + opaque_blocks.clear(); + front_words.clear(); + latest_block_write = 0; + latest_opaque_write = 0; last_sync_position = 0; published_local = 0; consumed_cache = 0; diff --git a/src/video_core/gpu_split.h b/src/video_core/gpu_split.h index 2e9250ff53..92bb0e3efb 100644 --- a/src/video_core/gpu_split.h +++ b/src/video_core/gpu_split.h @@ -3,6 +3,7 @@ #pragma once +#include #include #include #include @@ -10,6 +11,8 @@ #include #include #include +#include +#include #include "common/common_types.h" @@ -170,11 +173,106 @@ class GpuSplit { /// commands after it are fetched. void NoteSync() { last_sync_position = write_position; + NoteIdle(); + } + /// A draw was just sent. Draws write memory the front does not track (shader stores). + void NoteDraw() { + draws_queued = true; + } + /// A wait for idle was just sent: what the draws before it wrote is visible after it, as + /// the writes of compute work are. + void NoteIdle() { + if (draws_queued) { + draws_queued = false; + last_writer_position = write_position; + } + } + /// Increments each time the split stops: ring positions recorded before then are void. + [[nodiscard]] u64 Epoch() const { + return epoch; } /// Waits until the back executed the work sent before the last NoteSync. Returns whether any /// of it was still queued. bool WaitForSync(); + /// Work that may write guest memory at addresses the front does not track was just sent. + /// `source` names the sender for EDEN_NX2_MME_DMA_STATS. + void NoteMemoryBarrier(u32 source = 0) { + last_barrier_position = write_position; + last_barrier_source = source; + } + [[nodiscard]] u32 MemoryBarrierSource() const { + return last_barrier_source; + } + /// Where the last untracked memory write (or compute and copy work) was sent. + [[nodiscard]] u64 MemoryBarrierPosition() const { + return std::max(last_barrier_position, last_writer_position); + } + [[nodiscard]] u64 WriterPosition() const { + return last_writer_position; + } + [[nodiscard]] u64 Position() const { + return write_position; + } + /// How far the back has executed. + [[nodiscard]] u64 Consumed() { + consumed_cache = consumed.load(std::memory_order_acquire); + return consumed_cache; + } + /// Waits until the back executed everything sent before `target`. + void WaitForPosition(u64 target, DrainSite site) { + WaitFor(std::min(target, write_position), site); + } + + /// Work just sent writes [begin, end) of the GPU address space `space` (a memory manager). + /// `opaque`: the front does not know the data written. + void NoteMemoryWrite(const void* space, u64 begin, u64 end, bool opaque = true) { + if (end <= begin) { + return; + } + if (((end - 1) >> WriteBlockShift) - (begin >> WriteBlockShift) >= MaxWriteBlocks) { + NoteMemoryBarrier(0x40001); + return; + } + if (write_blocks.size() >= 8192 && Consumed() >= latest_block_write) { + write_blocks.clear(); + opaque_blocks.clear(); + } + if (write_blocks.size() >= 65536) { + write_blocks.clear(); + opaque_blocks.clear(); + NoteMemoryBarrier(0x40000); + } + NoteBlocks(write_blocks, space, begin, end); + latest_block_write = write_position; + if (opaque) { + NoteBlocks(opaque_blocks, space, begin, end); + latest_opaque_write = write_position; + } + } + /// The position the back must reach before [begin, end) of `space` can be read: the last + /// untracked write, or the last queued write that overlaps it. + [[nodiscard]] u64 ReadReadyPosition(const void* space, u64 begin, u64 end) { + return ReadyPosition(write_blocks, latest_block_write, space, begin, end); + } + /// As ReadReadyPosition, for writes whose data the front does not know. + [[nodiscard]] u64 OpaqueReadyPosition(const void* space, u64 begin, u64 end) { + return ReadyPosition(opaque_blocks, latest_opaque_write, space, begin, end); + } + /// The position after the last queued tracked write to [begin, end), untracked writes aside. + [[nodiscard]] u64 TrackedWritePosition(const void* space, u64 begin, u64 end); + + /// Front only: a constant buffer word sent inline and the ring position after its upload. + struct FrontWord { + u32 value; + u64 position; + }; + /// Front only: the words of every channel, keyed by device address, so the last upload of a + /// word wins whichever channel sent it. + [[nodiscard]] std::unordered_map& FrontWords() { + return front_words; + } + private: friend struct GpuSplitTestAccess; @@ -246,6 +344,34 @@ class GpuSplit { // Front side. u64 write_position{}; u64 last_writer_position{}; + u64 last_barrier_position{}; + bool draws_queued{}; + u64 epoch{}; + u32 last_barrier_source{}; + // Queued tracked writes: the ring position after the last write to each 64-byte block. + static constexpr u64 WriteBlockShift = 6; + static constexpr u64 MaxWriteBlocks = 1024; + std::unordered_map write_blocks; + std::unordered_map opaque_blocks; + std::unordered_map front_words; + u64 latest_block_write{}; + u64 latest_opaque_write{}; + // Keyed by the backing memory (`space` is a Tegra::MemoryManager), so writes and reads + // through different GPU addresses of the same memory meet. + void NoteBlocks(std::unordered_map& blocks, const void* space, u64 begin, u64 end); + u64 BlocksPosition(const std::unordered_map& blocks, const void* space, u64 begin, + u64 end) const; + u64 ReadyPosition(const std::unordered_map& blocks, u64 latest, const void* space, + u64 begin, u64 end) { + u64 target = MemoryBarrierPosition(); + if (end <= begin || blocks.empty() || latest <= target) { + return target; + } + if (((end - 1) >> WriteBlockShift) - (begin >> WriteBlockShift) >= MaxWriteBlocks) { + return std::max(target, latest); + } + return std::max(target, BlocksPosition(blocks, space, begin, end)); + } u64 last_sync_position{}; u64 published_local{}; u64 consumed_cache{}; diff --git a/src/video_core/gpu_thread.cpp b/src/video_core/gpu_thread.cpp index 8f25448101..5d334afdcc 100644 --- a/src/video_core/gpu_thread.cpp +++ b/src/video_core/gpu_thread.cpp @@ -4,6 +4,7 @@ // SPDX-FileCopyrightText: Copyright 2019 yuzu Emulator Project // SPDX-License-Identifier: GPL-2.0-or-later +#include #include #include @@ -22,6 +23,7 @@ #include "video_core/dma_pusher.h" #include "video_core/gpu.h" #include "video_core/gpu_thread.h" +#include "video_core/nx2.h" #include "video_core/gpu_split.h" #include "video_core/nx2_profile.h" #include "video_core/host1x/host1x.h" @@ -42,6 +44,42 @@ void SignalCommand(SynchState& state, u64 fence, bool block) { } } +// EDEN_NX2_MME_DMA_STATS: where the GPU thread's time goes while channels are suspended. +struct ThreadStats { + enum Kind : size_t { PendingEmpty, Idle, Count }; + std::array ns{}; + std::chrono::steady_clock::time_point last_log{std::chrono::steady_clock::now()}; + + struct Timer { + Timer(ThreadStats* stats_, Kind kind_) + : stats{stats_}, kind{kind_}, start{std::chrono::steady_clock::now()} {} + ~Timer() { + if (stats) { + stats->ns[kind] += static_cast( + std::chrono::duration_cast( + std::chrono::steady_clock::now() - start) + .count()); + } + } + ThreadStats* stats; + Kind kind; + std::chrono::steady_clock::time_point start; + }; + + void MaybeLog(bool pending) { + const auto now = std::chrono::steady_clock::now(); + if (now - last_log < std::chrono::seconds{5}) { + return; + } + last_log = now; + LOG_INFO(HW_GPU, + "GPU thread (5 s): {} ms waiting for commands with channels suspended, {} ms " + "idle with none suspended (suspended now: {})", + ns[PendingEmpty] / 1000000, ns[Idle] / 1000000, pending); + ns = {}; + } +}; + } // Anonymous namespace ThreadManager::ThreadManager(Core::System& system_) @@ -72,11 +110,17 @@ void ThreadManager::StartThread(VideoCore::RendererBase& renderer, Core::Fronten auto last_progress = std::chrono::steady_clock::now(); namespace Profile = VideoCommon::Nx2Profile; auto& split = VideoCommon::GpuSplit::Get(); + ThreadStats* const thread_stats = + std::getenv("EDEN_NX2_MME_DMA_STATS") != nullptr ? new ThreadStats{} : nullptr; while (!stop_token.stop_requested()) { + if (thread_stats) { + thread_stats->MaybeLog(scheduler.HasPendingChannels()); + } Profile::Report(); split.MaybeStart(system); if (scheduler.HasPendingChannels()) { // Suspended channels wait on semaphores the CPU releases: poll them meanwhile. + const u64 seen_latch = Tegra::Nx2SemaphoreWaiters::Get().LatchEpoch(); { Profile::Scope scope{Profile::Zone::ResumePending}; scheduler.ResumePending(system.GPU()); @@ -88,18 +132,23 @@ void ThreadManager::StartThread(VideoCore::RendererBase& renderer, Core::Fronten // present waited ~17 ms a frame in Kokiri village, with 50 us ~1 ms. A channel // still waiting after 2 s (menus, loading) goes back to 1 ms sleeps. Profile::Scope scope{Profile::Zone::PendingSleep}; + const ThreadStats::Timer timer{thread_stats, ThreadStats::PendingEmpty}; const auto waited = std::chrono::steady_clock::now() - last_progress; if (waited < std::chrono::microseconds{500}) { std::this_thread::yield(); } else if (waited < std::chrono::seconds{2}) { std::this_thread::sleep_for(std::chrono::microseconds{50}); } else { - std::this_thread::sleep_for(std::chrono::milliseconds{1}); + // A GPU release that meets a suspended acquire wakes this at once; + // CPU releases are still found by polling. + Tegra::Nx2SemaphoreWaiters::Get().WaitForLatch( + seen_latch, std::chrono::milliseconds{1}); } continue; } } else { Profile::Scope scope{Profile::Zone::Idle}; + const ThreadStats::Timer timer{thread_stats, ThreadStats::Idle}; state.queue.PopWait(next, stop_token); } if (stop_token.stop_requested()) { diff --git a/src/video_core/nx2.h b/src/video_core/nx2.h index e85e6142e7..366b3c3a64 100644 --- a/src/video_core/nx2.h +++ b/src/video_core/nx2.h @@ -6,6 +6,8 @@ #include #include #include +#include +#include #include #include #include @@ -84,6 +86,32 @@ inline void Nx2MarkCommandListPage(u64 device_address) { return (g_nx2_command_list_pages[page / 64].load(std::memory_order_relaxed) >> (page % 64)) & 1; } +// Reports the query cache holds until their host results are written (Lichii #243): per +// 16-byte slot hash, how many the back registered and how many it wrote out. Equal counts mean +// the words in memory are final, so the split front's MME DMA reads of them need no flush. +inline constexpr u64 Nx2PendingReportSlots = u64{1} << 16; +inline std::array, Nx2PendingReportSlots> g_nx2_reports_issued{}; +inline std::array, Nx2PendingReportSlots> g_nx2_reports_written{}; + +[[nodiscard]] inline u64 Nx2PendingReportSlot(u64 device_address) { + return (device_address >> 4) % Nx2PendingReportSlots; +} + +[[nodiscard]] inline bool Nx2ReportsPending(u64 device_address, u64 size) { + if (size == 0) { + return false; + } + const u64 last = (device_address + size - 1) >> 4; + for (u64 slot = device_address >> 4; slot <= last; ++slot) { + const u64 index = slot % Nx2PendingReportSlots; + if (g_nx2_reports_written[index].load(std::memory_order_acquire) != + g_nx2_reports_issued[index].load(std::memory_order_acquire)) { + return true; + } + } + return false; +} + /// A new emulation session starts: no page has been written by its GPU yet. inline void Nx2ClearGpuWrittenPages() { for (auto& word : g_nx2_gpu_written_pages) { @@ -92,6 +120,12 @@ inline void Nx2ClearGpuWrittenPages() { for (auto& word : g_nx2_command_list_pages) { word.store(0, std::memory_order_relaxed); } + for (auto& count : g_nx2_reports_issued) { + count.store(0, std::memory_order_relaxed); + } + for (auto& count : g_nx2_reports_written) { + count.store(0, std::memory_order_relaxed); + } g_nx2_has_command_list_pages.store(false, std::memory_order_relaxed); } @@ -404,12 +438,33 @@ class Nx2SemaphoreWaiters { if (count.load(std::memory_order_acquire) == 0) { return; } - std::scoped_lock lk{mutex}; - for (auto& w : waiters) { - if (w.address == device_address && !w.latched && Matches(w, value)) { - w.latched = true; + bool latched = false; + { + std::scoped_lock lk{mutex}; + for (auto& w : waiters) { + if (w.address == device_address && !w.latched && Matches(w, value)) { + w.latched = true; + latched = true; + } + } + if (latched) { + ++latch_epoch; } } + if (latched) { + latch_cv.notify_all(); + } + } + + [[nodiscard]] u64 LatchEpoch() { + std::scoped_lock lk{mutex}; + return latch_epoch; + } + + /// Sleeps until a waiter latches after `seen` (LatchEpoch) or `timeout` passes. + void WaitForLatch(u64 seen, std::chrono::microseconds timeout) { + std::unique_lock lk{mutex}; + latch_cv.wait_for(lk, timeout, [&] { return latch_epoch != seen; }); } private: @@ -435,6 +490,8 @@ class Nx2SemaphoreWaiters { } std::mutex mutex; + std::condition_variable latch_cv; + u64 latch_epoch{}; std::vector waiters; std::atomic count{}; }; diff --git a/src/video_core/query_cache/query_cache.h b/src/video_core/query_cache/query_cache.h index efa148bf06..63833ac236 100644 --- a/src/video_core/query_cache/query_cache.h +++ b/src/video_core/query_cache/query_cache.h @@ -286,6 +286,17 @@ void QueryCacheBase::CounterReport(GPUVAddr addr, QueryType counter_type std::function operation([this, is_synced, streamer, query_base = query, query_location, pointer, pointer_timestamp, cpu_addr, addr, asid = gpu_memory->GetID(), counter_type] { + // Counted on every return: the front's DMA reads wait while the counts differ. + struct Written { + bool registered; + DAddr address; + ~Written() { + if (registered) { + Tegra::g_nx2_reports_written[Tegra::Nx2PendingReportSlot(address)].fetch_add( + 1, std::memory_order_release); + } + } + } const written{!is_synced, cpu_addr}; if (True(query_base->flags & QueryFlagBits::IsInvalidated)) { if (const u64 trace = QueryTrace::Sample(QueryTrace::Event::Discard)) { LOG_INFO(HW_GPU, @@ -338,7 +349,14 @@ void QueryCacheBase::CounterReport(GPUVAddr addr, QueryType counter_type impl->pending_unregister.push_back(query_location); } }); + const auto issue = [&] { + if (!is_synced) { + Tegra::g_nx2_reports_issued[Tegra::Nx2PendingReportSlot(cpu_addr)].fetch_add( + 1, std::memory_order_release); + } + }; if (is_fence) { + issue(); impl->rasterizer.SignalFence(std::move(operation)); } else { if (!Settings::IsGPULevelHigh() && counter_type == QueryType::Payload) { @@ -362,6 +380,7 @@ void QueryCacheBase::CounterReport(GPUVAddr addr, QueryType counter_type streamer->Free(new_query_id); return; } + issue(); impl->rasterizer.SyncOperation(std::move(operation)); } if (is_synced) { diff --git a/src/video_core/renderer_vulkan/vk_pipeline_cache.cpp b/src/video_core/renderer_vulkan/vk_pipeline_cache.cpp index 81e2cacba7..25f7994943 100644 --- a/src/video_core/renderer_vulkan/vk_pipeline_cache.cpp +++ b/src/video_core/renderer_vulkan/vk_pipeline_cache.cpp @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -739,6 +740,32 @@ bool PreloadAllPipelines() { }(); return all; } + +// Adds the pipelines of an older version's used-pipeline list to this version's (a pipeline in both +// keeps its more recent use), so the pipelines an older build used are built at boot too. +void MergeUsedPipelines(const std::filesystem::path& destination, + const std::filesystem::path& source) { + constexpr u8 KeepAllAges = 255; + const auto older = LoadUsedPipelines(source, KeepAllAges); + if (older.empty()) { + return; + } + auto merged = LoadUsedPipelines(destination, KeepAllAges); + for (const auto& [hash, age] : older) { + auto [it, inserted] = merged.try_emplace(hash, age); + if (!inserted && age < it->second) { + it->second = age; + } + } + std::ofstream file{destination, std::ios::binary | std::ios::trunc}; + const u32 count = static_cast(merged.size()); + file.write(reinterpret_cast(&UsedPipelinesMagic), sizeof(UsedPipelinesMagic)); + file.write(reinterpret_cast(&count), sizeof(count)); + for (const auto& entry : merged) { + file.write(reinterpret_cast(&entry.first), sizeof(entry.first)); + file.write(reinterpret_cast(&entry.second), sizeof(entry.second)); + } +} } // Anonymous namespace PipelineCache::~PipelineCache() { @@ -957,6 +984,28 @@ void PipelineCache::LoadDiskResources(u64 title_id, std::stop_token stop_loading std::filesystem::copy_file(VideoCommon::VersionedPipelineCachePath(base_dir, "vulkan_used", 52), used_pipelines_filename, std::filesystem::copy_options::none, migration_error); + // Cache version 53 holds the same records as this version: #432 moved to 54 for the SPIR-V of + // its NaN-defined min/max, and every record is translated again when it loads. A cache of this + // version started before this import lacks the places a version 53 build had been, so the + // version 53 records are copied once beside it (vulkan_import, loaded after it) and its + // used-pipeline list is merged into this one, so those pipelines are built at boot again. The + // version 53 files stay as they are for the builds that still read them. + const auto import_filename = + VideoCommon::VersionedPipelineCachePath(base_dir, "vulkan_import", CACHE_VERSION); + auto import_done_filename = import_filename; + import_done_filename += ".done"; + std::error_code import_error; + if (!std::filesystem::exists(import_done_filename, import_error) && + VideoCommon::CopyCompatiblePipelineCache( + VideoCommon::VersionedPipelineCachePath(base_dir, "vulkan", 53), import_filename, + shader_magic, 53, CACHE_VERSION)) { + std::ofstream import_done{import_done_filename}; + import_done.close(); + MergeUsedPipelines(used_pipelines_filename, + VideoCommon::VersionedPipelineCachePath(base_dir, "vulkan_used", 53)); + LOG_INFO(Render_Vulkan, "Imported the cache version 53 pipelines to {}", + Common::FS::PathToUTF8String(import_filename)); + } used_pipelines = LoadUsedPipelines(used_pipelines_filename, MaxUsedPipelineAge(device)); used_pipelines_saved = std::chrono::steady_clock::now(); size_t skipped_pipelines = 0; @@ -1113,6 +1162,9 @@ void PipelineCache::LoadDiskResources(u64 title_id, std::stop_token stop_loading }}; VideoCommon::LoadPipelines(stop_loading, pipeline_cache_filename, CACHE_VERSION, load_compute, load_graphics); + // The imported cache version 53 records (see above). A record both files hold is built once. + VideoCommon::LoadPipelines(stop_loading, import_filename, CACHE_VERSION, load_compute, + load_graphics); for (auto& [key, candidates] : compute_envs) { OrderComputeTranslations(candidates); queue_compute(key, std::move(candidates)); @@ -1289,12 +1341,42 @@ void PipelineCache::EvictUnusedPipelines() { } // Each built pipeline holds ~0.45 MB of driver memory and none was ever destroyed, so it grew // with every place a session went (title 1: 1,500 on its title screen alone). One unused for - // this long is destroyed; it takes 2-3 ms to build again from the driver's pipeline cache. + // this long may be destroyed, but each one is translated again on the GPU thread when its place + // is revisited: Ocarina of Time evicted 6,182 in a 37 min session while ~1,000-2,400 stayed, + // and draws waited 230 s in all for pipelines being built. So pipelines are only evicted while + // more than 8,000 are cached (~3.6 GB of driver memory), and then the longest unused first, + // down to 7/8 of that. EDEN_NX2_PIPELINE_EVICT_ABOVE sets the limit (0 evicts as before). constexpr u64 EvictAfterFrames = 9000; if (frame_count <= EvictAfterFrames) { return; } + static const size_t evict_above = [] { + const char* value = std::getenv("EDEN_NX2_PIPELINE_EVICT_ABOVE"); + return value != nullptr ? static_cast(std::strtoull(value, nullptr, 10)) + : size_t{8000}; + }(); + if (graphics_cache.size() <= evict_above) { + return; + } const u64 cutoff = frame_count - EvictAfterFrames; + // The last use at or before which pipelines are evicted: the oldest used ones until the cache + // is back to 7/8 of the limit, or every one unused since the cutoff if that is fewer. Prebuilt + // pipelines no draw has used yet are left to their budget below. + std::vector last_uses; + for (const auto& entry : graphics_cache) { + const GraphicsPipeline* const candidate = entry.second.get(); + if (candidate != nullptr && candidate != current_pipeline && candidate->IsBuilt() && + candidate->WasUsed() && candidate->LastUse() < cutoff) { + last_uses.push_back(candidate->LastUse()); + } + } + const size_t excess = graphics_cache.size() - (evict_above - evict_above / 8); + u64 evict_through = cutoff - 1; + if (excess < last_uses.size()) { + const auto nth = last_uses.begin() + static_cast(excess - 1); + std::nth_element(last_uses.begin(), nth, last_uses.end()); + evict_through = last_uses[excess - 1]; + } const u64 tick = scheduler.CurrentTick(); // ~0.9 GB of driver memory at ~0.45 MB each. constexpr size_t MaxUnusedPrebuiltPipelines = 2000; @@ -1303,7 +1385,7 @@ void PipelineCache::EvictUnusedPipelines() { for (auto it = graphics_cache.begin(); it != graphics_cache.end();) { GraphicsPipeline* const pipeline = it->second.get(); if (pipeline == nullptr || pipeline == current_pipeline || !pipeline->IsBuilt() || - pipeline->LastUse() >= cutoff) { + pipeline->LastUse() >= cutoff || pipeline->LastUse() > evict_through) { ++it; continue; } diff --git a/src/video_core/renderer_vulkan/vk_rasterizer.cpp b/src/video_core/renderer_vulkan/vk_rasterizer.cpp index c336f4123f..46d6faceac 100644 --- a/src/video_core/renderer_vulkan/vk_rasterizer.cpp +++ b/src/video_core/renderer_vulkan/vk_rasterizer.cpp @@ -1810,18 +1810,29 @@ void RasterizerVulkan::FlushQueryResults(DAddr addr, u64 size) { // Below High GPU accuracy a fence runs its own callback at once, so wait for an operation // queued behind the pending report writes instead. On the split's back the fence's // submission may be deferred, and the fencing thread would wait for this thread to submit it. - std::mutex wait_mutex; - std::condition_variable wait_cv; - bool written{}; - fence_manager.SyncOperation([&] { - std::scoped_lock lock{wait_mutex}; - written = true; - wait_cv.notify_all(); + // Shared with the callback: it may run after a timed-out wait returned. + struct WaitState { + std::mutex mutex; + std::condition_variable cv; + bool written{}; + }; + const auto state = std::make_shared(); + fence_manager.SyncOperation([state] { + std::scoped_lock lock{state->mutex}; + state->written = true; + state->cv.notify_all(); }); fence_manager.SignalFence([] {}); scheduler.FlushDeferred(); - std::unique_lock lock{wait_mutex}; - wait_cv.wait(lock, [&written] { return written; }); + std::unique_lock lock{state->mutex}; + if (!state->cv.wait_for(lock, std::chrono::seconds{2}, [&state] { return state->written; })) { + static std::atomic warned{}; + if (!warned.exchange(true, std::memory_order_relaxed)) { + LOG_WARNING(Render_Vulkan, + "Query results at {:#x} not written after 2 s: reading them as they are", + addr); + } + } } void RasterizerVulkan::FlushAndInvalidateRegion(DAddr addr, u64 size,