From 888ae5378bc35a16cc3abc27c9acc5612929b510 Mon Sep 17 00:00:00 2001 From: Celmax Date: Fri, 6 Nov 2026 20:52:46 +0100 Subject: [PATCH 01/16] video_core: precise macro DMA reads and cheaper acquires on the split front The split's front drained the back before every native macro DMA read and every semaphore acquire. Games read through MME DMA often (Ocarina of Time, where this was measured: ~20,000 reads a second, ~48% of the front spent draining), so the two GPU threads kept waiting on each other. Applies to every Switch 2 title running on the split. - gpu_split: track queued writes per 64-byte block, plus a barrier position for work whose writes are not tracked by address. A front read waits only until the back has run the last queued work that may write its words. - dma_pusher: classify writers (compute and copy launches, as before) and barriers (host methods and other engines that write memory untracked); command pages already fetched as CPU-written skip the writer wait until the GPU writes them. - maxwell_3d: inline-to-memory uploads, report semaphores and inline constant buffer data note the ranges they write; reads of inline constant buffer words still queued are answered from the front's copy of them. Every 4096th read still drains, and a read of GPU-written memory turns the precise reads off. - puller: skip the back drain for greater-or-equal acquires already met in memory and for acquires the channel's own queued release meets; otherwise wait only for writers of the semaphore. Addresses queued GPU work met 8 times in a row are trusted, and the back checks each skipped acquire and stops trusting an address it finds unmet. - gpu_thread: a GPU release that latches a suspended acquire wakes the GPU thread at once instead of after its 1 ms poll. - EDEN_NX2_MME_DMA_STATS: why reads and acquires waited, barrier sources and GPU thread time spent with channels suspended. Measured in Ocarina of Time (Kokiri Forest, RTX 4060 Ti, Ryzen 9 5950X): ~35.5 -> ~39.3 fps. Co-Authored-By: Claude Opus 5.5 --- src/video_core/dma_pusher.cpp | 95 ++++++++++++- src/video_core/dma_pusher.h | 3 + src/video_core/engines/maxwell_3d.cpp | 163 ++++++++++++++++++++++- src/video_core/engines/maxwell_3d.h | 2 + src/video_core/engines/puller.cpp | 183 ++++++++++++++++++++++++++ src/video_core/engines/puller.h | 16 +++ src/video_core/gpu_split.cpp | 5 + src/video_core/gpu_split.h | 104 +++++++++++++++ src/video_core/gpu_thread.cpp | 51 ++++++- src/video_core/nx2.h | 47 ++++++- 10 files changed, 656 insertions(+), 13 deletions(-) diff --git a/src/video_core/dma_pusher.cpp b/src/video_core/dma_pusher.cpp index 391226c9f2..807c7c0c0c 100644 --- a/src/video_core/dma_pusher.cpp +++ b/src/video_core/dma_pusher.cpp @@ -44,6 +44,35 @@ bool DmaPusher::IsMemoryWriter(const Engines::EngineInterface& engine, Engines:: } return false; } +namespace { +// Host methods that can write guest memory without noting it themselves. +bool PullerMethodWritesMemory(u32 method) { + switch (static_cast(method)) { + case BufferMethods::BindObject: + case BufferMethods::Nop: + case BufferMethods::SemaphoreAddressHigh: + case BufferMethods::SemaphoreAddressLow: + case BufferMethods::SemaphoreSequencePayload: + case BufferMethods::SemaphoreOperation: // Releases note their own writes (Puller) + case BufferMethods::SemaphoreRelease: + case BufferMethods::NonStallInterrupt: + case BufferMethods::WrcacheFlush: + case BufferMethods::MemOpA: + case BufferMethods::MemOpB: + case BufferMethods::MemOpC: + case BufferMethods::MemOpD: + case BufferMethods::RefCnt: + case BufferMethods::SemaphoreAcquire: // Ampere: SEM_PAYLOAD_HI, like 0x17..0x19 + case BufferMethods::SyncpointPayload: + case BufferMethods::SyncpointOperation: + case BufferMethods::WaitForIdle: + case BufferMethods::Yield: + return false; + default: + return !(method >= 0x17 && method <= 0x19); + } +} +} // namespace DmaPusher::DmaPusher(Core::System& system_, MemoryManager& memory_manager_, Control::ChannelState& channel_state_) : system{system_} @@ -198,10 +227,16 @@ bool DmaPusher::Step() { // (the split only starts with unsafe reads; the settings can change mid-game). const bool use_safe = !VideoCommon::GpuSplit::Get().Active() && (Settings::IsDMALevelDefault() ? Settings::IsGPULevelHigh() : Settings::IsDMALevelSafe()); - if (!gpu_written && VideoCommon::GpuSplit::Get().Active() && - VideoCommon::GpuSplit::Get().WaitForWriters()) { - gpu_written = - IsGpuWrittenCommands(dma_state.dma_get, size_t{header.size} * sizeof(u32)); + if (!gpu_written && VideoCommon::GpuSplit::Get().Active()) { + const size_t bytes = size_t{header.size} * sizeof(u32); + if (!CpuListPages(dma_state.dma_get, bytes, false)) { + if (VideoCommon::GpuSplit::Get().WaitForWriters()) { + gpu_written = IsGpuWrittenCommands(dma_state.dma_get, bytes); + } + if (!gpu_written) { + CpuListPages(dma_state.dma_get, bytes, true); + } + } } // Resume and GPU-written entries need live words, even after a flush cleared dirty state. const bool submitted = (command_list.snapshot_suspensions & 1) == 0 && @@ -464,6 +499,9 @@ void DmaPusher::CallMethod(u32 argument) { dma_state.subchannel, dma_state.method_count, }); + if (PullerMethodWritesMemory(dma_state.method)) { + split.NoteMemoryBarrier(0x10000 | dma_state.method); + } } else { auto subchannel = subchannels[dma_state.subchannel]; if (subchannel == nullptr) { @@ -473,9 +511,16 @@ void DmaPusher::CallMethod(u32 argument) { const auto engine_type = subchannel_type[dma_state.subchannel]; const bool writer = split.Active() && IsMemoryWriter(*subchannel, engine_type, dma_state.method); + // Other engines' memory writes are not tracked by address (Maxwell3D::FrontDmaRead). + const bool untracked_writes = engine_type != Engines::EngineTypes::Maxwell3D && + engine_type != Engines::EngineTypes::KeplerCompute && + engine_type != Engines::EngineTypes::MaxwellDMA; SCOPE_EXIT { if (writer) { split.NoteWriter(); + } else if (untracked_writes) { + split.NoteMemoryBarrier(0x20000 | (static_cast(engine_type) << 12) | + (dma_state.method & 0xfff)); } }; if (split.Active()) { @@ -522,6 +567,12 @@ void DmaPusher::CallMultiMethod(const u32* base_start, u32 num_methods) { auto& split = VideoCommon::GpuSplit::Get(); if (dma_state.method < non_puller_methods) { puller.CallMultiMethod(*this, dma_state.method, dma_state.subchannel, base_start, num_methods, dma_state.method_count); + for (u32 i = 0; i < num_methods; ++i) { + if (PullerMethodWritesMemory(dma_state.method + (dma_state.non_incrementing ? 0 : i))) { + split.NoteMemoryBarrier(0x10000 | dma_state.method); + break; + } + } } else { auto subchannel = subchannels[dma_state.subchannel]; if (subchannel == nullptr) { @@ -531,9 +582,16 @@ void DmaPusher::CallMultiMethod(const u32* base_start, u32 num_methods) { const auto engine_type = subchannel_type[dma_state.subchannel]; const bool writer = split.Active() && IsMemoryWriter(*subchannel, engine_type, dma_state.method); + // Other engines' memory writes are not tracked by address (Maxwell3D::FrontDmaRead). + const bool untracked_writes = engine_type != Engines::EngineTypes::Maxwell3D && + engine_type != Engines::EngineTypes::KeplerCompute && + engine_type != Engines::EngineTypes::MaxwellDMA; SCOPE_EXIT { if (writer) { split.NoteWriter(); + } else if (untracked_writes) { + split.NoteMemoryBarrier(0x20000 | (static_cast(engine_type) << 12) | + (dma_state.method & 0xfff)); } }; if (split.Active()) { @@ -597,6 +655,35 @@ void DmaPusher::RefreshDeferredEntries(CommandList& list) { list.deferred_entries.clear(); } +bool DmaPusher::CpuListPages(GPUVAddr address, size_t size, bool mark) { + if (!Tegra::g_nx2_ampere.load(std::memory_order_relaxed) || size == 0) { + return false; + } + if (const u64 misses = Tegra::g_nx2_cpu_list_misses.load(std::memory_order_relaxed); + misses != reported_cpu_list_misses) { + reported_cpu_list_misses = misses; + LOG_WARNING(HW_GPU, "GPU wrote {} command list pages first fetched as CPU-written", misses); + } + for (GPUVAddr page = address & ~GPUVAddr{0xfff}; page < address + size; page += 0x1000) { + const auto device = memory_manager.GpuToCpuAddress(page); + if (!device) { + return false; + } + const u64 index = *device >> Tegra::Nx2WrittenPageBits; + if (index >= Tegra::g_nx2_cpu_list_pages.size() * 64) { + return false; + } + auto& word = Tegra::g_nx2_cpu_list_pages[index / 64]; + const u64 bit = u64{1} << (index % 64); + if (mark) { + word.fetch_or(bit, std::memory_order_relaxed); + } else if ((word.load(std::memory_order_relaxed) & bit) == 0) { + return false; + } + } + return true; +} + bool DmaPusher::IsGpuWrittenCommands(GPUVAddr address, size_t size) { if (!Tegra::g_nx2_ampere.load(std::memory_order_relaxed) || size == 0) { return false; diff --git a/src/video_core/dma_pusher.h b/src/video_core/dma_pusher.h index 8c85eb3688..4c622bd1c6 100644 --- a/src/video_core/dma_pusher.h +++ b/src/video_core/dma_pusher.h @@ -216,6 +216,9 @@ class DmaPusher final { void RecordParsedEntry(const CommandListHeader& header, std::span commands); /// Whether the entry lies on device pages the GPU has written (NX2). bool IsGpuWrittenCommands(GPUVAddr address, size_t size); + /// Whether every page of the range was fetched as a CPU-written command list (or marks them). + bool CpuListPages(GPUVAddr address, size_t size, bool mark); + u64 reported_cpu_list_misses{}; /// Copies the list's words from memory again, as of suspension count `now`. void RefreshSnapshot(CommandList& list, u64 now); /// Copies the list's deferred entries (CommandList::deferred_entries) from memory. diff --git a/src/video_core/engines/maxwell_3d.cpp b/src/video_core/engines/maxwell_3d.cpp index 86fb6d24ac..1635cdf332 100644 --- a/src/video_core/engines/maxwell_3d.cpp +++ b/src/video_core/engines/maxwell_3d.cpp @@ -11,6 +11,7 @@ #include #include #include +#include #include "common/assert.h" #include "common/bit_util.h" #include "common/scope_exit.h" @@ -1034,6 +1035,17 @@ struct Maxwell3D::FrontState { /// then can it have a deferred instanced draw to issue at the end of a macro. bool draw_pending{true}; bool skip_same{}; ///< SkipSameValueWrites() && FrontSkipSame(), read once + u32 dma_reads{}; + /// Constant buffer words sent inline: value and the ring position after the upload. + struct ShadowWord { + u32 value; + u64 position; + }; + std::unordered_map cb_shadow; + /// The inline-to-memory upload being sent writes [upload_begin, upload_end). + bool upload_tracked{}; + GPUVAddr upload_begin{}; + GPUVAddr upload_end{}; }; namespace { @@ -1183,6 +1195,79 @@ void Maxwell3D::FrontWrite(VideoCommon::GpuSplit& split, FrontState& fs, u32 met } else { split.Method(this, method, argument, is_last_call, nullptr); } + if (execution_mask[method]) { + FrontNoteWrites(split, fs, method, 1, &argument); + } +} + +void Maxwell3D::FrontNoteWrites(VideoCommon::GpuSplit& split, FrontState& fs, u32 method, + u32 amount, const u32* values) { + constexpr u32 cb_data = MAXWELL3D_REG_INDEX(const_buffer.buffer); + const auto covers = [method, amount](u32 index) { + return method <= index && index < method + amount; + }; + const auto address = [&fs](u32 high, u32 low) { + return (GPUVAddr{fs.regs[high]} << 32) | fs.regs[low]; + }; + const auto note = [&split, this](GPUVAddr begin, GPUVAddr end) { + split.NoteMemoryWrite(&memory_manager, begin, end, true); + }; + if (method >= cb_data && method < cb_data + 16) { + const GPUVAddr base = address(MAXWELL3D_REG_INDEX(const_buffer.address_high), + MAXWELL3D_REG_INDEX(const_buffer.address_low)); + const u64 offset_end = fs.regs[MAXWELL3D_REG_INDEX(const_buffer.offset)]; + const u64 bytes = u64{amount} * sizeof(u32); + if (offset_end < bytes) { + split.NoteMemoryBarrier(0x30001); + return; + } + const GPUVAddr begin = base + offset_end - bytes; + split.NoteMemoryWrite(&memory_manager, begin, base + offset_end, false); + // The words themselves, for MME DMA reads of them while the back has not written them. + if (fs.cb_shadow.size() >= 16384) { + const u64 done = split.Consumed(); + std::erase_if(fs.cb_shadow, [done](const auto& entry) { return entry.second.position <= done; }); + if (fs.cb_shadow.size() >= 16384) { + fs.cb_shadow.clear(); + } + } + const u64 position = split.Position(); + for (u32 i = 0; i < amount; ++i) { + fs.cb_shadow[begin + u64{i} * sizeof(u32)] = {values[i], position}; + } + return; + } + if (covers(MAXWELL3D_REG_INDEX(launch_dma))) { + // Inline-to-memory: a pitch-linear upload writes its lines from the destination address; + // a block-linear one or a release on completion is not tracked. + const u32 launch = fs.regs[MAXWELL3D_REG_INDEX(launch_dma)]; + const bool pitch_layout = (launch & 1) != 0; // memory_layout: Pitch + const bool release = ((launch >> 4) & 3) == 2; // completion_type: Release + const u64 line_length = fs.regs[MAXWELL3D_REG_INDEX(upload.line_length_in)]; + const u64 lines = fs.regs[MAXWELL3D_REG_INDEX(upload.line_count)]; + const u64 pitch = fs.regs[MAXWELL3D_REG_INDEX(upload.dest.pitch)]; + fs.upload_tracked = pitch_layout && !release && lines != 0; + if (fs.upload_tracked) { + fs.upload_begin = address(MAXWELL3D_REG_INDEX(upload.dest.address_high), + MAXWELL3D_REG_INDEX(upload.dest.address_low)); + fs.upload_end = fs.upload_begin + (std::max)(pitch, line_length) * (lines - 1) + line_length; + } else { + split.NoteMemoryBarrier(0x30002); + } + } + if (covers(MAXWELL3D_REG_INDEX(inline_data))) { + if (fs.upload_tracked) { + note(fs.upload_begin, fs.upload_end); + } else { + split.NoteMemoryBarrier(0x30003); + } + } + if (covers(MAXWELL3D_REG_INDEX(report_semaphore.query))) { + // A release writes its payload, or a 16-byte report, at the semaphore address. + const GPUVAddr begin = address(MAXWELL3D_REG_INDEX(report_semaphore.address_high), + MAXWELL3D_REG_INDEX(report_semaphore.address_low)); + note(begin, begin + 16); + } } void Maxwell3D::FrontCallMethod(Core::System& system, VideoCommon::GpuSplit& split, u32 method, @@ -1288,6 +1373,7 @@ void Maxwell3D::FrontCallMultiMethod(Core::System& system, VideoCommon::GpuSplit } } split.MultiMethod(this, method, base_start, amount, methods_pending, nullptr); + FrontNoteWrites(split, fs, method, amount, base_start); } void Maxwell3D::FrontRunMacro(Core::System& system, VideoCommon::GpuSplit& split, @@ -1357,8 +1443,68 @@ void Maxwell3D::FrontDmaRead(VideoCommon::GpuSplit& split, FrontState& fs, u32 m memory_manager.ReadBlockUnsafe(source, diag_before.data(), read_bytes); } VideoCommon::Nx2Profile::CountDmaRead(macro); - split.Drain(VideoCommon::DrainSite::MacroDmaRead); - if (VideoCommon::QueryDma::TraceEnabled()) { + // Wait only for queued work that may write the words: compute, copies, untracked + // writes and overlapping inline constant buffer data. Ocarina of Time reads ~20,000 times a + // second; a full drain each time kept the two GPU threads waiting on each other. Every 4096th + // read still drains and checks for GPU-written data, which turns the precise reads off. + static bool precise_disabled{}; + const bool precise = !precise_disabled && + !VideoCommon::QueryDma::TraceEnabled() && + !Tegra::Nx2CmdPageDiag::g_enabled && + fs.regs[MAXWELL3D_REG_INDEX(transform_feedback_enabled)] == 0 && + (++fs.dma_reads & 4095) != 0; + static thread_local std::vector shadow_words; + shadow_words.clear(); + if (precise) { + const u64 target = split.ReadReadyPosition(&memory_manager, source, source + read_bytes); + const u64 consumed = split.Consumed(); + // Constant buffer words sent inline after any other write to them are known here: + // read them without waiting for the back to store them. + if (target > consumed) { + const u64 opaque = + split.OpaqueReadyPosition(&memory_manager, source, source + read_bytes); + shadow_words.clear(); + for (u32 i = 0; i < words; ++i) { + const auto it = fs.cb_shadow.find(source + u64{i} * sizeof(u32)); + // Only words still queued: memory has the others, maybe rewritten by the CPU. + if (it == fs.cb_shadow.end() || it->second.position < opaque || + it->second.position <= consumed) { + shadow_words.clear(); + break; + } + shadow_words.push_back(it->second.value); + } + } + const bool from_shadow = !shadow_words.empty(); + // EDEN_NX2_MME_DMA_STATS: why precise reads wait (writer = compute or copy work). + if (static const bool stats = std::getenv("EDEN_NX2_MME_DMA_STATS") != nullptr; stats) { + static std::array causes{}; + const u64 writer = split.WriterPosition(); + const u64 barrier = split.MemoryBarrierPosition(); + const size_t cause = target <= consumed || from_shadow ? 0 + : target == writer ? 1 + : target == barrier ? 2 + : 3; + ++causes[cause]; + static std::unordered_map sources; + if (cause == 2) { + ++sources[split.MemoryBarrierSource()]; + } + if (((causes[0] + causes[1] + causes[2] + causes[3]) & 0xffff) == 0) { + LOG_INFO(HW_GPU, "MME DMA reads: no wait {}, writer {}, barrier {}, constant buffer {}", + causes[0], causes[1], causes[2], causes[3]); + for (const auto& [source, count] : sources) { + LOG_INFO(HW_GPU, "MME DMA read barrier source {:#x}: {}", source, count); + } + } + } + if (target > consumed && !from_shadow) { + split.WaitForPosition(target, VideoCommon::DrainSite::MacroDmaRead); + } + } else { + split.Drain(VideoCommon::DrainSite::MacroDmaRead); + } + if (!precise && VideoCommon::QueryDma::TraceEnabled()) { split.Run( [this, macro, method, source, read_bytes] { if (VideoCommon::QueryDma::HasPendingQueries(memory_manager, *rasterizer, source, @@ -1372,7 +1518,14 @@ void Maxwell3D::FrontDmaRead(VideoCommon::GpuSplit& split, FrontState& fs, u32 m if (VideoCommon::Nx2Profile::Enabled()) { VideoCommon::Nx2Profile::Add(VideoCommon::Nx2Profile::Counter::DmaReadBytes, read_bytes); } - if (memory_manager.IsMemoryDirty(source, read_bytes, VideoCommon::CacheType::BufferCache)) { + // The caches belong to the back, which may still be running after a precise wait. + if (!precise && + memory_manager.IsMemoryDirty(source, read_bytes, VideoCommon::CacheType::BufferCache)) { + if (!precise_disabled) { + precise_disabled = true; + LOG_WARNING(HW_GPU, "Native MME DMA read of GPU-written memory at {:#x}: precise reads off", + source); + } split.Run( [memory = &memory_manager, source, read_bytes] { memory->FlushRegion(source, read_bytes, VideoCommon::CacheType::BufferCache); @@ -1381,7 +1534,9 @@ void Maxwell3D::FrontDmaRead(VideoCommon::GpuSplit& split, FrontState& fs, u32 m } u32 changed{}; for (u32 i = 0; i < words; ++i) { - const u32 word = memory_manager.Read(source + u64{i} * sizeof(u32)); + const u32 word = !shadow_words.empty() + ? shadow_words[i] + : memory_manager.Read(source + u64{i} * sizeof(u32)); if (method == 0x158) { fs.parameters.push_back(word); } else if (ram + i < native_data_ram.size()) { diff --git a/src/video_core/engines/maxwell_3d.h b/src/video_core/engines/maxwell_3d.h index 9247deff42..5df606151a 100644 --- a/src/video_core/engines/maxwell_3d.h +++ b/src/video_core/engines/maxwell_3d.h @@ -3402,6 +3402,8 @@ class Maxwell3D final : public EngineInterface { void FrontRunMacro(Core::System& system, VideoCommon::GpuSplit& split, FrontState& state); void FrontDmaRead(VideoCommon::GpuSplit& split, FrontState& state, u32 macro, u32 method, u32 count); + void FrontNoteWrites(VideoCommon::GpuSplit& split, FrontState& state, u32 method, u32 amount, + const u32* values); void FrontVerify(VideoCommon::GpuSplit& split, FrontState& state); void FinishFrontMacro(Core::System& system); std::unique_ptr front; diff --git a/src/video_core/engines/puller.cpp b/src/video_core/engines/puller.cpp index 8fb6a43f47..7d175fbf0a 100644 --- a/src/video_core/engines/puller.cpp +++ b/src/video_core/engines/puller.cpp @@ -58,6 +58,43 @@ void OnBack(const Func& func) { } } +// EDEN_NX2_MME_DMA_STATS: how semaphore acquires end and how long they hold the front. +struct AcquireStats { + enum Kind : size_t { Fast, Own, Learned, AfterWait, AfterDrain, Suspended, Count }; + std::array counts{}; + u64 wait_ns{}; + u64 drain_ns{}; + u64 suspended_ns{}; + std::chrono::steady_clock::time_point last_log{std::chrono::steady_clock::now()}; + + static AcquireStats* Get() { + static const bool enabled = std::getenv("EDEN_NX2_MME_DMA_STATS") != nullptr; + static AcquireStats stats; + return enabled ? &stats : nullptr; + } + void Note(Kind kind) { + ++counts[kind]; + const auto now = std::chrono::steady_clock::now(); + if (now - last_log < std::chrono::seconds{5}) { + return; + } + last_log = now; + LOG_INFO(HW_GPU, + "Acquires (5 s): fast {}, own release {}, learned {}, met after wait {} ({} ms), " + "met after drain {} ({} ms), suspended {} ({} ms)", + counts[Fast], counts[Own], counts[Learned], counts[AfterWait], wait_ns / 1000000, + counts[AfterDrain], drain_ns / 1000000, counts[Suspended], suspended_ns / 1000000); + counts = {}; + wait_ns = drain_ns = suspended_ns = 0; + } +}; + +u64 NsSince(std::chrono::steady_clock::time_point start) { + return static_cast(std::chrono::duration_cast( + std::chrono::steady_clock::now() - start) + .count()); +} + void DrainBack() { VideoCommon::GpuSplit::Get().Drain(VideoCommon::DrainSite::Acquire); } @@ -130,6 +167,7 @@ bool Puller::ResumeAcquire(DmaPusher& dma_pusher) { // counting those made the count odd ("suspended") at random, so submit-time snapshots were // thrown away for about half of all entries (CommandList::snapshot_suspensions). if (pending_acquire) { + if (auto* stats = AcquireStats::Get()) stats->suspended_ns += NsSince(pending_acquire->since); pending_acquire.reset(); dma_pusher.suspensions.fetch_add(1, std::memory_order_release); Nx2SemaphoreWaiters::Get().Remove(this); @@ -211,6 +249,21 @@ void Puller::ProcessFenceActionMethod(DmaPusher& dma_pusher) { } } +void Puller::NoteRelease(GPUVAddr address, u32 payload) { + auto& split = VideoCommon::GpuSplit::Get(); + if (!split.Active()) { + return; + } + if (queued_releases.size() >= 1024) { + const u64 done = split.Consumed(); + std::erase_if(queued_releases, [done](const auto& entry) { return entry.second.position <= done; }); + if (queued_releases.size() >= 1024) { + queued_releases.clear(); + } + } + queued_releases[address] = {payload, split.Position()}; +} + void Puller::ProcessSemaphoreTriggerMethod(DmaPusher& dma_pusher) { const auto semaphoreOperationMask = 0xF; const auto op = GpuSemaphoreOperation(regs.semaphore_trigger & semaphoreOperationMask); @@ -238,6 +291,10 @@ void Puller::ProcessSemaphoreTriggerMethod(DmaPusher& dma_pusher) { OnBack([rasterizer = dma_pusher.rasterizer, sequence_address, properties, payload] { rasterizer->Query(sequence_address, VideoCommon::QueryType::Payload, properties, payload, 0); }); + const u64 size = properties == VideoCommon::QueryPropertiesFlags::IsAFence ? 4 : 16; + VideoCommon::GpuSplit::Get().NoteMemoryWrite(&dma_pusher.memory_manager, sequence_address, + sequence_address + size); + NoteRelease(sequence_address, payload); } else if (g_nx2_ampere) { // Switch 2 waits on semaphores the CPU (or another channel) releases, e.g. once it has // written the commands this channel runs next: suspend the channel until the value is @@ -255,8 +312,104 @@ void Puller::ProcessSemaphoreTriggerMethod(DmaPusher& dma_pusher) { return; } VideoCommon::Nx2Profile::Add(VideoCommon::Nx2Profile::Counter::AcquireChecks, 1); + // A greater-or-equal acquire whose value is already in guest memory is met: releases only + // raise the value, and one still queued for the back (or held in its caches) is newer. + // Skip draining the back for it (~8% of the front in Ocarina of Time); later command + // fetches still wait for the work before it (NoteSync). + auto& split = VideoCommon::GpuSplit::Get(); + if (op == GpuSemaphoreOperation::AcquireGequal && split.Active() && + acquire.Matches(dma_pusher.memory_manager.Read(acquire.address))) { + if (auto* stats = AcquireStats::Get()) stats->Note(AcquireStats::Fast); + split.NoteSync(); + return; + } + if (split.Active()) { + // This channel's own release, still queued, meets it once the back runs it, which is + // before anything sent after the acquire. + if (const auto it = queued_releases.find(acquire.address); + it != queued_releases.end() && it->second.position > split.Consumed() && + acquire.Matches(it->second.payload)) { + if (auto* stats = AcquireStats::Get()) stats->Note(AcquireStats::Own); + split.NoteSync(); + return; + } + } + if (split.Active()) { + // Only queued work that writes the semaphore (or untracked writes) can change it. + const u64 target = split.ReadReadyPosition(&dma_pusher.memory_manager, acquire.address, + acquire.address + 4); + if (static const bool stats = std::getenv("EDEN_NX2_MME_DMA_STATS") != nullptr; stats) { + static std::array causes{}; + const auto own = queued_releases.find(acquire.address); + const size_t cause = target <= split.Consumed() ? 0 + : target == split.WriterPosition() ? 1 + : target == split.MemoryBarrierPosition() ? 2 + : own != queued_releases.end() ? 3 + : 4; + ++causes[cause]; + if (cause == 3 && causes[3] < 16) { + LOG_INFO(HW_GPU, "Acquire op {} {:#x} at {:#x}: own release {:#x}", + static_cast(op), acquire.value, acquire.address, + own->second.payload); + } + if (((causes[0] + causes[1] + causes[2] + causes[3] + causes[4]) & 0x3fff) == 0) { + LOG_INFO(HW_GPU, + "Acquire waits: none {}, writer {}, barrier {} (source {:#x}), own " + "release {}, other write {}", + causes[0], causes[1], causes[2], split.MemoryBarrierSource(), + causes[3], causes[4]); + } + } + // An address queued GPU work met 8 times in a row (compute completion, releases) is + // trusted to be met by the work before it; the back checks each skipped one and stops + // trusting the address if it was not. + // Any queued write the back makes before this point (compute, releases, reports). + const bool queued_writer = target > split.Consumed(); + if (has_unmet_acquires.exchange(false, std::memory_order_acquire)) { + std::scoped_lock lock{unmet_mutex}; + for (const GPUVAddr unmet : unmet_acquires) { + gpu_met_acquires[unmet] = ~0U; + LOG_WARNING(HW_GPU, "Acquire at {:#x} was not met by the GPU work before it", + unmet); + } + unmet_acquires.clear(); + } + if (queued_writer) { + const auto it = gpu_met_acquires.find(acquire.address); + if (it != gpu_met_acquires.end() && it->second >= 8 && it->second != ~0U) { + split.Defer([this, memory = &dma_pusher.memory_manager, acquire_check = acquire] { + if (!acquire_check.Matches(memory->Read(acquire_check.address))) { + std::scoped_lock lock{unmet_mutex}; + unmet_acquires.push_back(acquire_check.address); + has_unmet_acquires.store(true, std::memory_order_release); + } + }); + if (auto* stats = AcquireStats::Get()) stats->Note(AcquireStats::Learned); + split.NoteSync(); + return; + } + } + const auto wait_start = std::chrono::steady_clock::now(); + split.WaitForPosition(target, VideoCommon::DrainSite::Acquire); + if (auto* stats = AcquireStats::Get()) stats->wait_ns += NsSince(wait_start); + const bool met = acquire.Matches(dma_pusher.memory_manager.Read(acquire.address)); + if (queued_writer) { + u32& count = gpu_met_acquires[acquire.address]; + if (count != ~0U) { + count = met ? count + 1 : 0; + } + } + if (met) { + if (auto* stats = AcquireStats::Get()) stats->Note(AcquireStats::AfterWait); + split.NoteSync(); + return; + } + } + const auto drain_start = std::chrono::steady_clock::now(); DrainBack(); + if (auto* stats = AcquireStats::Get()) stats->drain_ns += NsSince(drain_start); if (acquire.Matches(ReadSemaphoreWord(dma_pusher, acquire.address))) { + if (auto* stats = AcquireStats::Get()) stats->Note(AcquireStats::AfterDrain); return; } // Registered before the fences run, so a release among them that meets the value counts @@ -268,10 +421,13 @@ void Puller::ProcessSemaphoreTriggerMethod(DmaPusher& dma_pusher) { ReleaseFences(dma_pusher); if (!waiters.Latched(this) && !acquire.Matches(ReadSemaphoreWord(dma_pusher, acquire.address))) { pending_acquire = acquire; + pending_acquire->since = std::chrono::steady_clock::now(); + if (auto* stats = AcquireStats::Get()) stats->Note(AcquireStats::Suspended); dma_pusher.suspensions.fetch_add(1, std::memory_order_release); VideoCommon::Nx2Profile::Add(VideoCommon::Nx2Profile::Counter::AcquireWaits, 1); } else { waiters.Remove(this); + if (auto* stats = AcquireStats::Get()) stats->Note(AcquireStats::AfterDrain); } } else { DrainBack(); @@ -312,6 +468,8 @@ void Puller::ProcessSemaphoreRelease(DmaPusher& dma_pusher) { rasterizer->Query(sequence_address, VideoCommon::QueryType::Payload, VideoCommon::QueryPropertiesFlags::IsAFence, payload, 0); }); + VideoCommon::GpuSplit::Get().NoteMemoryWrite(&dma_pusher.memory_manager, sequence_address, + sequence_address + 4); } void Puller::ProcessSemaphoreAcquire(DmaPusher& dma_pusher) { @@ -341,8 +499,33 @@ void Puller::ProcessSemaphoreExecute(DmaPusher& dma_pusher, u32 argument) { rasterizer->Query(address, VideoCommon::QueryType::Payload, VideoCommon::QueryPropertiesFlags::IsAFence, payload, 0); }); + VideoCommon::GpuSplit::Get().NoteMemoryWrite(&dma_pusher.memory_manager, address, + address + 4); + NoteRelease(address, payload); return; } + auto& split = VideoCommon::GpuSplit::Get(); + if (operation <= 3 && operation != 1 && split.Active()) { + // This channel's own queued release meets it before anything sent after the acquire. + if (const auto it = queued_releases.find(address); + it != queued_releases.end() && it->second.position > split.Consumed()) { + const u32 released = it->second.payload; + if (operation == 0 ? released == payload + : operation == 2 ? released >= payload + : static_cast(released - payload) >= 0) { + split.NoteSync(); + return; + } + } + } + if ((operation == 2 || operation == 3) && split.Active()) { + // A greater-or-equal value already in memory stays met: skip the back drain. + const u32 current = dma_pusher.memory_manager.Read(address); + if (operation == 2 ? current >= payload : static_cast(current - payload) >= 0) { + split.NoteSync(); + return; + } + } // Acquires: check once after flushing pending fences; never block the GPU thread on them. DrainBack(); ReleaseFences(dma_pusher); diff --git a/src/video_core/engines/puller.h b/src/video_core/engines/puller.h index 2b9e2c5df3..463b6eda73 100644 --- a/src/video_core/engines/puller.h +++ b/src/video_core/engines/puller.h @@ -7,9 +7,12 @@ #pragma once #include +#include +#include #include #include #include +#include #include #include "common/bit_field.h" #include "common/common_funcs.h" @@ -149,6 +152,19 @@ class Puller final { [[nodiscard]] bool ExecuteMethodOnEngine(DmaPusher& dma_pusher, u32 method); std::optional pending_acquire; + /// Semaphore releases this channel queued for the split's back: payload and ring position. + struct QueuedRelease { + u32 payload; + u64 position; + }; + std::unordered_map queued_releases; + void NoteRelease(GPUVAddr address, u32 payload); + /// Acquires at an address that queued GPU work met each time: how often in a row, or ~0U + /// once the back found one unmet after skipping its wait. + std::unordered_map gpu_met_acquires; + std::mutex unmet_mutex; + std::vector unmet_acquires; + std::atomic has_unmet_acquires{false}; /// Mapping of command subchannels to their bound engine ids std::array bound_engines{}; diff --git a/src/video_core/gpu_split.cpp b/src/video_core/gpu_split.cpp index 5adb071273..535c4c8369 100644 --- a/src/video_core/gpu_split.cpp +++ b/src/video_core/gpu_split.cpp @@ -140,6 +140,11 @@ void GpuSplit::Stop() { ring.reset(); write_position = 0; last_writer_position = 0; + last_barrier_position = 0; + write_blocks.clear(); + opaque_blocks.clear(); + latest_block_write = 0; + latest_opaque_write = 0; last_sync_position = 0; published_local = 0; consumed_cache = 0; diff --git a/src/video_core/gpu_split.h b/src/video_core/gpu_split.h index 2e9250ff53..7b8381fe56 100644 --- a/src/video_core/gpu_split.h +++ b/src/video_core/gpu_split.h @@ -3,6 +3,7 @@ #pragma once +#include #include #include #include @@ -10,6 +11,8 @@ #include #include #include +#include +#include #include "common/common_types.h" @@ -175,6 +178,71 @@ class GpuSplit { /// of it was still queued. bool WaitForSync(); + /// Work that may write guest memory at addresses the front does not track was just sent. + /// `source` names the sender for EDEN_NX2_MME_DMA_STATS. + void NoteMemoryBarrier(u32 source = 0) { + last_barrier_position = write_position; + last_barrier_source = source; + } + [[nodiscard]] u32 MemoryBarrierSource() const { + return last_barrier_source; + } + /// Where the last untracked memory write (or compute and copy work) was sent. + [[nodiscard]] u64 MemoryBarrierPosition() const { + return std::max(last_barrier_position, last_writer_position); + } + [[nodiscard]] u64 WriterPosition() const { + return last_writer_position; + } + [[nodiscard]] u64 Position() const { + return write_position; + } + /// How far the back has executed. + [[nodiscard]] u64 Consumed() { + consumed_cache = consumed.load(std::memory_order_acquire); + return consumed_cache; + } + /// Waits until the back executed everything sent before `target`. + void WaitForPosition(u64 target, DrainSite site) { + WaitFor(std::min(target, write_position), site); + } + + /// Work just sent writes [begin, end) of the GPU address space `space` (a memory manager). + /// `opaque`: the front does not know the data written. + void NoteMemoryWrite(const void* space, u64 begin, u64 end, bool opaque = true) { + if (end <= begin) { + return; + } + if (((end - 1) >> WriteBlockShift) - (begin >> WriteBlockShift) >= MaxWriteBlocks) { + NoteMemoryBarrier(0x40001); + return; + } + if (write_blocks.size() >= 8192 && Consumed() >= latest_block_write) { + write_blocks.clear(); + opaque_blocks.clear(); + } + if (write_blocks.size() >= 65536) { + write_blocks.clear(); + opaque_blocks.clear(); + NoteMemoryBarrier(0x40000); + } + NoteBlocks(write_blocks, space, begin, end); + latest_block_write = write_position; + if (opaque) { + NoteBlocks(opaque_blocks, space, begin, end); + latest_opaque_write = write_position; + } + } + /// The position the back must reach before [begin, end) of `space` can be read: the last + /// untracked write, or the last queued write that overlaps it. + [[nodiscard]] u64 ReadReadyPosition(const void* space, u64 begin, u64 end) { + return ReadyPosition(write_blocks, latest_block_write, space, begin, end); + } + /// As ReadReadyPosition, for writes whose data the front does not know. + [[nodiscard]] u64 OpaqueReadyPosition(const void* space, u64 begin, u64 end) { + return ReadyPosition(opaque_blocks, latest_opaque_write, space, begin, end); + } + private: friend struct GpuSplitTestAccess; @@ -246,6 +314,42 @@ class GpuSplit { // Front side. u64 write_position{}; u64 last_writer_position{}; + u64 last_barrier_position{}; + u32 last_barrier_source{}; + // Queued tracked writes: the ring position after the last write to each 64-byte block. + static constexpr u64 WriteBlockShift = 6; + static constexpr u64 MaxWriteBlocks = 1024; + static u64 WriteBlockKey(const void* space, u64 block) { + // Aliasing only makes a read wait for an unrelated write. + return block ^ (static_cast(reinterpret_cast(space)) << 40); + } + std::unordered_map write_blocks; + std::unordered_map opaque_blocks; + u64 latest_block_write{}; + u64 latest_opaque_write{}; + void NoteBlocks(std::unordered_map& blocks, const void* space, u64 begin, u64 end) { + for (u64 block = begin >> WriteBlockShift; block <= (end - 1) >> WriteBlockShift; ++block) { + blocks[WriteBlockKey(space, block)] = write_position; + } + } + u64 ReadyPosition(const std::unordered_map& blocks, u64 latest, const void* space, + u64 begin, u64 end) { + u64 target = MemoryBarrierPosition(); + if (end <= begin || blocks.empty() || latest <= target) { + return target; + } + const u64 first = begin >> WriteBlockShift; + const u64 last = (end - 1) >> WriteBlockShift; + if (last - first >= MaxWriteBlocks) { + return std::max(target, latest); + } + for (u64 block = first; block <= last; ++block) { + if (const auto it = blocks.find(WriteBlockKey(space, block)); it != blocks.end()) { + target = std::max(target, it->second); + } + } + return target; + } u64 last_sync_position{}; u64 published_local{}; u64 consumed_cache{}; diff --git a/src/video_core/gpu_thread.cpp b/src/video_core/gpu_thread.cpp index 7f3756953b..91d3fcb912 100644 --- a/src/video_core/gpu_thread.cpp +++ b/src/video_core/gpu_thread.cpp @@ -4,6 +4,7 @@ // SPDX-FileCopyrightText: Copyright 2019 yuzu Emulator Project // SPDX-License-Identifier: GPL-2.0-or-later +#include #include #include @@ -18,6 +19,7 @@ #include "video_core/dma_pusher.h" #include "video_core/gpu.h" #include "video_core/gpu_thread.h" +#include "video_core/nx2.h" #include "video_core/gpu_split.h" #include "video_core/nx2_profile.h" #include "video_core/host1x/host1x.h" @@ -38,6 +40,42 @@ void SignalCommand(SynchState& state, u64 fence, bool block) { } } +// EDEN_NX2_MME_DMA_STATS: where the GPU thread's time goes while channels are suspended. +struct ThreadStats { + enum Kind : size_t { PendingEmpty, Idle, Count }; + std::array ns{}; + std::chrono::steady_clock::time_point last_log{std::chrono::steady_clock::now()}; + + struct Timer { + Timer(ThreadStats* stats_, Kind kind_) + : stats{stats_}, kind{kind_}, start{std::chrono::steady_clock::now()} {} + ~Timer() { + if (stats) { + stats->ns[kind] += static_cast( + std::chrono::duration_cast( + std::chrono::steady_clock::now() - start) + .count()); + } + } + ThreadStats* stats; + Kind kind; + std::chrono::steady_clock::time_point start; + }; + + void MaybeLog(bool pending) { + const auto now = std::chrono::steady_clock::now(); + if (now - last_log < std::chrono::seconds{5}) { + return; + } + last_log = now; + LOG_INFO(HW_GPU, + "GPU thread (5 s): {} ms waiting for commands with channels suspended, {} ms " + "idle with none suspended (suspended now: {})", + ns[PendingEmpty] / 1000000, ns[Idle] / 1000000, pending); + ns = {}; + } +}; + } // Anonymous namespace ThreadManager::ThreadManager(Core::System& system_) @@ -61,11 +99,17 @@ void ThreadManager::StartThread(VideoCore::RendererBase& renderer, Core::Fronten auto last_progress = std::chrono::steady_clock::now(); namespace Profile = VideoCommon::Nx2Profile; auto& split = VideoCommon::GpuSplit::Get(); + ThreadStats* const thread_stats = + std::getenv("EDEN_NX2_MME_DMA_STATS") != nullptr ? new ThreadStats{} : nullptr; while (!stop_token.stop_requested()) { + if (thread_stats) { + thread_stats->MaybeLog(scheduler.HasPendingChannels()); + } Profile::Report(); split.MaybeStart(system); if (scheduler.HasPendingChannels()) { // Suspended channels wait on semaphores the CPU releases: poll them meanwhile. + const u64 seen_latch = Tegra::Nx2SemaphoreWaiters::Get().LatchEpoch(); { Profile::Scope scope{Profile::Zone::ResumePending}; scheduler.ResumePending(system.GPU()); @@ -75,16 +119,21 @@ void ThreadManager::StartThread(VideoCore::RendererBase& renderer, Core::Fronten // millisecond before sleeping, or every wait also waits out a host sleep // (title 1 waits ~25 times a second). Profile::Scope scope{Profile::Zone::PendingSleep}; + const ThreadStats::Timer timer{thread_stats, ThreadStats::PendingEmpty}; if (std::chrono::steady_clock::now() - last_progress < std::chrono::microseconds{500}) { std::this_thread::yield(); } else { - std::this_thread::sleep_for(std::chrono::milliseconds{1}); + // A GPU release that meets a suspended acquire wakes this at once; + // CPU releases are still found by polling. + Tegra::Nx2SemaphoreWaiters::Get().WaitForLatch( + seen_latch, std::chrono::milliseconds{1}); } continue; } } else { Profile::Scope scope{Profile::Zone::Idle}; + const ThreadStats::Timer timer{thread_stats, ThreadStats::Idle}; state.queue.PopWait(next, stop_token); } if (stop_token.stop_requested()) { diff --git a/src/video_core/nx2.h b/src/video_core/nx2.h index fc0cfd0487..381567919e 100644 --- a/src/video_core/nx2.h +++ b/src/video_core/nx2.h @@ -6,6 +6,8 @@ #include #include #include +#include +#include #include #include #include @@ -28,6 +30,12 @@ inline std::atomic g_nx2_ampere{false}; inline constexpr u64 Nx2WrittenPageBits = 12; inline std::array, (u64{1} << (36 - Nx2WrittenPageBits)) / 64> g_nx2_gpu_written_pages{}; +// Pages the front fetched command lists from after the compute and copy work before them ran, +// with no GPU write so far: later fetches from them skip that wait (DmaPusher::Step). A GPU write +// to one clears its bit and counts a miss. +inline std::array, (u64{1} << (36 - Nx2WrittenPageBits)) / 64> g_nx2_cpu_list_pages{}; +inline std::atomic g_nx2_cpu_list_misses{0}; + inline void Nx2MarkGpuWritten(u64 device_address, u64 size) { if (size == 0) { return; @@ -40,6 +48,11 @@ inline void Nx2MarkGpuWritten(u64 device_address, u64 size) { if ((word.load(std::memory_order_relaxed) & bit) == 0) { word.fetch_or(bit, std::memory_order_relaxed); } + auto& cpu_word = g_nx2_cpu_list_pages[page / 64]; + if ((cpu_word.load(std::memory_order_relaxed) & bit) != 0) { + cpu_word.fetch_and(~bit, std::memory_order_relaxed); + g_nx2_cpu_list_misses.fetch_add(1, std::memory_order_relaxed); + } } } @@ -79,6 +92,9 @@ inline void Nx2ClearGpuWrittenPages() { for (auto& word : g_nx2_command_list_pages) { word.store(0, std::memory_order_relaxed); } + for (auto& word : g_nx2_cpu_list_pages) { + word.store(0, std::memory_order_relaxed); + } g_nx2_has_command_list_pages.store(false, std::memory_order_relaxed); } @@ -371,12 +387,33 @@ class Nx2SemaphoreWaiters { if (count.load(std::memory_order_acquire) == 0) { return; } - std::scoped_lock lk{mutex}; - for (auto& w : waiters) { - if (w.address == device_address && !w.latched && Matches(w, value)) { - w.latched = true; + bool latched = false; + { + std::scoped_lock lk{mutex}; + for (auto& w : waiters) { + if (w.address == device_address && !w.latched && Matches(w, value)) { + w.latched = true; + latched = true; + } + } + if (latched) { + ++latch_epoch; } } + if (latched) { + latch_cv.notify_all(); + } + } + + [[nodiscard]] u64 LatchEpoch() { + std::scoped_lock lk{mutex}; + return latch_epoch; + } + + /// Sleeps until a waiter latches after `seen` (LatchEpoch) or `timeout` passes. + void WaitForLatch(u64 seen, std::chrono::microseconds timeout) { + std::unique_lock lk{mutex}; + latch_cv.wait_for(lk, timeout, [&] { return latch_epoch != seen; }); } private: @@ -402,6 +439,8 @@ class Nx2SemaphoreWaiters { } std::mutex mutex; + std::condition_variable latch_cv; + u64 latch_epoch{}; std::vector waiters; std::atomic count{}; }; From a1b555012a2dca864e3ac4bf2e93ffc546744c7c Mon Sep 17 00:00:00 2001 From: Celmax Date: Fri, 6 Nov 2026 20:53:11 +0100 Subject: [PATCH 02/16] buffer_cache: check native uniform banks per page on every bind, compare in place Native programs read a 64 KiB uniform bank with indirect offsets, and writes to it can go unnoticed by the memory tracker. nx2 uploaded such a bank again once a frame; games that rewrite per-draw data in it between draws then draw with stale words for a frame now and then (white flashes in Ocarina of Time, where this was measured, Eglantine 031e07436c). - Check the bank on every bind, comparing each of its pages with a snapshot of its last contents, and mark only the pages that changed as CPU-modified so the normal upload path refreshes them. Unchanged pages keep their tracking (marking costs two mprotect calls per page). - Small uniform buffers within one device page are compared in place instead of being copied to the scratch first; the slot's copy is what gets uploaded, so a later reuse finds exactly the uploaded bytes. - InvalidateGPUCache gathers the CPU-written ranges first so the rasterizer takes each cache's lock once instead of once per range. - EDEN_NX2_MME_DMA_STATS: checked bank pages (same, new, changed, and whether the tracker knew of the change) and page protection counts. The tests now expect page-granular marks instead of a direct upload. Co-Authored-By: Claude Opus 5.5 --- src/core/memory.cpp | 50 +++++++++++++++ src/tests/video_core/buffer_cache_ports.cpp | 33 +++++----- src/tests/video_core/uniform_stream_reuse.cpp | 4 ++ src/video_core/buffer_cache/buffer_cache.h | 61 +++++++++++++++---- .../buffer_cache/buffer_cache_base.h | 6 +- .../buffer_cache/uniform_stream_reuse.h | 25 +++++--- src/video_core/gpu.cpp | 13 +++- src/video_core/rasterizer_interface.h | 7 +++ .../renderer_vulkan/vk_rasterizer.cpp | 16 +++++ .../renderer_vulkan/vk_rasterizer.h | 1 + 10 files changed, 176 insertions(+), 40 deletions(-) diff --git a/src/core/memory.cpp b/src/core/memory.cpp index fd289eb5f4..8c558eb98d 100644 --- a/src/core/memory.cpp +++ b/src/core/memory.cpp @@ -6,6 +6,8 @@ // SPDX-License-Identifier: GPL-2.0-or-later #include +#include +#include #include #include #include @@ -494,10 +496,58 @@ struct Memory::Impl { } } + // EDEN_NX2_MME_DMA_STATS: how often pages are protected again after an upload and + // unprotected by a CPU write, and how many distinct pages that is. + struct ProtectStats { + std::mutex mutex; + std::array calls{}; + std::array pages{}; + std::unordered_map protected_pages; + std::chrono::steady_clock::time_point last_log{std::chrono::steady_clock::now()}; + + static ProtectStats* Get() { + static const bool enabled = std::getenv("EDEN_NX2_MME_DMA_STATS") != nullptr; + static ProtectStats stats; + return enabled ? &stats : nullptr; + } + + void Note(u64 vaddr, u64 size, bool cached) { + std::scoped_lock lk{mutex}; + const u64 first = vaddr >> YUZU_PAGEBITS; + const u64 last = (vaddr + size - 1) >> YUZU_PAGEBITS; + ++calls[cached]; + pages[cached] += last - first + 1; + if (cached) { + for (u64 page = first; page <= last; ++page) { + ++protected_pages[page]; + } + } + const auto now = std::chrono::steady_clock::now(); + if (now - last_log < std::chrono::seconds{5}) { + return; + } + last_log = now; + size_t hot = 0; + for (const auto& [page, count] : protected_pages) { + hot += count >= 100 ? 1 : 0; + } + LOG_INFO(HW_Memory, + "Page protection (5 s): protect {} calls {} pages ({} distinct, {} protected " + "100+ times), unprotect {} calls {} pages", + calls[1], pages[1], protected_pages.size(), hot, calls[0], pages[0]); + calls = {}; + pages = {}; + protected_pages.clear(); + } + }; + void RasterizerMarkRegionCached(u64 vaddr, u64 size, bool cached) { if (vaddr == 0 || !AddressSpaceContains(*current_page_table, vaddr, size)) { return; } + if (auto* stats = ProtectStats::Get()) { + stats->Note(vaddr, size, cached); + } if (current_page_table->fastmem_arena) { Common::MemoryPermission perm{}; diff --git a/src/tests/video_core/buffer_cache_ports.cpp b/src/tests/video_core/buffer_cache_ports.cpp index 46db48933a..99e077e778 100644 --- a/src/tests/video_core/buffer_cache_ports.cpp +++ b/src/tests/video_core/buffer_cache_ports.cpp @@ -225,9 +225,6 @@ struct BufferCacheTestAccess { channel.uniform_buffers[0][0] = {.device_addr = address, .size = size, .buffer_id = id}; cache.BindHostGraphicsUniformBuffer(0, 0, 0, true); } - static void NextFrame(Cache& cache) { - cache.native_indirect_uniforms_this_frame.clear(); - } static u32 CpuMarks(Cache& cache) { return cache.memory_tracker.cpu_marks; } @@ -540,7 +537,7 @@ TEST_CASE("Command page readers remain ordered during batch completion", "[buffe REQUIRE(f.runtime.downloads == 1); } -TEST_CASE("Native uniform banks upload directly without overwriting GPU-owned bytes", +TEST_CASE("Native uniform banks mark only changed pages and keep GPU-owned bytes", "[buffer_ports]") { Fixture f; f.memory->EnableNativeTiling(); @@ -555,20 +552,26 @@ TEST_CASE("Native uniform banks upload directly without overwriting GPU-owned by std::memset(f.bytes + 0x10000, 0x5a, 0x10000); Access::Uniform(*f.cache, channel, id, 0x20000, 0x10000); REQUIRE(f.runtime.upload_bytes == 0x10000); - REQUIRE(Access::CpuMarks(*f.cache) == 0); + REQUIRE(Access::CpuMarks(*f.cache) == 16); REQUIRE(Access::Get(*f.cache, id).bytes[0xffff] == 0x5a); + const u32 uploads = f.runtime.uploads; + Access::Uniform(*f.cache, channel, id, 0x20000, 0x10000); + REQUIRE(f.runtime.uploads == uploads); + REQUIRE(Access::CpuMarks(*f.cache) == 16); + // A write the tracker missed is found on the next bind, and only its page is marked. + std::memset(f.bytes + 0x13000, 0x77, 0x1000); Access::Uniform(*f.cache, channel, id, 0x20000, 0x10000); - REQUIRE(f.runtime.uploads == 1); - Access::NextFrame(*f.cache); + REQUIRE(Access::CpuMarks(*f.cache) == 17); + REQUIRE(f.runtime.upload_bytes == 0x11000); + REQUIRE(Access::Get(*f.cache, id).bytes[0x3000] == 0x77); Access::Dirty(*f.cache, 0x21000, 0x1000); std::memset(f.bytes + 0x10000, 0xa5, 0x10000); Access::Uniform(*f.cache, channel, id, 0x20000, 0x10000); - REQUIRE(f.runtime.uploads == 1); + REQUIRE(f.runtime.upload_bytes == 0x11000); REQUIRE(Access::Get(*f.cache, id).bytes[0x1000] == 0x5a); } -TEST_CASE("Native uniform uploads stop at holes and keep unaligned bank boundaries", - "[buffer_ports]") { +TEST_CASE("Native uniform bank checks stop at holes", "[buffer_ports]") { Fixture f; f.memory->EnableNativeTiling(); Tegra::Control::ChannelState state{0}; @@ -582,18 +585,16 @@ TEST_CASE("Native uniform uploads stop at holes and keep unaligned bank boundari std::memset(f.bytes + 0x10000, 0x6b, 0x20000); f.device.Unmap(0x22000, 0x1000); Access::Uniform(*f.cache, channel, id, 0x20123, 0x10000); - REQUIRE(f.runtime.upload_bytes == 0x1edd); + REQUIRE(Access::CpuMarks(*f.cache) == 2); + REQUIRE(f.runtime.upload_bytes == 0x2000); const auto& data = Access::Get(*f.cache, id).bytes; - REQUIRE(data[0x122] == 0); REQUIRE(data[0x123] == 0x6b); REQUIRE(data[0x1fff] == 0x6b); REQUIRE(data[0x2000] == 0); REQUIRE(data[0x3000] == 0); - REQUIRE(Access::CpuMarks(*f.cache) == 0); f.device.Map(0x22000, 0x22000, 0x1000, f.asid); - Access::NextFrame(*f.cache); Access::Uniform(*f.cache, channel, id, 0x20123, 0x10000); - REQUIRE(f.runtime.upload_bytes == 0x11edd); + REQUIRE(Access::CpuMarks(*f.cache) == 17); + REQUIRE(data[0x2000] == 0x6b); REQUIRE(data[0x10122] == 0x6b); - REQUIRE(data[0x10123] == 0); } diff --git a/src/tests/video_core/uniform_stream_reuse.cpp b/src/tests/video_core/uniform_stream_reuse.cpp index 6059b70eeb..bcfee7e35c 100644 --- a/src/tests/video_core/uniform_stream_reuse.cpp +++ b/src/tests/video_core/uniform_stream_reuse.cpp @@ -23,6 +23,10 @@ struct Memory { Check(address + size <= bytes.size(), "read outside guest memory"); std::memcpy(destination, bytes.data() + address, size); } + template + T* GetPointer(u64 address) { + return address < bytes.size() ? reinterpret_cast(bytes.data() + address) : nullptr; + } }; struct Runtime { diff --git a/src/video_core/buffer_cache/buffer_cache.h b/src/video_core/buffer_cache/buffer_cache.h index a248222d09..5a6760f0ed 100644 --- a/src/video_core/buffer_cache/buffer_cache.h +++ b/src/video_core/buffer_cache/buffer_cache.h @@ -112,7 +112,6 @@ void BufferCache

::TickFrame() { return; } runtime.TickFrame(slot_buffers); - native_indirect_uniforms_this_frame.clear(); if (frame_tick % 600 == 0 && !uploaded_buffer_ranges.empty()) { for (const auto& [address, size] : uploaded_buffer_ranges) { device_memory.HintCold(address, size); @@ -1355,26 +1354,62 @@ void BufferCache

::BindHostGraphicsUniformBuffer(size_t stage, u32 index, u32 }(); // Native programs keep tables next to their code in a 64 KiB bank read with indirect offsets // (Splatoon Raiders: the dialogue box corners in c1). Writes to that memory can go unnoticed by - // the memory tracker, leaving stale host contents, so upload such a bank again once per frame. + // the memory tracker, leaving stale host contents, so check such a bank on every bind: games + // rewrite per-draw data between draws, and checking once per frame left it stale for a frame + // now and then (white flashes in Ocarina of Time, Eglantine 031e07436c). if (has_host_buffer && (*channel_state->uniform_buffer_sizes)[stage][index] == 0x10000 && size != 0 && gpu_memory != nullptr && gpu_memory->UsesNativeTiling() && - (Nx2Perf::Verify() || !native_indirect_uniforms_this_frame.contains(device_addr)) && - !memory_tracker.IsRegionGpuModified(device_addr, size) && - native_indirect_uniforms_this_frame.insert(device_addr).second) { + !memory_tracker.IsRegionGpuModified(device_addr, size)) { // The bank size is only an upper limit: stop at the end of the mapped memory. u64 mapped_size = 0; while (mapped_size < size && device_memory.template GetPointer(device_addr + mapped_size) != nullptr) { mapped_size += Core::DEVICE_PAGESIZE - ((device_addr + mapped_size) & Core::DEVICE_PAGEMASK); } - const u64 upload_size = (std::min)(mapped_size, size); - if (upload_size != 0) { - std::array copies{BufferCopy{ - .src_offset = 0, - .dst_offset = buffer.Offset(device_addr), - .size = upload_size, - }}; - UploadMemory(buffer, upload_size, upload_size, copies); + const u64 check_size = (std::min)(mapped_size, size); + // Banks slide through larger buffers, a new address per draw: compare each page with + // its last contents and mark only the pages that changed. + if (native_indirect_page_snapshots.size() >= 16384) { + native_indirect_page_snapshots.clear(); + } + const DAddr first_page = Common::AlignDown(device_addr, Core::DEVICE_PAGESIZE); + const DAddr end = device_addr + check_size; + for (DAddr page = first_page; page < end; page += Core::DEVICE_PAGESIZE) { + const u8* const guest = device_memory.template GetPointer(page); + if (guest == nullptr) { + continue; + } + auto [it, inserted] = native_indirect_page_snapshots.try_emplace(page); + const bool same = + !inserted && std::memcmp(it->second.data(), guest, Core::DEVICE_PAGESIZE) == 0; + // EDEN_NX2_MME_DMA_STATS: how many checked pages had changed. + if (static const bool stats = std::getenv("EDEN_NX2_MME_DMA_STATS") != nullptr; + stats) { + static std::array counts{}; + static std::array changed_state{}; + ++counts[same ? 0 : (inserted ? 1 : 2)]; + if (!same && !inserted) { + // Did the tracker already know (CPU-modified), or was the page clean, and + // was it GPU-modified? + const bool known = + memory_tracker.IsRegionCpuModified(page, Core::DEVICE_PAGESIZE); + const bool gpu = + memory_tracker.IsRegionGpuModified(page, Core::DEVICE_PAGESIZE); + ++changed_state[(known ? 1 : 0) | (gpu ? 2 : 0)]; + } + if (((counts[0] + counts[1] + counts[2]) & 0x3ffff) == 0) { + LOG_INFO(HW_GPU, + "Native bank pages checked: same {}, new {}, changed {} (tracker " + "clean {}, CPU-modified {}, GPU {}, both {})", + counts[0], counts[1], counts[2], changed_state[0], + changed_state[1], changed_state[2], changed_state[3]); + } + } + if (same) { + continue; + } + std::memcpy(it->second.data(), guest, Core::DEVICE_PAGESIZE); + memory_tracker.MarkRegionAsCpuModified(page, Core::DEVICE_PAGESIZE); } } const bool use_fast_buffer = needs_alignment_stream diff --git a/src/video_core/buffer_cache/buffer_cache_base.h b/src/video_core/buffer_cache/buffer_cache_base.h index e7971b444b..62fe646626 100644 --- a/src/video_core/buffer_cache/buffer_cache_base.h +++ b/src/video_core/buffer_cache/buffer_cache_base.h @@ -444,8 +444,10 @@ class BufferCache : public VideoCommon::ChannelSetupCaches native_indirect_uniforms_this_frame; + /// The guest words of each page of native programs' indirectly read banks when last checked: + /// unchanged pages keep their tracking (marking them CPU-modified costs two mprotect calls + /// per page). + std::unordered_map> native_indirect_page_snapshots; private: template diff --git a/src/video_core/buffer_cache/uniform_stream_reuse.h b/src/video_core/buffer_cache/uniform_stream_reuse.h index 7ba31c4717..30a31b42e4 100644 --- a/src/video_core/buffer_cache/uniform_stream_reuse.h +++ b/src/video_core/buffer_cache/uniform_stream_reuse.h @@ -97,8 +97,17 @@ template } Reuse& reuse = UniformStreamReuseState(); typename Reuse::Slot& slot = reuse.slots[stage][index]; - u8* const scratch = reuse.scratch.data(); - device_memory.ReadBlockUnsafe(device_addr, scratch, size); + // Within one device page the guest bytes are contiguous: use them in place instead of + // copying them to the scratch first. + constexpr u64 PageMask = (u64{1} << 12) - 1; + const u8* scratch = nullptr; + if ((device_addr & PageMask) + size <= PageMask + 1) { + scratch = device_memory.template GetPointer(device_addr); + } + if (scratch == nullptr) { + device_memory.ReadBlockUnsafe(device_addr, reuse.scratch.data(), size); + scratch = reuse.scratch.data(); + } const bool same_bytes = slot.size == size && std::memcmp(slot.bytes.data(), scratch, size) == 0; @@ -106,15 +115,17 @@ template runtime.RebindStreamUniform(slot.ref, size); return true; } - const std::span span = runtime.BindMappedUniformBuffer(stage, binding_index, size); - std::memcpy(span.data(), scratch, size); - slot.ref = runtime.LastStreamUniformRef(); - // After the allocation: the tick the stream region is tagged with is the one current now. - slot.tick = runtime.CurrentTick(); + // Upload the slot's copy: guest memory may change while this runs, and a later reuse must + // find exactly the bytes that were uploaded. if (!same_bytes) { slot.size = size; std::memcpy(slot.bytes.data(), scratch, size); } + const std::span span = runtime.BindMappedUniformBuffer(stage, binding_index, size); + std::memcpy(span.data(), slot.bytes.data(), size); + slot.ref = runtime.LastStreamUniformRef(); + // After the allocation: the tick the stream region is tagged with is the one current now. + slot.tick = runtime.CurrentTick(); return true; } diff --git a/src/video_core/gpu.cpp b/src/video_core/gpu.cpp index 2ec20d6da2..e33dfe4781 100644 --- a/src/video_core/gpu.cpp +++ b/src/video_core/gpu.cpp @@ -109,10 +109,19 @@ struct GPU::Impl { /// Synchronizes CPU writes with Host GPU memory. void InvalidateGPUCache() { - std::function callback_writes([this](PAddr address, size_t size) { - renderer->ReadRasterizer()->OnCacheInvalidation(address, size); + // Gathered first so the rasterizer takes each cache's lock once, not once per range: the + // CPU's write traps hold them too. + static thread_local std::vector> ranges; + ranges.clear(); + std::function callback_writes([](PAddr address, size_t size) { + if (address != 0 && size != 0) { + ranges.emplace_back(address, size); + } }); system.GatherGPUDirtyMemory(callback_writes); + if (!ranges.empty()) { + renderer->ReadRasterizer()->OnCacheInvalidations(ranges); + } } /// Signal the ending of command list. diff --git a/src/video_core/rasterizer_interface.h b/src/video_core/rasterizer_interface.h index 9cebc4f29f..b6f790e98b 100644 --- a/src/video_core/rasterizer_interface.h +++ b/src/video_core/rasterizer_interface.h @@ -153,6 +153,13 @@ class RasterizerInterface { /// Notify rasterizer that any caches of the specified region are desync with guest virtual void OnCacheInvalidation(PAddr addr, u64 size) = 0; + /// OnCacheInvalidation for each range, taking each cache's lock once. + virtual void OnCacheInvalidations(std::span> ranges) { + for (const auto& [addr, size] : ranges) { + OnCacheInvalidation(addr, size); + } + } + virtual bool TryCacheInvalidation(DAddr addr, u64 size) { if (MustFlushRegion(addr, size, VideoCommon::CacheType::BufferCache)) return false; OnCacheInvalidation(addr, size); diff --git a/src/video_core/renderer_vulkan/vk_rasterizer.cpp b/src/video_core/renderer_vulkan/vk_rasterizer.cpp index 628dd72d97..8a903039cf 100644 --- a/src/video_core/renderer_vulkan/vk_rasterizer.cpp +++ b/src/video_core/renderer_vulkan/vk_rasterizer.cpp @@ -1707,6 +1707,22 @@ void RasterizerVulkan::OnCacheInvalidation(DAddr addr, u64 size) { pipeline_cache.InvalidateRegion(addr, size); } +void RasterizerVulkan::OnCacheInvalidations(std::span> ranges) { + { + std::scoped_lock lock{texture_cache.mutex}; + for (const auto& [addr, size] : ranges) { + texture_cache.WriteMemory(addr, size); + } + } + { + std::scoped_lock lock{buffer_cache.mutex}; + for (const auto& [addr, size] : ranges) { + buffer_cache.WriteMemory(addr, size); + } + } + pipeline_cache.InvalidateRegions(ranges); +} + bool RasterizerVulkan::TryCacheInvalidation(DAddr addr, u64 size) { if (addr == 0 || size == 0) return true; { diff --git a/src/video_core/renderer_vulkan/vk_rasterizer.h b/src/video_core/renderer_vulkan/vk_rasterizer.h index c33b0ed3b3..00db6e401e 100644 --- a/src/video_core/renderer_vulkan/vk_rasterizer.h +++ b/src/video_core/renderer_vulkan/vk_rasterizer.h @@ -117,6 +117,7 @@ class RasterizerVulkan final : public VideoCore::RasterizerInterface, void PrepareRegionWrite(DAddr addr, u64 size, VideoCommon::CacheType which) override; void InnerInvalidation(std::span> sequences) override; void OnCacheInvalidation(DAddr addr, u64 size) override; + void OnCacheInvalidations(std::span> ranges) override; bool TryCacheInvalidation(DAddr addr, u64 size) override; bool OnCPUWrite(DAddr addr, u64 size) override; void InvalidateGPUCache() override; From 5ac3cc88f839a8bf9b569ae3d8597b0e60d3bfbd Mon Sep 17 00:00:00 2001 From: Celmax Date: Fri, 6 Nov 2026 20:53:27 +0100 Subject: [PATCH 03/16] video_core: write out pending query reports before macro DMA reads of them Builds on Lichii #215 by WeslieDE (which includes #204 by vouion): below High GPU accuracy the query cache holds report and semaphore writes until a fence, and a fence runs its callback at once, so macro DMA reads of report memory saw stale occlusion counts (white flashes in Ocarina of Time, where this was found). FlushQueryResults writes the region's reports out and waits for an operation queued behind the pending report writes. Without the split the macro engine flushes this way before its DMA reads. On the split's precise front (previous commits), the flush runs on the back only for reads of pages report_semaphore writes went to, and not for reads answered from queued inline constant buffer words. Co-Authored-By: Claude Opus 5.5 --- src/video_core/engines/maxwell_3d.cpp | 27 +++++++++++++++++++ src/video_core/macro.cpp | 8 +++--- src/video_core/query_cache/query_cache_base.h | 11 +++++--- src/video_core/query_dma.h | 15 +++++++++++ src/video_core/rasterizer_interface.h | 5 ++++ .../renderer_vulkan/vk_rasterizer.cpp | 22 +++++++++++++++ .../renderer_vulkan/vk_rasterizer.h | 1 + 7 files changed, 82 insertions(+), 7 deletions(-) diff --git a/src/video_core/engines/maxwell_3d.cpp b/src/video_core/engines/maxwell_3d.cpp index 1635cdf332..25fecd3ea1 100644 --- a/src/video_core/engines/maxwell_3d.cpp +++ b/src/video_core/engines/maxwell_3d.cpp @@ -12,6 +12,7 @@ #include #include #include +#include #include "common/assert.h" #include "common/bit_util.h" #include "common/scope_exit.h" @@ -1046,6 +1047,8 @@ struct Maxwell3D::FrontState { bool upload_tracked{}; GPUVAddr upload_begin{}; GPUVAddr upload_end{}; + /// Pages that report_semaphore writes went to. + std::unordered_set report_pages; }; namespace { @@ -1267,6 +1270,10 @@ void Maxwell3D::FrontNoteWrites(VideoCommon::GpuSplit& split, FrontState& fs, u3 const GPUVAddr begin = address(MAXWELL3D_REG_INDEX(report_semaphore.address_high), MAXWELL3D_REG_INDEX(report_semaphore.address_low)); note(begin, begin + 16); + if (fs.report_pages.size() < 4096) { + fs.report_pages.insert(begin >> 12); + fs.report_pages.insert((begin + 15) >> 12); + } } } @@ -1532,6 +1539,26 @@ void Maxwell3D::FrontDmaRead(VideoCommon::GpuSplit& split, FrontState& fs, u32 m }, VideoCommon::DrainSite::MacroDmaFlush); } + // A read of a page reports were sent to writes out the reports the query cache still holds, + // waiting for their host results (Lichii #215), or the guest reads stale occlusion counts. + const bool reads_reports = [&] { + if (!shadow_words.empty() || words == 0) { + return false; + } + for (GPUVAddr page = source >> 12; page <= (source + read_bytes - 1) >> 12; ++page) { + if (fs.report_pages.contains(page)) { + return true; + } + } + return false; + }(); + if (reads_reports) { + split.Run( + [this, source, read_bytes] { + VideoCommon::QueryDma::FlushPending(memory_manager, *rasterizer, source, read_bytes); + }, + VideoCommon::DrainSite::MacroDmaFlush); + } u32 changed{}; for (u32 i = 0; i < words; ++i) { const u32 word = !shadow_words.empty() diff --git a/src/video_core/macro.cpp b/src/video_core/macro.cpp index b6bca447a6..c1c29f4edc 100644 --- a/src/video_core/macro.cpp +++ b/src/video_core/macro.cpp @@ -1435,8 +1435,7 @@ void MacroEngine::Execute(Core::System& system, Engines::Maxwell3D& maxwell3d, u } } // Accurate GPU mode holds report and semaphore writes in the query cache until a - // fence: write them out, or occlusion predicates read stale values. The split only - // runs with immediate payload writes, and the query cache belongs to its back. + // fence: write them out, or occlusion predicates read stale values. const bool owns_caches = !VideoCommon::GpuSplit::Get().Active() || VideoCommon::GpuSplit::OnBackThread(); if (owns_caches && VideoCommon::QueryDma::TraceEnabled() && @@ -1445,8 +1444,9 @@ void MacroEngine::Execute(Core::System& system, Engines::Maxwell3D& maxwell3d, u VideoCommon::QueryDma::Trace(memory, maxwell3d.Rasterizer(), method, address, source, read_bytes); } - if (!VideoCommon::GpuSplit::Get().Active()) { - memory.FlushRegion(source, read_bytes, VideoCommon::CacheType::QueryCache); + if (owns_caches) { + VideoCommon::QueryDma::FlushPending(memory, maxwell3d.Rasterizer(), source, + read_bytes); } if (memory.IsMemoryDirty(source, read_bytes, VideoCommon::CacheType::BufferCache)) { memory.FlushRegion(source, read_bytes, VideoCommon::CacheType::BufferCache); diff --git a/src/video_core/query_cache/query_cache_base.h b/src/video_core/query_cache/query_cache_base.h index 5a60e42986..5adcdbcf96 100644 --- a/src/video_core/query_cache/query_cache_base.h +++ b/src/video_core/query_cache/query_cache_base.h @@ -63,14 +63,19 @@ class QueryCacheBase : public VideoCommon::ChannelSetupCaches(addr, size, [this, &result](QueryLocation location) { result |= SemiFlushQueryDirty(location); return result; }); - if (result) { - RequestGuestHostSync(); - } + return result; } static u64 BuildMask(std::span types) { diff --git a/src/video_core/query_dma.h b/src/video_core/query_dma.h index fdcc22341d..c236cca622 100644 --- a/src/video_core/query_dma.h +++ b/src/video_core/query_dma.h @@ -43,6 +43,21 @@ inline bool HasPendingQueries(const Tegra::MemoryManager& memory, return false; } +/// Writes the reports in [source, source + size) to guest memory, waiting for pending host +/// results. Runs where the rasterizer's caches are owned. +inline void FlushPending(const Tegra::MemoryManager& memory, + VideoCore::RasterizerInterface& rasterizer, GPUVAddr source, size_t size) { + constexpr GPUVAddr page_size = 0x1000; + const GPUVAddr end = source + size; + for (GPUVAddr begin = source; begin < end;) { + const GPUVAddr page_end = (std::min)((begin & ~(page_size - 1)) + page_size, end); + if (const auto device = memory.GpuToCpuAddress(begin)) { + rasterizer.FlushQueryResults(*device, page_end - begin); + } + begin = page_end; + } +} + /// Logs a read of pending reports with the words it reads, and the same words once the next /// fence has popped and the results have landed. inline void Trace(const Tegra::MemoryManager& memory, VideoCore::RasterizerInterface& rasterizer, diff --git a/src/video_core/rasterizer_interface.h b/src/video_core/rasterizer_interface.h index b6f790e98b..499cccee38 100644 --- a/src/video_core/rasterizer_interface.h +++ b/src/video_core/rasterizer_interface.h @@ -91,6 +91,11 @@ class RasterizerInterface { /// Release all pending fences. virtual void ReleaseFences(bool force = true) = 0; + /// Writes the region's query reports to guest memory, waiting for pending host results. + virtual void FlushQueryResults(DAddr addr, u64 size) { + FlushRegion(addr, size, VideoCommon::CacheType::QueryCache); + } + /// Notify rasterizer that all caches should be flushed to Switch memory virtual void FlushAll() = 0; diff --git a/src/video_core/renderer_vulkan/vk_rasterizer.cpp b/src/video_core/renderer_vulkan/vk_rasterizer.cpp index 8a903039cf..7a72aeda6c 100644 --- a/src/video_core/renderer_vulkan/vk_rasterizer.cpp +++ b/src/video_core/renderer_vulkan/vk_rasterizer.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -1783,6 +1784,27 @@ void RasterizerVulkan::ReleaseFences(bool force) { fence_manager.WaitPendingFences(force); } +void RasterizerVulkan::FlushQueryResults(DAddr addr, u64 size) { + if (addr == 0 || size == 0 || !query_cache.SemiFlushRegion(addr, size)) { + return; + } + // Below High GPU accuracy a fence runs its own callback at once, so wait for an operation + // queued behind the pending report writes instead. On the split's back the fence's + // submission may be deferred, and the fencing thread would wait for this thread to submit it. + std::mutex wait_mutex; + std::condition_variable wait_cv; + bool written{}; + fence_manager.SyncOperation([&] { + std::scoped_lock lock{wait_mutex}; + written = true; + wait_cv.notify_all(); + }); + fence_manager.SignalFence([] {}); + scheduler.FlushDeferred(); + std::unique_lock lock{wait_mutex}; + wait_cv.wait(lock, [&written] { return written; }); +} + void RasterizerVulkan::FlushAndInvalidateRegion(DAddr addr, u64 size, VideoCommon::CacheType which) { if (Settings::IsGPULevelHigh()) { diff --git a/src/video_core/renderer_vulkan/vk_rasterizer.h b/src/video_core/renderer_vulkan/vk_rasterizer.h index 00db6e401e..0207c8b81f 100644 --- a/src/video_core/renderer_vulkan/vk_rasterizer.h +++ b/src/video_core/renderer_vulkan/vk_rasterizer.h @@ -128,6 +128,7 @@ class RasterizerVulkan final : public VideoCore::RasterizerInterface, void SignalSyncPoint(u32 value) override; void SignalReference() override; void ReleaseFences(bool force = true) override; + void FlushQueryResults(DAddr addr, u64 size) override; void FlushAndInvalidateRegion( DAddr addr, u64 size, VideoCommon::CacheType which = VideoCommon::CacheType::All) override; void WaitForIdle() override; From ab9daa64f4e582f7af54e09aecabff897e674abe Mon Sep 17 00:00:00 2001 From: Celmax Date: Fri, 6 Nov 2026 21:43:51 +0100 Subject: [PATCH 04/16] video_core: don't serve constant buffer words written in Replay from the split front In ShadowRamControl::Replay the back writes the shadow RAM values, not the method arguments, so the front's cb_shadow copy of those words was wrong. Note such writes as opaque and keep them out of cb_shadow: macro DMA reads of them wait for the back. Covers single and multi method writes. Reported by CodeRabbit on #345. Co-Authored-By: Claude Opus 5.5 --- src/video_core/engines/maxwell_3d.cpp | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/video_core/engines/maxwell_3d.cpp b/src/video_core/engines/maxwell_3d.cpp index 25fecd3ea1..e6e7a5a7b6 100644 --- a/src/video_core/engines/maxwell_3d.cpp +++ b/src/video_core/engines/maxwell_3d.cpp @@ -1225,7 +1225,12 @@ void Maxwell3D::FrontNoteWrites(VideoCommon::GpuSplit& split, FrontState& fs, u3 return; } const GPUVAddr begin = base + offset_end - bytes; - split.NoteMemoryWrite(&memory_manager, begin, base + offset_end, false); + // In Replay the back writes the shadow RAM values, not these words: reads wait for it. + const bool replay = fs.control == Regs::ShadowRamControl::Replay; + split.NoteMemoryWrite(&memory_manager, begin, base + offset_end, replay); + if (replay) { + return; + } // The words themselves, for MME DMA reads of them while the back has not written them. if (fs.cb_shadow.size() >= 16384) { const u64 done = split.Consumed(); From f4434caca1a52b96bcefc25d6062f8a746da7e7b Mon Sep 17 00:00:00 2001 From: Celmax Date: Fri, 6 Nov 2026 21:56:42 +0100 Subject: [PATCH 05/16] dma_pusher: drop the CPU-written command page skip A command fetch from a page first fetched as a CPU-written list skipped the wait for queued compute and copy work. Nothing guarantees that such work can't write that page before the back runs it, so the front could parse stale words (Ocarina rewrites 28 such pages while loading). Command fetches wait for queued writers again, as on nx2. This costs ~2-3 ms per frame in Kokiri Forest. Reported by CodeRabbit on #345. Co-Authored-By: Claude Opus 5.5 --- src/video_core/dma_pusher.cpp | 43 ++++------------------------------- src/video_core/dma_pusher.h | 3 --- src/video_core/nx2.h | 14 ------------ 3 files changed, 4 insertions(+), 56 deletions(-) diff --git a/src/video_core/dma_pusher.cpp b/src/video_core/dma_pusher.cpp index 807c7c0c0c..9b2c49ff62 100644 --- a/src/video_core/dma_pusher.cpp +++ b/src/video_core/dma_pusher.cpp @@ -227,16 +227,10 @@ bool DmaPusher::Step() { // (the split only starts with unsafe reads; the settings can change mid-game). const bool use_safe = !VideoCommon::GpuSplit::Get().Active() && (Settings::IsDMALevelDefault() ? Settings::IsGPULevelHigh() : Settings::IsDMALevelSafe()); - if (!gpu_written && VideoCommon::GpuSplit::Get().Active()) { - const size_t bytes = size_t{header.size} * sizeof(u32); - if (!CpuListPages(dma_state.dma_get, bytes, false)) { - if (VideoCommon::GpuSplit::Get().WaitForWriters()) { - gpu_written = IsGpuWrittenCommands(dma_state.dma_get, bytes); - } - if (!gpu_written) { - CpuListPages(dma_state.dma_get, bytes, true); - } - } + if (!gpu_written && VideoCommon::GpuSplit::Get().Active() && + VideoCommon::GpuSplit::Get().WaitForWriters()) { + gpu_written = + IsGpuWrittenCommands(dma_state.dma_get, size_t{header.size} * sizeof(u32)); } // Resume and GPU-written entries need live words, even after a flush cleared dirty state. const bool submitted = (command_list.snapshot_suspensions & 1) == 0 && @@ -655,35 +649,6 @@ void DmaPusher::RefreshDeferredEntries(CommandList& list) { list.deferred_entries.clear(); } -bool DmaPusher::CpuListPages(GPUVAddr address, size_t size, bool mark) { - if (!Tegra::g_nx2_ampere.load(std::memory_order_relaxed) || size == 0) { - return false; - } - if (const u64 misses = Tegra::g_nx2_cpu_list_misses.load(std::memory_order_relaxed); - misses != reported_cpu_list_misses) { - reported_cpu_list_misses = misses; - LOG_WARNING(HW_GPU, "GPU wrote {} command list pages first fetched as CPU-written", misses); - } - for (GPUVAddr page = address & ~GPUVAddr{0xfff}; page < address + size; page += 0x1000) { - const auto device = memory_manager.GpuToCpuAddress(page); - if (!device) { - return false; - } - const u64 index = *device >> Tegra::Nx2WrittenPageBits; - if (index >= Tegra::g_nx2_cpu_list_pages.size() * 64) { - return false; - } - auto& word = Tegra::g_nx2_cpu_list_pages[index / 64]; - const u64 bit = u64{1} << (index % 64); - if (mark) { - word.fetch_or(bit, std::memory_order_relaxed); - } else if ((word.load(std::memory_order_relaxed) & bit) == 0) { - return false; - } - } - return true; -} - bool DmaPusher::IsGpuWrittenCommands(GPUVAddr address, size_t size) { if (!Tegra::g_nx2_ampere.load(std::memory_order_relaxed) || size == 0) { return false; diff --git a/src/video_core/dma_pusher.h b/src/video_core/dma_pusher.h index 4c622bd1c6..8c85eb3688 100644 --- a/src/video_core/dma_pusher.h +++ b/src/video_core/dma_pusher.h @@ -216,9 +216,6 @@ class DmaPusher final { void RecordParsedEntry(const CommandListHeader& header, std::span commands); /// Whether the entry lies on device pages the GPU has written (NX2). bool IsGpuWrittenCommands(GPUVAddr address, size_t size); - /// Whether every page of the range was fetched as a CPU-written command list (or marks them). - bool CpuListPages(GPUVAddr address, size_t size, bool mark); - u64 reported_cpu_list_misses{}; /// Copies the list's words from memory again, as of suspension count `now`. void RefreshSnapshot(CommandList& list, u64 now); /// Copies the list's deferred entries (CommandList::deferred_entries) from memory. diff --git a/src/video_core/nx2.h b/src/video_core/nx2.h index 381567919e..74f09eb3db 100644 --- a/src/video_core/nx2.h +++ b/src/video_core/nx2.h @@ -30,12 +30,6 @@ inline std::atomic g_nx2_ampere{false}; inline constexpr u64 Nx2WrittenPageBits = 12; inline std::array, (u64{1} << (36 - Nx2WrittenPageBits)) / 64> g_nx2_gpu_written_pages{}; -// Pages the front fetched command lists from after the compute and copy work before them ran, -// with no GPU write so far: later fetches from them skip that wait (DmaPusher::Step). A GPU write -// to one clears its bit and counts a miss. -inline std::array, (u64{1} << (36 - Nx2WrittenPageBits)) / 64> g_nx2_cpu_list_pages{}; -inline std::atomic g_nx2_cpu_list_misses{0}; - inline void Nx2MarkGpuWritten(u64 device_address, u64 size) { if (size == 0) { return; @@ -48,11 +42,6 @@ inline void Nx2MarkGpuWritten(u64 device_address, u64 size) { if ((word.load(std::memory_order_relaxed) & bit) == 0) { word.fetch_or(bit, std::memory_order_relaxed); } - auto& cpu_word = g_nx2_cpu_list_pages[page / 64]; - if ((cpu_word.load(std::memory_order_relaxed) & bit) != 0) { - cpu_word.fetch_and(~bit, std::memory_order_relaxed); - g_nx2_cpu_list_misses.fetch_add(1, std::memory_order_relaxed); - } } } @@ -92,9 +81,6 @@ inline void Nx2ClearGpuWrittenPages() { for (auto& word : g_nx2_command_list_pages) { word.store(0, std::memory_order_relaxed); } - for (auto& word : g_nx2_cpu_list_pages) { - word.store(0, std::memory_order_relaxed); - } g_nx2_has_command_list_pages.store(false, std::memory_order_relaxed); } From 0fbc02269916150236b883d02b36e1d0012b08eb Mon Sep 17 00:00:00 2001 From: Celmax Date: Fri, 6 Nov 2026 22:20:36 +0100 Subject: [PATCH 06/16] video_core: key split-front write tracking by backing memory, check GPU-written DMA sources From CodeRabbit's review of #345: - Queued writes and report pages are keyed by device address, so a write and a macro DMA read through different GPU addresses of the same memory meet. - Past 4096 report pages, every macro DMA read flushes pending query reports instead of missing the new pages. - A precise macro DMA read of pages the GPU has written lets the back check the buffer cache and download newer data first; waiting for the back alone left it in a host buffer. Co-Authored-By: Claude Opus 5.5 --- src/video_core/engines/maxwell_3d.cpp | 48 +++++++++++++++++++++---- src/video_core/gpu_split.cpp | 51 +++++++++++++++++++++++++++ src/video_core/gpu_split.h | 27 +++++--------- 3 files changed, 102 insertions(+), 24 deletions(-) diff --git a/src/video_core/engines/maxwell_3d.cpp b/src/video_core/engines/maxwell_3d.cpp index e6e7a5a7b6..7b6d3a345b 100644 --- a/src/video_core/engines/maxwell_3d.cpp +++ b/src/video_core/engines/maxwell_3d.cpp @@ -1047,8 +1047,9 @@ struct Maxwell3D::FrontState { bool upload_tracked{}; GPUVAddr upload_begin{}; GPUVAddr upload_end{}; - /// Pages that report_semaphore writes went to. - std::unordered_set report_pages; + /// Device pages that report_semaphore writes went to; past 4096 of them, every read flushes. + std::unordered_set report_pages; + bool report_pages_full{}; }; namespace { @@ -1275,9 +1276,13 @@ void Maxwell3D::FrontNoteWrites(VideoCommon::GpuSplit& split, FrontState& fs, u3 const GPUVAddr begin = address(MAXWELL3D_REG_INDEX(report_semaphore.address_high), MAXWELL3D_REG_INDEX(report_semaphore.address_low)); note(begin, begin + 16); - if (fs.report_pages.size() < 4096) { - fs.report_pages.insert(begin >> 12); - fs.report_pages.insert((begin + 15) >> 12); + for (const GPUVAddr page : {begin, begin + 15}) { + const auto device = memory_manager.GpuToCpuAddress(page); + if (!device || fs.report_pages.size() >= 4096) { + fs.report_pages_full = true; + break; + } + fs.report_pages.insert(*device >> 12); } } } @@ -1516,6 +1521,33 @@ void Maxwell3D::FrontDmaRead(VideoCommon::GpuSplit& split, FrontState& fs, u32 m } else { split.Drain(VideoCommon::DrainSite::MacroDmaRead); } + // GPU-written words may still be only in a host buffer: the back checks and downloads them. + if (precise && shadow_words.empty() && words != 0) { + bool gpu_written = false; + for (GPUVAddr page = source & ~GPUVAddr{0xfff}; page < source + read_bytes && !gpu_written; + page += 0x1000) { + const auto device = memory_manager.GpuToCpuAddress(page); + gpu_written = device && Tegra::Nx2IsGpuWritten(*device); + } + if (gpu_written) { + if (static const bool stats = std::getenv("EDEN_NX2_MME_DMA_STATS") != nullptr; stats) { + static u64 checks{}; + if ((++checks & 0x3ff) == 0) { + LOG_INFO(HW_GPU, "MME DMA reads of GPU-written pages checked on the back: {}", + checks); + } + } + split.Run( + [memory = &memory_manager, source, read_bytes] { + if (memory->IsMemoryDirty(source, read_bytes, + VideoCommon::CacheType::BufferCache)) { + memory->FlushRegion(source, read_bytes, + VideoCommon::CacheType::BufferCache); + } + }, + VideoCommon::DrainSite::MacroDmaFlush); + } + } if (!precise && VideoCommon::QueryDma::TraceEnabled()) { split.Run( [this, macro, method, source, read_bytes] { @@ -1550,8 +1582,12 @@ void Maxwell3D::FrontDmaRead(VideoCommon::GpuSplit& split, FrontState& fs, u32 m if (!shadow_words.empty() || words == 0) { return false; } + if (fs.report_pages_full) { + return true; + } for (GPUVAddr page = source >> 12; page <= (source + read_bytes - 1) >> 12; ++page) { - if (fs.report_pages.contains(page)) { + const auto device = memory_manager.GpuToCpuAddress(page << 12); + if (device && fs.report_pages.contains(*device >> 12)) { return true; } } diff --git a/src/video_core/gpu_split.cpp b/src/video_core/gpu_split.cpp index 535c4c8369..1b97a59eec 100644 --- a/src/video_core/gpu_split.cpp +++ b/src/video_core/gpu_split.cpp @@ -124,6 +124,57 @@ void GpuSplit::MaybeStart(Core::System& system_) { front_macros ? ", macros on the first" : ""); } +namespace { +constexpr u64 PageShift = 12; + +// The key of a 64-byte block: its device address, or its GPU address in `space` while unmapped. +template +void ForEachBlockKey(const void* space, u64 begin, u64 end, u64 block_shift, Func&& func) { + const auto* memory_manager = static_cast(space); + const u64 per_page = u64{1} << (PageShift - block_shift); + u64 block = begin >> block_shift; + const u64 last = (end - 1) >> block_shift; + while (block <= last) { + const u64 page = block >> (PageShift - block_shift); + const u64 page_last = std::min(last, (page + 1) * per_page - 1); + const auto device = memory_manager->GpuToCpuAddress(page << PageShift); + for (; block <= page_last; ++block) { + const u64 offset = block & (per_page - 1); + func(device ? ((*device >> block_shift) | offset) & ~(u64{1} << 63) + : (block ^ (static_cast(reinterpret_cast(space)) << 40)) | + (u64{1} << 63)); + } + } +} +} // Anonymous namespace + +void GpuSplit::NoteBlocks(std::unordered_map& blocks, const void* space, u64 begin, + u64 end) { + ForEachBlockKey(space, begin, end, WriteBlockShift, + [&](u64 key) { blocks[key] = write_position; }); +} + +u64 GpuSplit::BlocksPosition(const std::unordered_map& blocks, const void* space, + u64 begin, u64 end) const { + u64 position = 0; + ForEachBlockKey(space, begin, end, WriteBlockShift, [&](u64 key) { + if (const auto it = blocks.find(key); it != blocks.end()) { + position = std::max(position, it->second); + } + }); + return position; +} + +u64 GpuSplit::TrackedWritePosition(const void* space, u64 begin, u64 end) { + if (end <= begin || write_blocks.empty() || latest_block_write <= Consumed()) { + return 0; + } + if (((end - 1) >> WriteBlockShift) - (begin >> WriteBlockShift) >= MaxWriteBlocks) { + return latest_block_write; + } + return BlocksPosition(write_blocks, space, begin, end); +} + void GpuSplit::Stop() { if (!back_thread.joinable()) { return; diff --git a/src/video_core/gpu_split.h b/src/video_core/gpu_split.h index 7b8381fe56..6f1ab8d333 100644 --- a/src/video_core/gpu_split.h +++ b/src/video_core/gpu_split.h @@ -242,6 +242,8 @@ class GpuSplit { [[nodiscard]] u64 OpaqueReadyPosition(const void* space, u64 begin, u64 end) { return ReadyPosition(opaque_blocks, latest_opaque_write, space, begin, end); } + /// The position after the last queued tracked write to [begin, end), untracked writes aside. + [[nodiscard]] u64 TrackedWritePosition(const void* space, u64 begin, u64 end); private: friend struct GpuSplitTestAccess; @@ -319,36 +321,25 @@ class GpuSplit { // Queued tracked writes: the ring position after the last write to each 64-byte block. static constexpr u64 WriteBlockShift = 6; static constexpr u64 MaxWriteBlocks = 1024; - static u64 WriteBlockKey(const void* space, u64 block) { - // Aliasing only makes a read wait for an unrelated write. - return block ^ (static_cast(reinterpret_cast(space)) << 40); - } std::unordered_map write_blocks; std::unordered_map opaque_blocks; u64 latest_block_write{}; u64 latest_opaque_write{}; - void NoteBlocks(std::unordered_map& blocks, const void* space, u64 begin, u64 end) { - for (u64 block = begin >> WriteBlockShift; block <= (end - 1) >> WriteBlockShift; ++block) { - blocks[WriteBlockKey(space, block)] = write_position; - } - } + // Keyed by the backing memory (`space` is a Tegra::MemoryManager), so writes and reads + // through different GPU addresses of the same memory meet. + void NoteBlocks(std::unordered_map& blocks, const void* space, u64 begin, u64 end); + u64 BlocksPosition(const std::unordered_map& blocks, const void* space, u64 begin, + u64 end) const; u64 ReadyPosition(const std::unordered_map& blocks, u64 latest, const void* space, u64 begin, u64 end) { u64 target = MemoryBarrierPosition(); if (end <= begin || blocks.empty() || latest <= target) { return target; } - const u64 first = begin >> WriteBlockShift; - const u64 last = (end - 1) >> WriteBlockShift; - if (last - first >= MaxWriteBlocks) { + if (((end - 1) >> WriteBlockShift) - (begin >> WriteBlockShift) >= MaxWriteBlocks) { return std::max(target, latest); } - for (u64 block = first; block <= last; ++block) { - if (const auto it = blocks.find(WriteBlockKey(space, block)); it != blocks.end()) { - target = std::max(target, it->second); - } - } - return target; + return std::max(target, BlocksPosition(blocks, space, begin, end)); } u64 last_sync_position{}; u64 published_local{}; From e3fc3850a05c5bcd34ff36ab78a412031e327afe Mon Sep 17 00:00:00 2001 From: Celmax Date: Fri, 6 Nov 2026 22:20:36 +0100 Subject: [PATCH 07/16] puller: skip semaphore drains only when no queued or undownloaded write can change the value The greater-or-equal fast paths (SEM_ACQUIRE and SEM_EXECUTE) assumed releases only raise the value. A queued release can lower it, and GPU-written data may still be in the buffer cache, so they now also require that no tracked write to the semaphore is queued and that its page is not GPU-written. Reported by CodeRabbit on #345. Co-Authored-By: Claude Opus 5.5 --- src/video_core/engines/puller.cpp | 21 ++++++++++++++++----- src/video_core/engines/puller.h | 2 ++ 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/src/video_core/engines/puller.cpp b/src/video_core/engines/puller.cpp index 7d175fbf0a..61ada85f38 100644 --- a/src/video_core/engines/puller.cpp +++ b/src/video_core/engines/puller.cpp @@ -249,6 +249,16 @@ void Puller::ProcessFenceActionMethod(DmaPusher& dma_pusher) { } } +bool Puller::StableSemaphore(DmaPusher& dma_pusher, GPUVAddr address) { + auto& split = VideoCommon::GpuSplit::Get(); + if (split.TrackedWritePosition(&dma_pusher.memory_manager, address, address + 4) > + split.Consumed()) { + return false; + } + const auto device = dma_pusher.memory_manager.GpuToCpuAddress(address); + return device && !Nx2IsGpuWritten(*device); +} + void Puller::NoteRelease(GPUVAddr address, u32 payload) { auto& split = VideoCommon::GpuSplit::Get(); if (!split.Active()) { @@ -312,12 +322,13 @@ void Puller::ProcessSemaphoreTriggerMethod(DmaPusher& dma_pusher) { return; } VideoCommon::Nx2Profile::Add(VideoCommon::Nx2Profile::Counter::AcquireChecks, 1); - // A greater-or-equal acquire whose value is already in guest memory is met: releases only - // raise the value, and one still queued for the back (or held in its caches) is newer. - // Skip draining the back for it (~8% of the front in Ocarina of Time); later command + // A greater-or-equal acquire whose value is already in guest memory is met, unless a + // queued write (a release that lowers it) or GPU-written data not yet downloaded may change + // it. Skip draining the back for it (~8% of the front in Ocarina of Time); later command // fetches still wait for the work before it (NoteSync). auto& split = VideoCommon::GpuSplit::Get(); if (op == GpuSemaphoreOperation::AcquireGequal && split.Active() && + StableSemaphore(dma_pusher, acquire.address) && acquire.Matches(dma_pusher.memory_manager.Read(acquire.address))) { if (auto* stats = AcquireStats::Get()) stats->Note(AcquireStats::Fast); split.NoteSync(); @@ -518,8 +529,8 @@ void Puller::ProcessSemaphoreExecute(DmaPusher& dma_pusher, u32 argument) { } } } - if ((operation == 2 || operation == 3) && split.Active()) { - // A greater-or-equal value already in memory stays met: skip the back drain. + if ((operation == 2 || operation == 3) && split.Active() && StableSemaphore(dma_pusher, address)) { + // A greater-or-equal value already in memory with no queued write to it: skip the drain. const u32 current = dma_pusher.memory_manager.Read(address); if (operation == 2 ? current >= payload : static_cast(current - payload) >= 0) { split.NoteSync(); diff --git a/src/video_core/engines/puller.h b/src/video_core/engines/puller.h index 463b6eda73..d14f1a8b47 100644 --- a/src/video_core/engines/puller.h +++ b/src/video_core/engines/puller.h @@ -158,6 +158,8 @@ class Puller final { u64 position; }; std::unordered_map queued_releases; + /// Whether no queued write or undownloaded GPU write can change the semaphore at `address`. + bool StableSemaphore(DmaPusher& dma_pusher, GPUVAddr address); void NoteRelease(GPUVAddr address, u32 payload); /// Acquires at an address that queued GPU work met each time: how often in a row, or ~0U /// once the back found one unmet after skipping its wait. From 8a8d134efba4d7d00d3e29fce05268f5dedf16d0 Mon Sep 17 00:00:00 2001 From: Celmax Date: Fri, 6 Nov 2026 22:20:36 +0100 Subject: [PATCH 08/16] dma_pusher: recheck GPU-written command pages after every writer wait WaitForWriters returns false when the writers have already finished, which skipped the recheck: a writer that marked the pages after the first check left the fetch on an old submitted snapshot. Reported by CodeRabbit on #345. Co-Authored-By: Claude Opus 5.5 --- src/video_core/dma_pusher.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/video_core/dma_pusher.cpp b/src/video_core/dma_pusher.cpp index 9b2c49ff62..b8eeb3c3b2 100644 --- a/src/video_core/dma_pusher.cpp +++ b/src/video_core/dma_pusher.cpp @@ -227,8 +227,10 @@ bool DmaPusher::Step() { // (the split only starts with unsafe reads; the settings can change mid-game). const bool use_safe = !VideoCommon::GpuSplit::Get().Active() && (Settings::IsDMALevelDefault() ? Settings::IsGPULevelHigh() : Settings::IsDMALevelSafe()); - if (!gpu_written && VideoCommon::GpuSplit::Get().Active() && - VideoCommon::GpuSplit::Get().WaitForWriters()) { + // Recheck even when the wait returns at once: a writer may have finished (and marked the + // pages) since the check above. + if (!gpu_written && VideoCommon::GpuSplit::Get().Active()) { + VideoCommon::GpuSplit::Get().WaitForWriters(); gpu_written = IsGpuWrittenCommands(dma_state.dma_get, size_t{header.size} * sizeof(u32)); } From cc808c195f966b5995564e74170ed3adef908919 Mon Sep 17 00:00:00 2001 From: Celmax Date: Fri, 6 Nov 2026 23:02:33 +0100 Subject: [PATCH 09/16] video_core: make split-front acquires and macro DMA reads sound (review of #345) - Drop learned acquires: an acquire that queued work may meet now waits for it. Skipping it on past successes let the commands after an unmet acquire run. - StableSemaphore also waits for queued compute and copy work. - A channel's own queued release meets an acquire only if no later write to the word is queued, and not at High GPU accuracy (releases land at fence time). - Key the constant buffer shadow by device address, like the write tracking, and keep only the words ProcessCBMultiData writes within the buffer size. - Track report pages globally: the front marks the 3D reports it queues, the back marks every report left pending in the query cache (any engine or channel), instead of a per-channel list capped at 4096 pages. - Bound the FlushQueryResults wait to 2 s with state the callback shares. Co-Authored-By: Claude Opus 5.5 --- src/video_core/engines/maxwell_3d.cpp | 65 +++++++++---- src/video_core/engines/maxwell_3d.h | 3 + src/video_core/engines/puller.cpp | 96 +++++++------------ src/video_core/engines/puller.h | 9 +- src/video_core/nx2.h | 31 ++++++ .../renderer_vulkan/vk_rasterizer.cpp | 37 +++++-- 6 files changed, 146 insertions(+), 95 deletions(-) diff --git a/src/video_core/engines/maxwell_3d.cpp b/src/video_core/engines/maxwell_3d.cpp index 7b6d3a345b..5a958d9492 100644 --- a/src/video_core/engines/maxwell_3d.cpp +++ b/src/video_core/engines/maxwell_3d.cpp @@ -1037,19 +1037,18 @@ struct Maxwell3D::FrontState { bool draw_pending{true}; bool skip_same{}; ///< SkipSameValueWrites() && FrontSkipSame(), read once u32 dma_reads{}; - /// Constant buffer words sent inline: value and the ring position after the upload. + /// Constant buffer words sent inline, keyed by device address like the split's write + /// tracking (so GPU addresses aliasing one word meet): value and the ring position after + /// the upload. struct ShadowWord { u32 value; u64 position; }; - std::unordered_map cb_shadow; + std::unordered_map cb_shadow; /// The inline-to-memory upload being sent writes [upload_begin, upload_end). bool upload_tracked{}; GPUVAddr upload_begin{}; GPUVAddr upload_end{}; - /// Device pages that report_semaphore writes went to; past 4096 of them, every read flushes. - std::unordered_set report_pages; - bool report_pages_full{}; }; namespace { @@ -1204,6 +1203,25 @@ void Maxwell3D::FrontWrite(VideoCommon::GpuSplit& split, FrontState& fs, u32 met } } +template +bool Maxwell3D::DeviceWords(GPUVAddr begin, u32 count, Func&& func) { + GPUVAddr page = ~GPUVAddr{}; + std::optional page_device; + for (u32 i = 0; i < count; ++i) { + const GPUVAddr address = begin + u64{i} * sizeof(u32); + if ((address & ~GPUVAddr{0xfff}) != page) { + page = address & ~GPUVAddr{0xfff}; + page_device = memory_manager.GpuToCpuAddress(page); + } + const std::optional device = + page_device ? std::optional{*page_device + (address & 0xfff)} : std::nullopt; + if (!func(i, device)) { + return false; + } + } + return true; +} + void Maxwell3D::FrontNoteWrites(VideoCommon::GpuSplit& split, FrontState& fs, u32 method, u32 amount, const u32* values) { constexpr u32 cb_data = MAXWELL3D_REG_INDEX(const_buffer.buffer); @@ -1240,10 +1258,18 @@ void Maxwell3D::FrontNoteWrites(VideoCommon::GpuSplit& split, FrontState& fs, u3 fs.cb_shadow.clear(); } } + // Only the words ProcessCBMultiData writes: it drops those past the buffer's end. + const u64 offset = offset_end - bytes; + const u64 size = fs.regs[MAXWELL3D_REG_INDEX(const_buffer.size)]; + const u64 room = offset < size ? size - offset : 0; + const u32 count = static_cast(std::min(amount, room / sizeof(u32))); const u64 position = split.Position(); - for (u32 i = 0; i < amount; ++i) { - fs.cb_shadow[begin + u64{i} * sizeof(u32)] = {values[i], position}; - } + DeviceWords(begin, count, [&](u32 i, std::optional device) { + if (device) { + fs.cb_shadow[*device] = {values[i], position}; + } + return true; + }); return; } if (covers(MAXWELL3D_REG_INDEX(launch_dma))) { @@ -1277,12 +1303,9 @@ void Maxwell3D::FrontNoteWrites(VideoCommon::GpuSplit& split, FrontState& fs, u3 MAXWELL3D_REG_INDEX(report_semaphore.address_low)); note(begin, begin + 16); for (const GPUVAddr page : {begin, begin + 15}) { - const auto device = memory_manager.GpuToCpuAddress(page); - if (!device || fs.report_pages.size() >= 4096) { - fs.report_pages_full = true; - break; + if (const auto device = memory_manager.GpuToCpuAddress(page)) { + Tegra::Nx2MarkReportPages(*device, 1); } - fs.report_pages.insert(*device >> 12); } } } @@ -1481,15 +1504,18 @@ void Maxwell3D::FrontDmaRead(VideoCommon::GpuSplit& split, FrontState& fs, u32 m const u64 opaque = split.OpaqueReadyPosition(&memory_manager, source, source + read_bytes); shadow_words.clear(); - for (u32 i = 0; i < words; ++i) { - const auto it = fs.cb_shadow.find(source + u64{i} * sizeof(u32)); + const bool complete = DeviceWords(source, words, [&](u32, std::optional device) { + const auto it = device ? fs.cb_shadow.find(*device) : fs.cb_shadow.end(); // Only words still queued: memory has the others, maybe rewritten by the CPU. if (it == fs.cb_shadow.end() || it->second.position < opaque || it->second.position <= consumed) { - shadow_words.clear(); - break; + return false; } shadow_words.push_back(it->second.value); + return true; + }); + if (!complete) { + shadow_words.clear(); } } const bool from_shadow = !shadow_words.empty(); @@ -1582,12 +1608,9 @@ void Maxwell3D::FrontDmaRead(VideoCommon::GpuSplit& split, FrontState& fs, u32 m if (!shadow_words.empty() || words == 0) { return false; } - if (fs.report_pages_full) { - return true; - } for (GPUVAddr page = source >> 12; page <= (source + read_bytes - 1) >> 12; ++page) { const auto device = memory_manager.GpuToCpuAddress(page << 12); - if (device && fs.report_pages.contains(*device >> 12)) { + if (device && Tegra::Nx2IsReportPage(*device)) { return true; } } diff --git a/src/video_core/engines/maxwell_3d.h b/src/video_core/engines/maxwell_3d.h index 5df606151a..000067d351 100644 --- a/src/video_core/engines/maxwell_3d.h +++ b/src/video_core/engines/maxwell_3d.h @@ -3404,6 +3404,9 @@ class Maxwell3D final : public EngineInterface { u32 count); void FrontNoteWrites(VideoCommon::GpuSplit& split, FrontState& state, u32 method, u32 amount, const u32* values); + /// Calls func(index, device address) for `count` words from `begin`, while it returns true. + template + bool DeviceWords(GPUVAddr begin, u32 count, Func&& func); void FrontVerify(VideoCommon::GpuSplit& split, FrontState& state); void FinishFrontMacro(Core::System& system); std::unique_ptr front; diff --git a/src/video_core/engines/puller.cpp b/src/video_core/engines/puller.cpp index 61ada85f38..ef4111d65d 100644 --- a/src/video_core/engines/puller.cpp +++ b/src/video_core/engines/puller.cpp @@ -60,7 +60,7 @@ void OnBack(const Func& func) { // EDEN_NX2_MME_DMA_STATS: how semaphore acquires end and how long they hold the front. struct AcquireStats { - enum Kind : size_t { Fast, Own, Learned, AfterWait, AfterDrain, Suspended, Count }; + enum Kind : size_t { Fast, Own, AfterWait, AfterDrain, Suspended, Count }; std::array counts{}; u64 wait_ns{}; u64 drain_ns{}; @@ -80,9 +80,9 @@ struct AcquireStats { } last_log = now; LOG_INFO(HW_GPU, - "Acquires (5 s): fast {}, own release {}, learned {}, met after wait {} ({} ms), " + "Acquires (5 s): fast {}, own release {}, met after wait {} ({} ms), " "met after drain {} ({} ms), suspended {} ({} ms)", - counts[Fast], counts[Own], counts[Learned], counts[AfterWait], wait_ns / 1000000, + counts[Fast], counts[Own], counts[AfterWait], wait_ns / 1000000, counts[AfterDrain], drain_ns / 1000000, counts[Suspended], suspended_ns / 1000000); counts = {}; wait_ns = drain_ns = suspended_ns = 0; @@ -251,8 +251,9 @@ void Puller::ProcessFenceActionMethod(DmaPusher& dma_pusher) { bool Puller::StableSemaphore(DmaPusher& dma_pusher, GPUVAddr address) { auto& split = VideoCommon::GpuSplit::Get(); - if (split.TrackedWritePosition(&dma_pusher.memory_manager, address, address + 4) > - split.Consumed()) { + // Queued compute and copy work may write it too, untracked. + if (std::max(split.TrackedWritePosition(&dma_pusher.memory_manager, address, address + 4), + split.MemoryBarrierPosition()) > split.Consumed()) { return false; } const auto device = dma_pusher.memory_manager.GpuToCpuAddress(address); @@ -274,6 +275,24 @@ void Puller::NoteRelease(GPUVAddr address, u32 payload) { queued_releases[address] = {payload, split.Position()}; } +std::optional Puller::OwnQueuedRelease(DmaPusher& dma_pusher, GPUVAddr address) { + auto& split = VideoCommon::GpuSplit::Get(); + // At High GPU accuracy the release is written when its fence signals, after the back ran it. + if (!split.Active() || Settings::IsGPULevelHigh()) { + return std::nullopt; + } + const auto it = queued_releases.find(address); + if (it == queued_releases.end() || it->second.position <= split.Consumed()) { + return std::nullopt; + } + // Another write queued after it (a reset by compute, copy or another release) wins. + if (split.ReadReadyPosition(&dma_pusher.memory_manager, address, address + 4) > + it->second.position) { + return std::nullopt; + } + return it->second.payload; +} + void Puller::ProcessSemaphoreTriggerMethod(DmaPusher& dma_pusher) { const auto semaphoreOperationMask = 0xF; const auto op = GpuSemaphoreOperation(regs.semaphore_trigger & semaphoreOperationMask); @@ -334,16 +353,13 @@ void Puller::ProcessSemaphoreTriggerMethod(DmaPusher& dma_pusher) { split.NoteSync(); return; } - if (split.Active()) { - // This channel's own release, still queued, meets it once the back runs it, which is - // before anything sent after the acquire. - if (const auto it = queued_releases.find(acquire.address); - it != queued_releases.end() && it->second.position > split.Consumed() && - acquire.Matches(it->second.payload)) { - if (auto* stats = AcquireStats::Get()) stats->Note(AcquireStats::Own); - split.NoteSync(); - return; - } + // This channel's own release, still queued, meets it once the back runs it, which is + // before anything sent after the acquire. + if (const auto released = OwnQueuedRelease(dma_pusher, acquire.address); + released && acquire.Matches(*released)) { + if (auto* stats = AcquireStats::Get()) stats->Note(AcquireStats::Own); + split.NoteSync(); + return; } if (split.Active()) { // Only queued work that writes the semaphore (or untracked writes) can change it. @@ -371,46 +387,10 @@ void Puller::ProcessSemaphoreTriggerMethod(DmaPusher& dma_pusher) { causes[3], causes[4]); } } - // An address queued GPU work met 8 times in a row (compute completion, releases) is - // trusted to be met by the work before it; the back checks each skipped one and stops - // trusting the address if it was not. - // Any queued write the back makes before this point (compute, releases, reports). - const bool queued_writer = target > split.Consumed(); - if (has_unmet_acquires.exchange(false, std::memory_order_acquire)) { - std::scoped_lock lock{unmet_mutex}; - for (const GPUVAddr unmet : unmet_acquires) { - gpu_met_acquires[unmet] = ~0U; - LOG_WARNING(HW_GPU, "Acquire at {:#x} was not met by the GPU work before it", - unmet); - } - unmet_acquires.clear(); - } - if (queued_writer) { - const auto it = gpu_met_acquires.find(acquire.address); - if (it != gpu_met_acquires.end() && it->second >= 8 && it->second != ~0U) { - split.Defer([this, memory = &dma_pusher.memory_manager, acquire_check = acquire] { - if (!acquire_check.Matches(memory->Read(acquire_check.address))) { - std::scoped_lock lock{unmet_mutex}; - unmet_acquires.push_back(acquire_check.address); - has_unmet_acquires.store(true, std::memory_order_release); - } - }); - if (auto* stats = AcquireStats::Get()) stats->Note(AcquireStats::Learned); - split.NoteSync(); - return; - } - } const auto wait_start = std::chrono::steady_clock::now(); split.WaitForPosition(target, VideoCommon::DrainSite::Acquire); if (auto* stats = AcquireStats::Get()) stats->wait_ns += NsSince(wait_start); - const bool met = acquire.Matches(dma_pusher.memory_manager.Read(acquire.address)); - if (queued_writer) { - u32& count = gpu_met_acquires[acquire.address]; - if (count != ~0U) { - count = met ? count + 1 : 0; - } - } - if (met) { + if (acquire.Matches(dma_pusher.memory_manager.Read(acquire.address))) { if (auto* stats = AcquireStats::Get()) stats->Note(AcquireStats::AfterWait); split.NoteSync(); return; @@ -516,14 +496,12 @@ void Puller::ProcessSemaphoreExecute(DmaPusher& dma_pusher, u32 argument) { return; } auto& split = VideoCommon::GpuSplit::Get(); - if (operation <= 3 && operation != 1 && split.Active()) { + if (operation <= 3 && operation != 1) { // This channel's own queued release meets it before anything sent after the acquire. - if (const auto it = queued_releases.find(address); - it != queued_releases.end() && it->second.position > split.Consumed()) { - const u32 released = it->second.payload; - if (operation == 0 ? released == payload - : operation == 2 ? released >= payload - : static_cast(released - payload) >= 0) { + if (const auto released = OwnQueuedRelease(dma_pusher, address)) { + if (operation == 0 ? *released == payload + : operation == 2 ? *released >= payload + : static_cast(*released - payload) >= 0) { split.NoteSync(); return; } diff --git a/src/video_core/engines/puller.h b/src/video_core/engines/puller.h index d14f1a8b47..9caff39fb9 100644 --- a/src/video_core/engines/puller.h +++ b/src/video_core/engines/puller.h @@ -161,12 +161,9 @@ class Puller final { /// Whether no queued write or undownloaded GPU write can change the semaphore at `address`. bool StableSemaphore(DmaPusher& dma_pusher, GPUVAddr address); void NoteRelease(GPUVAddr address, u32 payload); - /// Acquires at an address that queued GPU work met each time: how often in a row, or ~0U - /// once the back found one unmet after skipping its wait. - std::unordered_map gpu_met_acquires; - std::mutex unmet_mutex; - std::vector unmet_acquires; - std::atomic has_unmet_acquires{false}; + /// The payload of this channel's queued release at `address`, if it is still queued and is + /// the last write to it before what is sent now. + std::optional OwnQueuedRelease(DmaPusher& dma_pusher, GPUVAddr address); /// Mapping of command subchannels to their bound engine ids std::array bound_engines{}; diff --git a/src/video_core/nx2.h b/src/video_core/nx2.h index 74f09eb3db..7979875d2e 100644 --- a/src/video_core/nx2.h +++ b/src/video_core/nx2.h @@ -73,6 +73,34 @@ inline void Nx2MarkCommandListPage(u64 device_address) { return (g_nx2_command_list_pages[page / 64].load(std::memory_order_relaxed) >> (page % 64)) & 1; } +// Device pages that reports were sent to (same layout as above): the front's MME DMA reads of +// them write out the reports the query cache still holds first. The front marks the 3D reports +// it queues; the back marks every report left pending, from any engine or channel. +inline std::array, (u64{1} << (36 - Nx2WrittenPageBits)) / 64> g_nx2_report_pages{}; + +inline void Nx2MarkReportPages(u64 device_address, u64 size) { + if (size == 0) { + return; + } + const u64 last = std::min((device_address + size - 1) >> Nx2WrittenPageBits, + g_nx2_report_pages.size() * 64 - 1); + for (u64 page = device_address >> Nx2WrittenPageBits; page <= last; ++page) { + auto& word = g_nx2_report_pages[page / 64]; + const u64 bit = u64{1} << (page % 64); + if ((word.load(std::memory_order_relaxed) & bit) == 0) { + word.fetch_or(bit, std::memory_order_relaxed); + } + } +} + +[[nodiscard]] inline bool Nx2IsReportPage(u64 device_address) { + const u64 page = device_address >> Nx2WrittenPageBits; + if (page >= g_nx2_report_pages.size() * 64) { + return false; + } + return (g_nx2_report_pages[page / 64].load(std::memory_order_relaxed) >> (page % 64)) & 1; +} + /// A new emulation session starts: no page has been written by its GPU yet. inline void Nx2ClearGpuWrittenPages() { for (auto& word : g_nx2_gpu_written_pages) { @@ -81,6 +109,9 @@ inline void Nx2ClearGpuWrittenPages() { for (auto& word : g_nx2_command_list_pages) { word.store(0, std::memory_order_relaxed); } + for (auto& word : g_nx2_report_pages) { + word.store(0, std::memory_order_relaxed); + } g_nx2_has_command_list_pages.store(false, std::memory_order_relaxed); } diff --git a/src/video_core/renderer_vulkan/vk_rasterizer.cpp b/src/video_core/renderer_vulkan/vk_rasterizer.cpp index 7a72aeda6c..1f916efabd 100644 --- a/src/video_core/renderer_vulkan/vk_rasterizer.cpp +++ b/src/video_core/renderer_vulkan/vk_rasterizer.cpp @@ -63,6 +63,7 @@ #include "video_core/vulkan_common/vulkan_device.h" #include "video_core/vulkan_common/vulkan_wrapper.h" #include "video_core/memory_manager.h" +#include "video_core/nx2.h" #include "video_core/nx2_debug_control.h" #include "video_core/nx2_profile.h" @@ -1512,6 +1513,13 @@ void RasterizerVulkan::ResetCounter(VideoCommon::QueryType type) { void RasterizerVulkan::Query(GPUVAddr gpu_addr, VideoCommon::QueryType type, VideoCommon::QueryPropertiesFlags flags, u32 payload, u32 subreport) { query_cache.CounterReport(gpu_addr, type, flags, payload, subreport); + // The split's front reads reports with MME DMA: it flushes the pages left pending here. + if (VideoCommon::GpuSplit::Get().Active()) { + if (const auto device = gpu_memory ? gpu_memory->GpuToCpuAddress(gpu_addr) : std::nullopt; + device && query_cache.IsRegionGpuModified(*device, 16)) { + Tegra::Nx2MarkReportPages(*device, 16); + } + } } void RasterizerVulkan::BindGraphicsUniformBuffer(size_t stage, u32 index, GPUVAddr gpu_addr, @@ -1791,18 +1799,29 @@ void RasterizerVulkan::FlushQueryResults(DAddr addr, u64 size) { // Below High GPU accuracy a fence runs its own callback at once, so wait for an operation // queued behind the pending report writes instead. On the split's back the fence's // submission may be deferred, and the fencing thread would wait for this thread to submit it. - std::mutex wait_mutex; - std::condition_variable wait_cv; - bool written{}; - fence_manager.SyncOperation([&] { - std::scoped_lock lock{wait_mutex}; - written = true; - wait_cv.notify_all(); + // Shared with the callback: it may run after a timed-out wait returned. + struct WaitState { + std::mutex mutex; + std::condition_variable cv; + bool written{}; + }; + const auto state = std::make_shared(); + fence_manager.SyncOperation([state] { + std::scoped_lock lock{state->mutex}; + state->written = true; + state->cv.notify_all(); }); fence_manager.SignalFence([] {}); scheduler.FlushDeferred(); - std::unique_lock lock{wait_mutex}; - wait_cv.wait(lock, [&written] { return written; }); + std::unique_lock lock{state->mutex}; + if (!state->cv.wait_for(lock, std::chrono::seconds{2}, [&state] { return state->written; })) { + static std::atomic warned{}; + if (!warned.exchange(true, std::memory_order_relaxed)) { + LOG_WARNING(Render_Vulkan, + "Query results at {:#x} not written after 2 s: reading them as they are", + addr); + } + } } void RasterizerVulkan::FlushAndInvalidateRegion(DAddr addr, u64 size, From 803011bf3c0a21b97fc60a6755cdb33d7d1d7e46 Mon Sep 17 00:00:00 2001 From: Jordan Date: Fri, 6 Nov 2026 22:22:26 +0000 Subject: [PATCH 10/16] video_core: rename shadowing locals from #345 so MSVC builds MSVC 14.51 treats C4456/C4457/C4458 as errors: the stats loop in Maxwell3D's macro DMA read reused 'source' and 'count', and Query's 'device' hid RasterizerVulkan::device. No behaviour change. Co-Authored-By: Claude Opus 5.5 --- src/video_core/engines/maxwell_3d.cpp | 5 +++-- src/video_core/renderer_vulkan/vk_rasterizer.cpp | 6 +++--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/video_core/engines/maxwell_3d.cpp b/src/video_core/engines/maxwell_3d.cpp index 9fcda4b426..28e2b3f1cc 100644 --- a/src/video_core/engines/maxwell_3d.cpp +++ b/src/video_core/engines/maxwell_3d.cpp @@ -1585,8 +1585,9 @@ void Maxwell3D::FrontDmaRead(VideoCommon::GpuSplit& split, FrontState& fs, u32 m if (((causes[0] + causes[1] + causes[2] + causes[3]) & 0xffff) == 0) { LOG_INFO(HW_GPU, "MME DMA reads: no wait {}, writer {}, barrier {}, constant buffer {}", causes[0], causes[1], causes[2], causes[3]); - for (const auto& [source, count] : sources) { - LOG_INFO(HW_GPU, "MME DMA read barrier source {:#x}: {}", source, count); + for (const auto& [barrier_source, barrier_count] : sources) { + LOG_INFO(HW_GPU, "MME DMA read barrier source {:#x}: {}", barrier_source, + barrier_count); } } } diff --git a/src/video_core/renderer_vulkan/vk_rasterizer.cpp b/src/video_core/renderer_vulkan/vk_rasterizer.cpp index e4b1d2fdda..21acb2effe 100644 --- a/src/video_core/renderer_vulkan/vk_rasterizer.cpp +++ b/src/video_core/renderer_vulkan/vk_rasterizer.cpp @@ -1519,9 +1519,9 @@ void RasterizerVulkan::Query(GPUVAddr gpu_addr, VideoCommon::QueryType type, query_cache.CounterReport(gpu_addr, type, flags, payload, subreport); // The split's front reads reports with MME DMA: it flushes the pages left pending here. if (VideoCommon::GpuSplit::Get().Active()) { - if (const auto device = gpu_memory ? gpu_memory->GpuToCpuAddress(gpu_addr) : std::nullopt; - device && query_cache.IsRegionGpuModified(*device, 16)) { - Tegra::Nx2MarkReportPages(*device, 16); + if (const auto report_address = gpu_memory ? gpu_memory->GpuToCpuAddress(gpu_addr) : std::nullopt; + report_address && query_cache.IsRegionGpuModified(*report_address, 16)) { + Tegra::Nx2MarkReportPages(*report_address, 16); } } } From d6612978729d8e29c05aa02a91f4381deccccf5c Mon Sep 17 00:00:00 2001 From: Celmax Date: Sat, 7 Nov 2026 09:27:28 +0100 Subject: [PATCH 11/16] video_core: flush before macro DMA reads only while their reports are pending Report pages stayed marked for the whole session, so nearly every read of them drained the back to flush the query cache: madebyjordan counted only 540 of 2.08 million such reads with a pending result in the words read. The query cache now counts, per 16-byte slot hash, the reports it registers and the results it writes out, as in Lichii #243 by tibzejoker. The front flushes only when the counts differ for the words read. The wait before it has already run the queued reports the read may see, so the back has registered them. The page bitmap is gone. EDEN_NX2_MME_DMA_STATS also reports how many reads flushed. Co-Authored-By: Claude Opus 5.5 --- src/video_core/engines/maxwell_3d.cpp | 25 ++++++---- src/video_core/nx2.h | 47 ++++++++++--------- src/video_core/query_cache/query_cache.h | 19 ++++++++ .../renderer_vulkan/vk_rasterizer.cpp | 7 --- 4 files changed, 58 insertions(+), 40 deletions(-) diff --git a/src/video_core/engines/maxwell_3d.cpp b/src/video_core/engines/maxwell_3d.cpp index 28e2b3f1cc..9cc30080d3 100644 --- a/src/video_core/engines/maxwell_3d.cpp +++ b/src/video_core/engines/maxwell_3d.cpp @@ -1310,11 +1310,6 @@ void Maxwell3D::FrontNoteWrites(VideoCommon::GpuSplit& split, FrontState& fs, u3 const GPUVAddr begin = address(MAXWELL3D_REG_INDEX(report_semaphore.address_high), MAXWELL3D_REG_INDEX(report_semaphore.address_low)); note(begin, begin + 16); - for (const GPUVAddr page : {begin, begin + 15}) { - if (const auto device = memory_manager.GpuToCpuAddress(page)) { - Tegra::Nx2MarkReportPages(*device, 1); - } - } } } @@ -1652,20 +1647,30 @@ void Maxwell3D::FrontDmaRead(VideoCommon::GpuSplit& split, FrontState& fs, u32 m }, VideoCommon::DrainSite::MacroDmaFlush); } - // A read of a page reports were sent to writes out the reports the query cache still holds, - // waiting for their host results (Lichii #215), or the guest reads stale occlusion counts. + // A read of reports the query cache still holds writes them out first, waiting for their + // host results (Lichii #215), or the guest reads stale occlusion counts. The wait above ran + // the queued reports the read may see, so the back has registered them. const bool reads_reports = [&] { if (!shadow_words.empty() || words == 0) { return false; } - for (GPUVAddr page = source >> 12; page <= (source + read_bytes - 1) >> 12; ++page) { - const auto device = memory_manager.GpuToCpuAddress(page << 12); - if (device && Tegra::Nx2IsReportPage(*device)) { + for (GPUVAddr begin = source; begin < source + read_bytes;) { + const GPUVAddr end = std::min((begin | 0xfff) + 1, source + read_bytes); + const auto device = memory_manager.GpuToCpuAddress(begin); + if (device && Tegra::Nx2ReportsPending(*device, end - begin)) { return true; } + begin = end; } return false; }(); + if (static const bool stats = std::getenv("EDEN_NX2_MME_DMA_STATS") != nullptr; stats) { + static u64 reads{}, flushes{}; + flushes += reads_reports ? 1 : 0; + if ((++reads & 0xffff) == 0) { + LOG_INFO(HW_GPU, "MME DMA reads: {} of {} flushed pending reports", flushes, reads); + } + } if (reads_reports) { split.Run( [this, source, read_bytes] { diff --git a/src/video_core/nx2.h b/src/video_core/nx2.h index fb22a10b12..6fdceb7c1d 100644 --- a/src/video_core/nx2.h +++ b/src/video_core/nx2.h @@ -73,32 +73,30 @@ inline void Nx2MarkCommandListPage(u64 device_address) { return (g_nx2_command_list_pages[page / 64].load(std::memory_order_relaxed) >> (page % 64)) & 1; } -// Device pages that reports were sent to (same layout as above): the front's MME DMA reads of -// them write out the reports the query cache still holds first. The front marks the 3D reports -// it queues; the back marks every report left pending, from any engine or channel. -inline std::array, (u64{1} << (36 - Nx2WrittenPageBits)) / 64> g_nx2_report_pages{}; +// Reports the query cache holds until their host results are written (Lichii #243): per +// 16-byte slot hash, how many the back registered and how many it wrote out. Equal counts mean +// the words in memory are final, so the split front's MME DMA reads of them need no flush. +inline constexpr u64 Nx2PendingReportSlots = u64{1} << 16; +inline std::array, Nx2PendingReportSlots> g_nx2_reports_issued{}; +inline std::array, Nx2PendingReportSlots> g_nx2_reports_written{}; + +[[nodiscard]] inline u64 Nx2PendingReportSlot(u64 device_address) { + return (device_address >> 4) % Nx2PendingReportSlots; +} -inline void Nx2MarkReportPages(u64 device_address, u64 size) { +[[nodiscard]] inline bool Nx2ReportsPending(u64 device_address, u64 size) { if (size == 0) { - return; + return false; } - const u64 last = std::min((device_address + size - 1) >> Nx2WrittenPageBits, - g_nx2_report_pages.size() * 64 - 1); - for (u64 page = device_address >> Nx2WrittenPageBits; page <= last; ++page) { - auto& word = g_nx2_report_pages[page / 64]; - const u64 bit = u64{1} << (page % 64); - if ((word.load(std::memory_order_relaxed) & bit) == 0) { - word.fetch_or(bit, std::memory_order_relaxed); + const u64 last = (device_address + size - 1) >> 4; + for (u64 slot = device_address >> 4; slot <= last; ++slot) { + const u64 index = slot % Nx2PendingReportSlots; + if (g_nx2_reports_written[index].load(std::memory_order_acquire) != + g_nx2_reports_issued[index].load(std::memory_order_acquire)) { + return true; } } -} - -[[nodiscard]] inline bool Nx2IsReportPage(u64 device_address) { - const u64 page = device_address >> Nx2WrittenPageBits; - if (page >= g_nx2_report_pages.size() * 64) { - return false; - } - return (g_nx2_report_pages[page / 64].load(std::memory_order_relaxed) >> (page % 64)) & 1; + return false; } /// A new emulation session starts: no page has been written by its GPU yet. @@ -109,8 +107,11 @@ inline void Nx2ClearGpuWrittenPages() { for (auto& word : g_nx2_command_list_pages) { word.store(0, std::memory_order_relaxed); } - for (auto& word : g_nx2_report_pages) { - word.store(0, std::memory_order_relaxed); + for (auto& count : g_nx2_reports_issued) { + count.store(0, std::memory_order_relaxed); + } + for (auto& count : g_nx2_reports_written) { + count.store(0, std::memory_order_relaxed); } g_nx2_has_command_list_pages.store(false, std::memory_order_relaxed); } diff --git a/src/video_core/query_cache/query_cache.h b/src/video_core/query_cache/query_cache.h index 1cd92dcfe2..ae927936aa 100644 --- a/src/video_core/query_cache/query_cache.h +++ b/src/video_core/query_cache/query_cache.h @@ -286,6 +286,17 @@ void QueryCacheBase::CounterReport(GPUVAddr addr, QueryType counter_type std::function operation([this, is_synced, streamer, query_base = query, query_location, pointer, pointer_timestamp, cpu_addr, addr, asid = gpu_memory->GetID(), counter_type] { + // Counted on every return: the front's DMA reads wait while the counts differ. + struct Written { + bool registered; + DAddr address; + ~Written() { + if (registered) { + Tegra::g_nx2_reports_written[Tegra::Nx2PendingReportSlot(address)].fetch_add( + 1, std::memory_order_release); + } + } + } const written{!is_synced, cpu_addr}; if (True(query_base->flags & QueryFlagBits::IsInvalidated)) { if (const u64 trace = QueryTrace::Sample(QueryTrace::Event::Discard)) { LOG_INFO(HW_GPU, @@ -338,7 +349,14 @@ void QueryCacheBase::CounterReport(GPUVAddr addr, QueryType counter_type impl->pending_unregister.push_back(query_location); } }); + const auto issue = [&] { + if (!is_synced) { + Tegra::g_nx2_reports_issued[Tegra::Nx2PendingReportSlot(cpu_addr)].fetch_add( + 1, std::memory_order_release); + } + }; if (is_fence) { + issue(); impl->rasterizer.SignalFence(std::move(operation)); } else { if (!Settings::IsGPULevelHigh() && counter_type == QueryType::Payload) { @@ -362,6 +380,7 @@ void QueryCacheBase::CounterReport(GPUVAddr addr, QueryType counter_type streamer->Free(new_query_id); return; } + issue(); impl->rasterizer.SyncOperation(std::move(operation)); } if (is_synced) { diff --git a/src/video_core/renderer_vulkan/vk_rasterizer.cpp b/src/video_core/renderer_vulkan/vk_rasterizer.cpp index 21acb2effe..32d88ad249 100644 --- a/src/video_core/renderer_vulkan/vk_rasterizer.cpp +++ b/src/video_core/renderer_vulkan/vk_rasterizer.cpp @@ -1517,13 +1517,6 @@ void RasterizerVulkan::ResetCounter(VideoCommon::QueryType type) { void RasterizerVulkan::Query(GPUVAddr gpu_addr, VideoCommon::QueryType type, VideoCommon::QueryPropertiesFlags flags, u32 payload, u32 subreport) { query_cache.CounterReport(gpu_addr, type, flags, payload, subreport); - // The split's front reads reports with MME DMA: it flushes the pages left pending here. - if (VideoCommon::GpuSplit::Get().Active()) { - if (const auto report_address = gpu_memory ? gpu_memory->GpuToCpuAddress(gpu_addr) : std::nullopt; - report_address && query_cache.IsRegionGpuModified(*report_address, 16)) { - Tegra::Nx2MarkReportPages(*report_address, 16); - } - } } void RasterizerVulkan::BindGraphicsUniformBuffer(size_t stage, u32 index, GPUVAddr gpu_addr, From d9d5250c3289dee3f27f92676729bcfa7bbe70d5 Mon Sep 17 00:00:00 2001 From: Celmax Date: Sat, 7 Nov 2026 09:27:28 +0100 Subject: [PATCH 12/16] video_core: share the split front's constant buffer words across channels Each channel kept its own copy, so a word uploaded by one channel and then by another could be served with the first value. The copy now lives in GpuSplit, keyed by device address, and the last upload of a word wins whichever channel sent it. Reported by CodeRabbit. Co-Authored-By: Claude Opus 5.5 --- src/video_core/engines/maxwell_3d.cpp | 24 +++++++++--------------- src/video_core/gpu_split.cpp | 1 + src/video_core/gpu_split.h | 12 ++++++++++++ 3 files changed, 22 insertions(+), 15 deletions(-) diff --git a/src/video_core/engines/maxwell_3d.cpp b/src/video_core/engines/maxwell_3d.cpp index 9cc30080d3..51ec7b19a9 100644 --- a/src/video_core/engines/maxwell_3d.cpp +++ b/src/video_core/engines/maxwell_3d.cpp @@ -1037,14 +1037,6 @@ struct Maxwell3D::FrontState { bool draw_pending{true}; bool skip_same{}; ///< SkipSameValueWrites() && FrontSkipSame(), read once u32 dma_reads{}; - /// Constant buffer words sent inline, keyed by device address like the split's write - /// tracking (so GPU addresses aliasing one word meet): value and the ring position after - /// the upload. - struct ShadowWord { - u32 value; - u64 position; - }; - std::unordered_map cb_shadow; /// The inline-to-memory upload being sent writes [upload_begin, upload_end). bool upload_tracked{}; GPUVAddr upload_begin{}; @@ -1259,11 +1251,12 @@ void Maxwell3D::FrontNoteWrites(VideoCommon::GpuSplit& split, FrontState& fs, u3 return; } // The words themselves, for MME DMA reads of them while the back has not written them. - if (fs.cb_shadow.size() >= 16384) { + auto& front_words = split.FrontWords(); + if (front_words.size() >= 16384) { const u64 done = split.Consumed(); - std::erase_if(fs.cb_shadow, [done](const auto& entry) { return entry.second.position <= done; }); - if (fs.cb_shadow.size() >= 16384) { - fs.cb_shadow.clear(); + std::erase_if(front_words, [done](const auto& entry) { return entry.second.position <= done; }); + if (front_words.size() >= 16384) { + front_words.clear(); } } // Only the words ProcessCBMultiData writes: it drops those past the buffer's end. @@ -1274,7 +1267,7 @@ void Maxwell3D::FrontNoteWrites(VideoCommon::GpuSplit& split, FrontState& fs, u3 const u64 position = split.Position(); DeviceWords(begin, count, [&](u32 i, std::optional device) { if (device) { - fs.cb_shadow[*device] = {values[i], position}; + front_words[*device] = {values[i], position}; } return true; }); @@ -1549,9 +1542,10 @@ void Maxwell3D::FrontDmaRead(VideoCommon::GpuSplit& split, FrontState& fs, u32 m split.OpaqueReadyPosition(&memory_manager, source, source + read_bytes); shadow_words.clear(); const bool complete = DeviceWords(source, words, [&](u32, std::optional device) { - const auto it = device ? fs.cb_shadow.find(*device) : fs.cb_shadow.end(); + const auto& front_words = split.FrontWords(); + const auto it = device ? front_words.find(*device) : front_words.end(); // Only words still queued: memory has the others, maybe rewritten by the CPU. - if (it == fs.cb_shadow.end() || it->second.position < opaque || + if (it == front_words.end() || it->second.position < opaque || it->second.position <= consumed) { return false; } diff --git a/src/video_core/gpu_split.cpp b/src/video_core/gpu_split.cpp index 2ef72c80ad..7f34755a69 100644 --- a/src/video_core/gpu_split.cpp +++ b/src/video_core/gpu_split.cpp @@ -194,6 +194,7 @@ void GpuSplit::Stop() { last_barrier_position = 0; write_blocks.clear(); opaque_blocks.clear(); + front_words.clear(); latest_block_write = 0; latest_opaque_write = 0; last_sync_position = 0; diff --git a/src/video_core/gpu_split.h b/src/video_core/gpu_split.h index 6f1ab8d333..48774b0e21 100644 --- a/src/video_core/gpu_split.h +++ b/src/video_core/gpu_split.h @@ -245,6 +245,17 @@ class GpuSplit { /// The position after the last queued tracked write to [begin, end), untracked writes aside. [[nodiscard]] u64 TrackedWritePosition(const void* space, u64 begin, u64 end); + /// Front only: a constant buffer word sent inline and the ring position after its upload. + struct FrontWord { + u32 value; + u64 position; + }; + /// Front only: the words of every channel, keyed by device address, so the last upload of a + /// word wins whichever channel sent it. + [[nodiscard]] std::unordered_map& FrontWords() { + return front_words; + } + private: friend struct GpuSplitTestAccess; @@ -323,6 +334,7 @@ class GpuSplit { static constexpr u64 MaxWriteBlocks = 1024; std::unordered_map write_blocks; std::unordered_map opaque_blocks; + std::unordered_map front_words; u64 latest_block_write{}; u64 latest_opaque_write{}; // Keyed by the backing memory (`space` is a Tegra::MemoryManager), so writes and reads From c2c3fdbbd774dc79ea8555d7bbce4b93842f62df Mon Sep 17 00:00:00 2001 From: Celmax Date: Sat, 7 Nov 2026 11:00:30 +0100 Subject: [PATCH 13/16] video_core: keep FrontTrackInstanceLoop out of line for GCC 13 GCC 13.3 at -O3 unswitches the per-word loop of FrontCallMultiMethod on the switch that #178 inlined into it. The loop that the other methods take then has no case for the macro upload methods: load_mme.instruction never reaches AddCode, the front's macro RAM stays empty, and every Turing macro fails with "instruction outside uploaded RAM" (black screen at boot in Ocarina of Time). In the object file the function has no AddCode or ClearCode call left; with the helper kept out of line both are back. MSVC and clang builds are not affected. Co-Authored-By: Claude Opus 5.5 --- src/video_core/engines/maxwell_3d.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/video_core/engines/maxwell_3d.cpp b/src/video_core/engines/maxwell_3d.cpp index 29a9ceab3d..efec97aca2 100644 --- a/src/video_core/engines/maxwell_3d.cpp +++ b/src/video_core/engines/maxwell_3d.cpp @@ -1375,7 +1375,9 @@ void Maxwell3D::FrontRunMacro(Core::System& system, VideoCommon::GpuSplit& split } } -void Maxwell3D::FrontTrackInstanceLoop(FrontState& fs, u32 method, u32 value) { +// Not inlined: GCC 13 -O3 unswitches FrontCallMultiMethod's loop on this switch and drops the +// macro upload cases from the loop the other methods take (an empty macro RAM on the front). +[[gnu::noinline]] void Maxwell3D::FrontTrackInstanceLoop(FrontState& fs, u32 method, u32 value) { switch (method) { case MAXWELL3D_REG_INDEX(draw.begin): // Instance id, bits 27:26: First leaves an instance loop, Subsequent enters one. From 6e79fe68214bad425b3d60eba4090387337231de Mon Sep 17 00:00:00 2001 From: Celmax Date: Sat, 7 Nov 2026 12:29:15 +0100 Subject: [PATCH 14/16] video_core: order writing draws, restarted splits and GPU-written semaphores on the split front (review of #345) - A draw that writes a storage or texture buffer makes the front count the draws before the next wait for idle as writers, so macro DMA reads after it wait for them. Nothing changes while no draw writes memory. - Each split stop starts a new epoch; a channel's queued releases from an earlier one are dropped instead of meeting acquires. - An acquire met after its wait is accepted only if its page is not GPU-written; otherwise the drain path flushes the word first. Co-Authored-By: Claude Opus 5.5 --- src/video_core/buffer_cache/buffer_cache.h | 12 +++++++++ .../buffer_cache/buffer_cache_base.h | 2 ++ src/video_core/engines/maxwell_3d.cpp | 27 +++++++++++++++++++ src/video_core/engines/puller.cpp | 14 +++++++++- src/video_core/engines/puller.h | 3 +++ src/video_core/gpu_split.cpp | 2 ++ src/video_core/gpu_split.h | 22 +++++++++++++++ src/video_core/nx2.h | 4 +++ 8 files changed, 85 insertions(+), 1 deletion(-) diff --git a/src/video_core/buffer_cache/buffer_cache.h b/src/video_core/buffer_cache/buffer_cache.h index c5b31a09c4..f45bd0a501 100644 --- a/src/video_core/buffer_cache/buffer_cache.h +++ b/src/video_core/buffer_cache/buffer_cache.h @@ -1609,12 +1609,23 @@ bool BufferCache

::BindMultiRangeStorage(const Binding& binding, bool is_writt } } +template +void BufferCache

::NoteDrawWritesMemory() { + if (!Tegra::g_nx2_draws_write_memory.exchange(true, std::memory_order_relaxed)) { + LOG_INFO(HW_GPU, "A draw writes a storage or texture buffer: the split front now orders " + "draws before macro DMA reads at waits for idle"); + } +} + template void BufferCache

::BindHostGraphicsStorageBuffers(size_t stage) { u32 binding_index = 0; ForEachEnabledBit(channel_state->enabled_storage_buffers[stage], [&](u32 index) { const Binding& binding = channel_state->storage_buffers[stage][index]; const bool is_written = ((channel_state->written_storage_buffers[stage] >> index) & 1) != 0; + if (is_written) { + NoteDrawWritesMemory(); + } if (BindMultiRangeStorage(binding, is_written, graphics_segments)) { return; } @@ -1649,6 +1660,7 @@ void BufferCache

::BindHostGraphicsTextureBuffers(size_t stage) { const bool is_written = ((channel_state->written_texture_buffers[stage] >> index) & 1) != 0; if (is_written) { + NoteDrawWritesMemory(); MarkWrittenBuffer(binding.buffer_id, binding.device_addr, size); } diff --git a/src/video_core/buffer_cache/buffer_cache_base.h b/src/video_core/buffer_cache/buffer_cache_base.h index 56b0f3546b..708a14a65e 100644 --- a/src/video_core/buffer_cache/buffer_cache_base.h +++ b/src/video_core/buffer_cache/buffer_cache_base.h @@ -515,6 +515,8 @@ class BufferCache : public VideoCommon::ChannelSetupCaches= 1024) { const u64 done = split.Consumed(); std::erase_if(queued_releases, [done](const auto& entry) { return entry.second.position <= done; }); @@ -281,6 +289,7 @@ std::optional Puller::OwnQueuedRelease(DmaPusher& dma_pusher, GPUVAddr addr if (!split.Active() || Settings::IsGPULevelHigh()) { return std::nullopt; } + CheckReleaseEpoch(); const auto it = queued_releases.find(address); if (it == queued_releases.end() || it->second.position <= split.Consumed()) { return std::nullopt; @@ -390,7 +399,10 @@ void Puller::ProcessSemaphoreTriggerMethod(DmaPusher& dma_pusher) { const auto wait_start = std::chrono::steady_clock::now(); split.WaitForPosition(target, VideoCommon::DrainSite::Acquire); if (auto* stats = AcquireStats::Get()) stats->wait_ns += NsSince(wait_start); - if (acquire.Matches(dma_pusher.memory_manager.Read(acquire.address))) { + // A GPU-written word may be newer in a host buffer: the drain below flushes it. + const auto device = dma_pusher.memory_manager.GpuToCpuAddress(acquire.address); + if (device && !Nx2IsGpuWritten(*device) && + acquire.Matches(dma_pusher.memory_manager.Read(acquire.address))) { if (auto* stats = AcquireStats::Get()) stats->Note(AcquireStats::AfterWait); split.NoteSync(); return; diff --git a/src/video_core/engines/puller.h b/src/video_core/engines/puller.h index 9caff39fb9..8dba782f02 100644 --- a/src/video_core/engines/puller.h +++ b/src/video_core/engines/puller.h @@ -158,6 +158,9 @@ class Puller final { u64 position; }; std::unordered_map queued_releases; + /// GpuSplit::Epoch when queued_releases was filled: a restarted split voids its positions. + u64 queued_releases_epoch{}; + void CheckReleaseEpoch(); /// Whether no queued write or undownloaded GPU write can change the semaphore at `address`. bool StableSemaphore(DmaPusher& dma_pusher, GPUVAddr address); void NoteRelease(GPUVAddr address, u32 payload); diff --git a/src/video_core/gpu_split.cpp b/src/video_core/gpu_split.cpp index 7f34755a69..7aef1d5ca0 100644 --- a/src/video_core/gpu_split.cpp +++ b/src/video_core/gpu_split.cpp @@ -192,6 +192,8 @@ void GpuSplit::Stop() { write_position = 0; last_writer_position = 0; last_barrier_position = 0; + draws_queued = false; + ++epoch; write_blocks.clear(); opaque_blocks.clear(); front_words.clear(); diff --git a/src/video_core/gpu_split.h b/src/video_core/gpu_split.h index 48774b0e21..23e59c8049 100644 --- a/src/video_core/gpu_split.h +++ b/src/video_core/gpu_split.h @@ -15,6 +15,7 @@ #include #include "common/common_types.h" +#include "video_core/nx2.h" namespace Core { class System; @@ -173,6 +174,25 @@ class GpuSplit { /// commands after it are fetched. void NoteSync() { last_sync_position = write_position; + NoteIdle(); + } + /// A draw was just sent. Draws write memory the front does not track (shader stores). + void NoteDraw() { + if (Tegra::g_nx2_draws_write_memory.load(std::memory_order_relaxed)) { + draws_queued = true; + } + } + /// A wait for idle was just sent: what the draws before it wrote is visible after it, as + /// the writes of compute work are. + void NoteIdle() { + if (draws_queued) { + draws_queued = false; + last_writer_position = write_position; + } + } + /// Increments each time the split stops: ring positions recorded before then are void. + [[nodiscard]] u64 Epoch() const { + return epoch; } /// Waits until the back executed the work sent before the last NoteSync. Returns whether any /// of it was still queued. @@ -328,6 +348,8 @@ class GpuSplit { u64 write_position{}; u64 last_writer_position{}; u64 last_barrier_position{}; + bool draws_queued{}; + u64 epoch{}; u32 last_barrier_source{}; // Queued tracked writes: the ring position after the last write to each 64-byte block. static constexpr u64 WriteBlockShift = 6; diff --git a/src/video_core/nx2.h b/src/video_core/nx2.h index 6fdceb7c1d..59df8dff23 100644 --- a/src/video_core/nx2.h +++ b/src/video_core/nx2.h @@ -22,6 +22,10 @@ namespace Tegra { // Ampere classes and the Turing+ macro ISA, which Eden's Maxwell engines don't implement. inline std::atomic g_nx2_ampere{false}; +// Set once a draw wrote a storage or texture buffer: from then on the split's front counts +// draws as memory writers at the next wait for idle (macro DMA reads of what draws wrote). +inline std::atomic g_nx2_draws_write_memory{false}; + // Device pages the GPU has written through the buffer cache (shader stores, copies), ever. Title 1 // runs command lists the GPU writes each frame (20-byte entries its culling fills in); read // without a flush they are stale and the draws behind them (trees) go missing. The bits never From 6775229b593f81682273f74ae798a7ffd4170b7c Mon Sep 17 00:00:00 2001 From: Celmax Date: Sat, 7 Nov 2026 15:38:51 +0100 Subject: [PATCH 15/16] video_core: count every draw as a writer at the next wait for idle on the split front The flag that enabled this was set by the back when it bound a written storage or texture buffer, so the front could pass the wait for idle after the first writing draw before the back had seen it, and a following macro DMA read could miss its writes (CodeRabbit on #345). Track draws unconditionally instead, from draw.begin: in an instance loop the front skips draw.end and the back defers the draw until the wait for idle flushes it. Tests cover a plain and a deferred instanced draw before a wait for idle, host syncs, and a macro read position waiting for a stalled back. Ocarina of Time in Kokiri village: macro DMA reads that wait for writers stay at the 313 from boot, fps unchanged. Co-Authored-By: Claude Opus 5.5 --- src/tests/video_core/maxwell_fast_method.cpp | 84 +++++++++++++++++++ src/video_core/buffer_cache/buffer_cache.h | 12 --- .../buffer_cache/buffer_cache_base.h | 1 - src/video_core/engines/maxwell_3d.cpp | 4 +- src/video_core/gpu_split.h | 5 +- src/video_core/nx2.h | 1 - 6 files changed, 88 insertions(+), 19 deletions(-) diff --git a/src/tests/video_core/maxwell_fast_method.cpp b/src/tests/video_core/maxwell_fast_method.cpp index d105be4b98..8c33a8ed30 100644 --- a/src/tests/video_core/maxwell_fast_method.cpp +++ b/src/tests/video_core/maxwell_fast_method.cpp @@ -3,6 +3,7 @@ #include #include +#include #include #include #include @@ -12,6 +13,7 @@ #include #include +#include #include "common/page_table.h" #include "common/scope_exit.h" @@ -816,3 +818,85 @@ TEST_CASE("Maxwell3D keeps state written after the split stops over the front's CHECK(engine.shadow_state.reg_array[state] == 0x5678); CHECK(engine.regs.reg_array[state] == 0x5678); } + +TEST_CASE("Split front orders queued draws before reads at the next wait for idle", + "[video_core][gpu_split_dependencies]") { + using Access = VideoCommon::GpuSplitTestAccess; + constexpr u32 WaitForIdle = MAXWELL3D_REG_INDEX(wait_for_idle); + constexpr GPUVAddr address = 0x100000; + // Instance id in draw.begin bits 26-27: a plain draw, or the first draw of an instanced + // loop that the back defers until the wait for idle flushes it. + const u32 begin = GENERATE(0u, 1u << 26); + CAPTURE(begin); + Fixture f; + VideoCommon::GpuSplit split; + Access::Initialize(split, f.system); + auto& engine = f.fast; + const auto ready = [&] { return split.ReadReadyPosition(&f.memory, address, address + 4); }; + + // A wait for idle without draws orders nothing. + engine.FrontCallMethod(f.system, split, WaitForIdle, 0, true); + REQUIRE(ready() == 0); + // The back has executed nothing, so nothing has told the front this draw writes memory. + engine.FrontCallMethod(f.system, split, DrawBegin, begin, true); + engine.FrontCallMethod(f.system, split, DrawEnd, 0, true); + // Hardware orders macro reads after draws only across a wait for idle. + REQUIRE(ready() == 0); + engine.FrontCallMethod(f.system, split, WaitForIdle, 0, true); + REQUIRE(ready() == Access::End(split)); + REQUIRE(split.WriterPosition() == Access::End(split)); + // Recorded once: a second wait for idle does not move it. + const u64 after_draw = Access::End(split); + engine.FrontCallMethod(f.system, split, WaitForIdle, 0, true); + REQUIRE(ready() == after_draw); +} + +TEST_CASE("Split host syncs order queued draws as waits for idle do", + "[video_core][gpu_split_dependencies]") { + using Access = VideoCommon::GpuSplitTestAccess; + Fixture f; + VideoCommon::GpuSplit split; + Access::Initialize(split, f.system); + split.NoteSync(); + REQUIRE(split.WriterPosition() == 0); + f.fast.FrontCallMethod(f.system, split, DrawBegin, 0, true); + f.fast.FrontCallMethod(f.system, split, DrawEnd, 0, true); + REQUIRE(split.WriterPosition() == 0); + split.NoteSync(); + REQUIRE(split.WriterPosition() == Access::End(split)); +} + +TEST_CASE("Split front read after a draw and a wait for idle waits for a delayed back", + "[video_core][gpu_split_dependencies]") { + using Access = VideoCommon::GpuSplitTestAccess; + constexpr GPUVAddr address = 0x100000; + Fixture f; + f.system.Initialize(); + VideoCommon::GpuSplit split; + Access::Start(split, f.system); + SCOPE_EXIT { split.Stop(); }; + auto& engine = f.fast; + // The front drains once as it starts: start it before the back stalls. + engine.FrontCallMethod(f.system, split, f.state_methods.front(), 0x1234, true); + std::atomic release{false}; + std::atomic drawn{false}; + // The back stalls before the first draw it executes, as it can behind real work. + split.Defer([&release] { + while (!release.load(std::memory_order_acquire)) { + std::this_thread::yield(); + } + }); + engine.FrontCallMethod(f.system, split, DrawBegin, 0, true); + engine.FrontCallMethod(f.system, split, DrawEnd, 0, true); + // Stands for the draw's shader stores. + split.Defer([&drawn] { drawn.store(true, std::memory_order_release); }); + engine.FrontCallMethod(f.system, split, MAXWELL3D_REG_INDEX(wait_for_idle), 0, true); + const u64 target = split.ReadReadyPosition(&f.memory, address, address + 4); + REQUIRE(target > split.Consumed()); + std::jthread releaser([&release] { + std::this_thread::sleep_for(std::chrono::milliseconds{20}); + release.store(true, std::memory_order_release); + }); + split.WaitForPosition(target, VideoCommon::DrainSite::MacroDmaRead); + REQUIRE(drawn.load(std::memory_order_acquire)); +} diff --git a/src/video_core/buffer_cache/buffer_cache.h b/src/video_core/buffer_cache/buffer_cache.h index f45bd0a501..c5b31a09c4 100644 --- a/src/video_core/buffer_cache/buffer_cache.h +++ b/src/video_core/buffer_cache/buffer_cache.h @@ -1609,23 +1609,12 @@ bool BufferCache

::BindMultiRangeStorage(const Binding& binding, bool is_writt } } -template -void BufferCache

::NoteDrawWritesMemory() { - if (!Tegra::g_nx2_draws_write_memory.exchange(true, std::memory_order_relaxed)) { - LOG_INFO(HW_GPU, "A draw writes a storage or texture buffer: the split front now orders " - "draws before macro DMA reads at waits for idle"); - } -} - template void BufferCache

::BindHostGraphicsStorageBuffers(size_t stage) { u32 binding_index = 0; ForEachEnabledBit(channel_state->enabled_storage_buffers[stage], [&](u32 index) { const Binding& binding = channel_state->storage_buffers[stage][index]; const bool is_written = ((channel_state->written_storage_buffers[stage] >> index) & 1) != 0; - if (is_written) { - NoteDrawWritesMemory(); - } if (BindMultiRangeStorage(binding, is_written, graphics_segments)) { return; } @@ -1660,7 +1649,6 @@ void BufferCache

::BindHostGraphicsTextureBuffers(size_t stage) { const bool is_written = ((channel_state->written_texture_buffers[stage] >> index) & 1) != 0; if (is_written) { - NoteDrawWritesMemory(); MarkWrittenBuffer(binding.buffer_id, binding.device_addr, size); } diff --git a/src/video_core/buffer_cache/buffer_cache_base.h b/src/video_core/buffer_cache/buffer_cache_base.h index 708a14a65e..24309bf1b5 100644 --- a/src/video_core/buffer_cache/buffer_cache_base.h +++ b/src/video_core/buffer_cache/buffer_cache_base.h @@ -515,7 +515,6 @@ class BufferCache : public VideoCommon::ChannelSetupCaches #include "common/common_types.h" -#include "video_core/nx2.h" namespace Core { class System; @@ -178,9 +177,7 @@ class GpuSplit { } /// A draw was just sent. Draws write memory the front does not track (shader stores). void NoteDraw() { - if (Tegra::g_nx2_draws_write_memory.load(std::memory_order_relaxed)) { - draws_queued = true; - } + draws_queued = true; } /// A wait for idle was just sent: what the draws before it wrote is visible after it, as /// the writes of compute work are. diff --git a/src/video_core/nx2.h b/src/video_core/nx2.h index 59df8dff23..839ed27946 100644 --- a/src/video_core/nx2.h +++ b/src/video_core/nx2.h @@ -24,7 +24,6 @@ inline std::atomic g_nx2_ampere{false}; // Set once a draw wrote a storage or texture buffer: from then on the split's front counts // draws as memory writers at the next wait for idle (macro DMA reads of what draws wrote). -inline std::atomic g_nx2_draws_write_memory{false}; // Device pages the GPU has written through the buffer cache (shader stores, copies), ever. Title 1 // runs command lists the GPU writes each frame (20-byte entries its culling fills in); read From a49a1a322bbe6ed102201bb96edf51be47e10b5a Mon Sep 17 00:00:00 2001 From: Ihitko <31191757+Ihitko@users.noreply.github.com> Date: Sat, 7 Nov 2026 14:31:12 -0500 Subject: [PATCH 16/16] vulkan: keep used pipelines below 8,000, import the cache version 53 records - EvictUnusedPipelines only evicts while more than 8,000 graphics pipelines are cached, and then the longest unused ones a draw has used, down to 7/8 of the limit (EDEN_NX2_PIPELINE_EVICT_ABOVE; 0 restores the old behavior). Prebuilt pipelines no draw has used keep #550's budget. Ocarina of Time evicted 6,182 pipelines in a 37 min session (~1,000-2,400 cached at a time) and draws waited 230 s in all for pipelines being built again. - The first boot copies the cache version 53 records beside this version's (vulkan_import, loaded after it) and merges the v53 used-pipeline list, so pipelines a v53 build used are built at boot again. The records are the same format: #432 bumped the version only for the SPIR-V it emits, and every record is translated again when it loads. The v53 files are left as they are. Skipping draws on unbuilt pipelines is left to #465's native_async_draws setting. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01LTxNSeWfimx1VbMuL9DgoN --- .../renderer_vulkan/vk_pipeline_cache.cpp | 86 ++++++++++++++++++- 1 file changed, 84 insertions(+), 2 deletions(-) diff --git a/src/video_core/renderer_vulkan/vk_pipeline_cache.cpp b/src/video_core/renderer_vulkan/vk_pipeline_cache.cpp index 81e2cacba7..25f7994943 100644 --- a/src/video_core/renderer_vulkan/vk_pipeline_cache.cpp +++ b/src/video_core/renderer_vulkan/vk_pipeline_cache.cpp @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -739,6 +740,32 @@ bool PreloadAllPipelines() { }(); return all; } + +// Adds the pipelines of an older version's used-pipeline list to this version's (a pipeline in both +// keeps its more recent use), so the pipelines an older build used are built at boot too. +void MergeUsedPipelines(const std::filesystem::path& destination, + const std::filesystem::path& source) { + constexpr u8 KeepAllAges = 255; + const auto older = LoadUsedPipelines(source, KeepAllAges); + if (older.empty()) { + return; + } + auto merged = LoadUsedPipelines(destination, KeepAllAges); + for (const auto& [hash, age] : older) { + auto [it, inserted] = merged.try_emplace(hash, age); + if (!inserted && age < it->second) { + it->second = age; + } + } + std::ofstream file{destination, std::ios::binary | std::ios::trunc}; + const u32 count = static_cast(merged.size()); + file.write(reinterpret_cast(&UsedPipelinesMagic), sizeof(UsedPipelinesMagic)); + file.write(reinterpret_cast(&count), sizeof(count)); + for (const auto& entry : merged) { + file.write(reinterpret_cast(&entry.first), sizeof(entry.first)); + file.write(reinterpret_cast(&entry.second), sizeof(entry.second)); + } +} } // Anonymous namespace PipelineCache::~PipelineCache() { @@ -957,6 +984,28 @@ void PipelineCache::LoadDiskResources(u64 title_id, std::stop_token stop_loading std::filesystem::copy_file(VideoCommon::VersionedPipelineCachePath(base_dir, "vulkan_used", 52), used_pipelines_filename, std::filesystem::copy_options::none, migration_error); + // Cache version 53 holds the same records as this version: #432 moved to 54 for the SPIR-V of + // its NaN-defined min/max, and every record is translated again when it loads. A cache of this + // version started before this import lacks the places a version 53 build had been, so the + // version 53 records are copied once beside it (vulkan_import, loaded after it) and its + // used-pipeline list is merged into this one, so those pipelines are built at boot again. The + // version 53 files stay as they are for the builds that still read them. + const auto import_filename = + VideoCommon::VersionedPipelineCachePath(base_dir, "vulkan_import", CACHE_VERSION); + auto import_done_filename = import_filename; + import_done_filename += ".done"; + std::error_code import_error; + if (!std::filesystem::exists(import_done_filename, import_error) && + VideoCommon::CopyCompatiblePipelineCache( + VideoCommon::VersionedPipelineCachePath(base_dir, "vulkan", 53), import_filename, + shader_magic, 53, CACHE_VERSION)) { + std::ofstream import_done{import_done_filename}; + import_done.close(); + MergeUsedPipelines(used_pipelines_filename, + VideoCommon::VersionedPipelineCachePath(base_dir, "vulkan_used", 53)); + LOG_INFO(Render_Vulkan, "Imported the cache version 53 pipelines to {}", + Common::FS::PathToUTF8String(import_filename)); + } used_pipelines = LoadUsedPipelines(used_pipelines_filename, MaxUsedPipelineAge(device)); used_pipelines_saved = std::chrono::steady_clock::now(); size_t skipped_pipelines = 0; @@ -1113,6 +1162,9 @@ void PipelineCache::LoadDiskResources(u64 title_id, std::stop_token stop_loading }}; VideoCommon::LoadPipelines(stop_loading, pipeline_cache_filename, CACHE_VERSION, load_compute, load_graphics); + // The imported cache version 53 records (see above). A record both files hold is built once. + VideoCommon::LoadPipelines(stop_loading, import_filename, CACHE_VERSION, load_compute, + load_graphics); for (auto& [key, candidates] : compute_envs) { OrderComputeTranslations(candidates); queue_compute(key, std::move(candidates)); @@ -1289,12 +1341,42 @@ void PipelineCache::EvictUnusedPipelines() { } // Each built pipeline holds ~0.45 MB of driver memory and none was ever destroyed, so it grew // with every place a session went (title 1: 1,500 on its title screen alone). One unused for - // this long is destroyed; it takes 2-3 ms to build again from the driver's pipeline cache. + // this long may be destroyed, but each one is translated again on the GPU thread when its place + // is revisited: Ocarina of Time evicted 6,182 in a 37 min session while ~1,000-2,400 stayed, + // and draws waited 230 s in all for pipelines being built. So pipelines are only evicted while + // more than 8,000 are cached (~3.6 GB of driver memory), and then the longest unused first, + // down to 7/8 of that. EDEN_NX2_PIPELINE_EVICT_ABOVE sets the limit (0 evicts as before). constexpr u64 EvictAfterFrames = 9000; if (frame_count <= EvictAfterFrames) { return; } + static const size_t evict_above = [] { + const char* value = std::getenv("EDEN_NX2_PIPELINE_EVICT_ABOVE"); + return value != nullptr ? static_cast(std::strtoull(value, nullptr, 10)) + : size_t{8000}; + }(); + if (graphics_cache.size() <= evict_above) { + return; + } const u64 cutoff = frame_count - EvictAfterFrames; + // The last use at or before which pipelines are evicted: the oldest used ones until the cache + // is back to 7/8 of the limit, or every one unused since the cutoff if that is fewer. Prebuilt + // pipelines no draw has used yet are left to their budget below. + std::vector last_uses; + for (const auto& entry : graphics_cache) { + const GraphicsPipeline* const candidate = entry.second.get(); + if (candidate != nullptr && candidate != current_pipeline && candidate->IsBuilt() && + candidate->WasUsed() && candidate->LastUse() < cutoff) { + last_uses.push_back(candidate->LastUse()); + } + } + const size_t excess = graphics_cache.size() - (evict_above - evict_above / 8); + u64 evict_through = cutoff - 1; + if (excess < last_uses.size()) { + const auto nth = last_uses.begin() + static_cast(excess - 1); + std::nth_element(last_uses.begin(), nth, last_uses.end()); + evict_through = last_uses[excess - 1]; + } const u64 tick = scheduler.CurrentTick(); // ~0.9 GB of driver memory at ~0.45 MB each. constexpr size_t MaxUnusedPrebuiltPipelines = 2000; @@ -1303,7 +1385,7 @@ void PipelineCache::EvictUnusedPipelines() { for (auto it = graphics_cache.begin(); it != graphics_cache.end();) { GraphicsPipeline* const pipeline = it->second.get(); if (pipeline == nullptr || pipeline == current_pipeline || !pipeline->IsBuilt() || - pipeline->LastUse() >= cutoff) { + pipeline->LastUse() >= cutoff || pipeline->LastUse() > evict_through) { ++it; continue; }