From f9b24e53baa0638fbb4b162e9833fd39720b313c Mon Sep 17 00:00:00 2001 From: DjouWish Date: Sat, 7 Nov 2026 17:30:55 +0100 Subject: [PATCH 001/105] video_core, vulkan: AMD and split GPU thread fixes for Ocarina of Time Measured on The Legend of Zelda: Ocarina of Time (0400486024DEC000), Ryzen 7 5800X + Radeon RX 9070 XT 16 GB, AMD driver 32.0.32015.7029, Windows 11. Each change was A/B tested in game behind a switch first. - vk_query_cache: copy only the slots of the samples queries being synced and zero the gaps (the wide copy waited on slots no query ended, ~8 ms each on AMD); on AMD's driver, preset the synced slots to 1024 samples and copy without WAIT_BIT, and read host results without waiting, so a late result counts as visible. 9 -> 22 FPS in Kokiri Forest, no vanishing scene. - texture_cache: cap the collector at 3/5 of device memory when that is above 5 GiB. At 5 GiB a 16 GB card re-created ~10k images and re-uploaded 13-16 GB every 22 s; 14 -> 24 FPS at a busy spot. - vk_pipeline_cache: skip native draws whose pipeline still builds by default on AMD's driver (EDEN_NX2_ASYNC_DISCARD=0/1 still decides): 1-1.4 s freezes per new pipeline while running. - pipeline_optimization: build compute e0a992208d87eeb8 unoptimized on AMD (1.3-4 s stalls). - vk_rasterizer: submit every 512 draws on AMD's driver (128 elsewhere; EDEN_NX2_FLUSH_DRAWS still decides): 256 gave +10-12 FPS where it was already fast, 512 a further 35 -> 50 at one spot. - decoders: split unswizzles of 1 MiB or more by lines over a quarter of the host threads (at most 4) plus the caller: +2-4 FPS running, +1-2 standing; 6 threads or a 256 KiB threshold made busy spots drop harder. - puller: take a semaphore acquire without draining the split back when the word in guest memory (on a page no GPU copy holds newer data for) already meets it: 73% of ~2800 acquires/s, 573k checked against the drained read without a contradiction; publishes and notes a sync point as the drain did. Co-Authored-By: Claude Opus 5.5 --- src/video_core/engines/puller.cpp | 20 ++- .../renderer_vulkan/pipeline_optimization.h | 3 +- .../renderer_vulkan/submission_policy.h | 2 + .../renderer_vulkan/vk_pipeline_cache.cpp | 17 ++- .../renderer_vulkan/vk_query_cache.cpp | 139 ++++++++++++++---- .../renderer_vulkan/vk_rasterizer.cpp | 8 +- src/video_core/texture_cache/texture_cache.h | 11 +- src/video_core/textures/decoders.cpp | 124 +++++++++++++++- 8 files changed, 280 insertions(+), 44 deletions(-) diff --git a/src/video_core/engines/puller.cpp b/src/video_core/engines/puller.cpp index 8fb6a43f47..466943e8a8 100644 --- a/src/video_core/engines/puller.cpp +++ b/src/video_core/engines/puller.cpp @@ -255,8 +255,26 @@ void Puller::ProcessSemaphoreTriggerMethod(DmaPusher& dma_pusher) { return; } VideoCommon::Nx2Profile::Add(VideoCommon::Nx2Profile::Counter::AcquireChecks, 1); + // Most acquires wait on a value the CPU released already (73% of ~2800/s in Ocarina of + // Time's Kokiri Forest), so the drain before the check only stalls both halves of the + // split GPU thread. When the word in guest memory, on a page no GPU copy holds newer data + // for, already meets the value, take the acquire without draining (checked against the + // drained read on 573k acquires: never contradicted). + if (auto& split = VideoCommon::GpuSplit::Get(); split.Active()) { + if (const auto device = dma_pusher.memory_manager.GpuToCpuAddress(acquire.address); + device && !Tegra::Nx2IsGpuWritten(*device) && + acquire.Matches(dma_pusher.memory_manager.Read(acquire.address))) { + // What the drain also did: the back gets what was sent (a release or syncpoint + // the CPU waits on may be among it: without this the title hung at boot), and + // command lists the GPU wrote before this point wait for it when read. + split.Publish(); + split.NoteSync(); + return; + } + } DrainBack(); - if (acquire.Matches(ReadSemaphoreWord(dma_pusher, acquire.address))) { + const bool drained_match = acquire.Matches(ReadSemaphoreWord(dma_pusher, acquire.address)); + if (drained_match) { return; } // Registered before the fences run, so a release among them that meets the value counts diff --git a/src/video_core/renderer_vulkan/pipeline_optimization.h b/src/video_core/renderer_vulkan/pipeline_optimization.h index 33a2fdd790..8d88ef329a 100644 --- a/src/video_core/renderer_vulkan/pipeline_optimization.h +++ b/src/video_core/renderer_vulkan/pipeline_optimization.h @@ -12,9 +12,10 @@ namespace Vulkan { inline bool BuildsUnoptimized(VkDriverId driver_id, u64 shader_hash) { // The Legend of Zelda: Ocarina of Time (0400486024DEC000), driver 2.0.353 on an RX 6600 XT - static constexpr std::array amd_proprietary{ + static constexpr std::array amd_proprietary{ 0xeff3ccbe1438679dULL, // compute shader built as the AV1 intro starts: 6 min (#200) 0xf9ea14a22394f5b1ULL, // compute shader built as the AV1 intro starts: 18 min (#200) + 0xe0a992208d87eeb8ULL, // Kokiri Forest compute shader: 1.3-4 s stalls (RX 9070 XT, 2.0.416) }; return driver_id == VK_DRIVER_ID_AMD_PROPRIETARY && std::ranges::find(amd_proprietary, shader_hash) != amd_proprietary.end(); diff --git a/src/video_core/renderer_vulkan/submission_policy.h b/src/video_core/renderer_vulkan/submission_policy.h index a109edc713..dc8566cb85 100644 --- a/src/video_core/renderer_vulkan/submission_policy.h +++ b/src/video_core/renderer_vulkan/submission_policy.h @@ -11,6 +11,8 @@ namespace Vulkan::SubmissionPolicy { inline constexpr u32 DefaultNativeBudget = 128; +/// Draws per submission on AMD's proprietary driver (see RasterizerVulkan::FlushWork). +inline constexpr u32 AmdNativeBudget = 512; enum class Action { None, diff --git a/src/video_core/renderer_vulkan/vk_pipeline_cache.cpp b/src/video_core/renderer_vulkan/vk_pipeline_cache.cpp index a95bb8a4de..66641fe44f 100644 --- a/src/video_core/renderer_vulkan/vk_pipeline_cache.cpp +++ b/src/video_core/renderer_vulkan/vk_pipeline_cache.cpp @@ -1200,8 +1200,8 @@ void PipelineCache::EvictUnusedPipelines() { } /// Returns `pipeline` when it is built or the current draw must wait for it, and nullptr when the -/// draw may be skipped while it builds (asynchronous shaders; native draws only with -/// EDEN_NX2_ASYNC_DISCARD=1). Each nullptr returned for an unbuilt pipeline is counted for the log. +/// draw may be skipped while it builds (asynchronous shaders; native draws only on AMD's driver or +/// with EDEN_NX2_ASYNC_DISCARD=1). Each nullptr returned for an unbuilt pipeline is counted for the log. GraphicsPipeline* PipelineCache::BuiltPipeline(GraphicsPipeline* pipeline) const noexcept { if (pipeline->IsBuilt()) { return pipeline; @@ -1209,12 +1209,15 @@ GraphicsPipeline* PipelineCache::BuiltPipeline(GraphicsPipeline* pipeline) const if (!use_asynchronous_shaders) { return pipeline; } - // EDEN_NX2_ASYNC_DISCARD=1: skip native draws whose pipeline is still building instead of - // waiting for it (experimental; the draw may be missing for a frame or more). - /// Reads EDEN_NX2_ASYNC_DISCARD once and logs when it is on. - static const bool discard_native = [] { + // Skip native draws whose pipeline is still building instead of waiting for it (the draw + // may be missing for a frame or more). On by default on AMD's proprietary driver, whose + // pipeline builds froze Ocarina of Time for 1-1.4 s each while running into a new area; + // EDEN_NX2_ASYNC_DISCARD=1/0 turns it on or off on any driver. + static const bool discard_native = [this] { const char* value = std::getenv("EDEN_NX2_ASYNC_DISCARD"); - const bool enabled = value != nullptr && value[0] == '1'; + const bool enabled = value != nullptr + ? value[0] == '1' + : device.GetDriverID() == VK_DRIVER_ID_AMD_PROPRIETARY; if (enabled) { LOG_INFO(Render_Vulkan, "EDEN_NX2_ASYNC_DISCARD: native draws skip unbuilt pipelines"); } diff --git a/src/video_core/renderer_vulkan/vk_query_cache.cpp b/src/video_core/renderer_vulkan/vk_query_cache.cpp index 4d61dd8023..f8d626c7bb 100644 --- a/src/video_core/renderer_vulkan/vk_query_cache.cpp +++ b/src/video_core/renderer_vulkan/vk_query_cache.cpp @@ -85,6 +85,26 @@ class SamplesQueryBank : public VideoCommon::BankBase { void Sync(size_t start, size_t size) { const auto& dev = device.GetLogical(); + // AMD's driver: read what is available without blocking; a result that has not landed + // counts as visible (1024 samples), as in the GPU-side resolve (SyncWrites). + if (device.GetDriverID() == VK_DRIVER_ID_AMD_PROPRIETARY) { + std::array, BANK_SIZE> values{}; + const VkResult query_result = dev.GetQueryResults( + *query_pool, static_cast(start), static_cast(size), + sizeof(values[0]) * size, values.data(), sizeof(values[0]), + VK_QUERY_RESULT_64_BIT | VK_QUERY_RESULT_WITH_AVAILABILITY_BIT); + if (query_result == VK_ERROR_DEVICE_LOST) { + device.ReportLoss(); + throw vk::Exception(query_result); + } + if (query_result != VK_SUCCESS && query_result != VK_NOT_READY) { + throw vk::Exception(query_result); + } + for (size_t i = 0; i < size; ++i) { + host_results[start + i] = values[i].second != 0 ? values[i].first : 1024; + } + return; + } const VkResult query_result = dev.GetQueryResults( *query_pool, static_cast(start), static_cast(size), sizeof(u64) * size, &host_results[start], sizeof(u64), VK_QUERY_RESULT_64_BIT | VK_QUERY_RESULT_WAIT_BIT); @@ -271,36 +291,95 @@ class SamplesStreamer : public BaseStreamer { resolve_buffers.push_back(resolve_buffer_index); size_t base_offset = 0; - ApplyBanksWideOp(pending_sync, [&](SamplesQueryBank* bank, size_t start, - size_t amount) { - size_t bank_id = bank->GetIndex(); - auto& resolve_buffer = buffers[resolve_buffer_index]; - VkQueryPool query_pool = bank->GetInnerPool(); - scheduler.RequestOutsideRenderPassOperationContext(); - scheduler.Record([start, amount, base_offset, query_pool, - buffer = *resolve_buffer](vk::CommandBuffer cmdbuf) { - const VkBufferMemoryBarrier copy_query_pool_barrier{ - .sType = VK_STRUCTURE_TYPE_BUFFER_MEMORY_BARRIER, - .pNext = nullptr, - .srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT, - .dstAccessMask = VK_ACCESS_TRANSFER_READ_BIT, - .srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, - .dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, - .buffer = buffer, - .offset = base_offset, - .size = amount * SamplesQueryBank::QUERY_SIZE, - }; - - cmdbuf.CopyQueryPoolResults( - query_pool, static_cast(start), static_cast(amount), buffer, - static_cast(base_offset), SamplesQueryBank::QUERY_SIZE, - VK_QUERY_RESULT_WAIT_BIT | VK_QUERY_RESULT_64_BIT); - cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_TRANSFER_BIT, - VK_PIPELINE_STAGE_TRANSFER_BIT, 0, copy_query_pool_barrier); - }); - offsets[bank_id] = {start, base_offset}; - base_offset += amount * SamplesQueryBank::QUERY_SIZE; - }); + // Copy only the slots of the queries being synced and zero the slots between them. The + // wide copy waits (WAIT_BIT) on every slot of the span; a slot in it no query ended + // stalls the GPU until the driver gives up (~8 ms each on AMD, 10 times a frame in + // Ocarina of Time's Kokiri Forest). + { + std::map>> runs_by_bank; + for (auto q : pending_sync) { + ApplyBankOp(GetQuery(q), [&](SamplesQueryBank* bank, size_t start, size_t amount) { + runs_by_bank[bank->GetIndex()].emplace_back(start, start + amount); + }); + } + for (auto& [bank_id, runs] : runs_by_bank) { + std::ranges::sort(runs); + std::vector> merged; + for (const auto& run : runs) { + if (!merged.empty() && run.first <= merged.back().second) { + merged.back().second = (std::max)(merged.back().second, run.second); + } else { + merged.push_back(run); + } + } + const size_t span_start = merged.front().first; + const size_t span_end = merged.back().second; + VkQueryPool query_pool = bank_pool.GetBank(bank_id).GetInnerPool(); + auto& resolve_buffer = buffers[resolve_buffer_index]; + // AMD's driver: even after a barrier on all prior work, a result can land after + // the copy (with WAIT_BIT each copy waited ~9.6 ms). Preset every synced slot to a + // "visible" count (1024 samples) and copy without waiting: Vulkan leaves the slots + // of unavailable queries unwritten, so a late result reads as visible (an occluded + // object drawn) instead of hidden (the scene vanished) or a stall (9 -> 22 FPS). + const bool preset_visible = device.GetDriverID() == VK_DRIVER_ID_AMD_PROPRIETARY; + const bool no_wait = preset_visible; + scheduler.RequestOutsideRenderPassOperationContext(); + scheduler.Record([merged, span_start, span_end, base_offset, query_pool, + buffer = *resolve_buffer, no_wait, + preset_visible](vk::CommandBuffer cmdbuf) { + constexpr size_t query_size = SamplesQueryBank::QUERY_SIZE; + for (size_t i = 0; i + 1 < merged.size(); ++i) { + const size_t gap_begin = merged[i].second; + const size_t gap_end = merged[i + 1].first; + cmdbuf.FillBuffer(buffer, base_offset + (gap_begin - span_start) * query_size, + (gap_end - gap_begin) * query_size, 0); + } + if (preset_visible) { + constexpr u32 visible_samples = 1024; + for (const auto& [start, end] : merged) { + for (size_t slot = start; slot < end; ++slot) { + const size_t offset = base_offset + (slot - span_start) * query_size; + cmdbuf.FillBuffer(buffer, offset, 4, visible_samples); + cmdbuf.FillBuffer(buffer, offset + 4, 4, 0); + } + } + // The copy below writes over these presets (write after write). + static constexpr VkMemoryBarrier PRESET_BARRIER{ + .sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER, + .pNext = nullptr, + .srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT, + .dstAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT, + }; + cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_TRANSFER_BIT, + VK_PIPELINE_STAGE_TRANSFER_BIT, 0, PRESET_BARRIER); + } + for (const auto& [start, end] : merged) { + cmdbuf.CopyQueryPoolResults( + query_pool, static_cast(start), static_cast(end - start), + buffer, base_offset + (start - span_start) * query_size, query_size, + no_wait ? VkQueryResultFlags{VK_QUERY_RESULT_64_BIT} + : VkQueryResultFlags{VK_QUERY_RESULT_WAIT_BIT | + VK_QUERY_RESULT_64_BIT}); + } + const VkBufferMemoryBarrier copy_query_pool_barrier{ + .sType = VK_STRUCTURE_TYPE_BUFFER_MEMORY_BARRIER, + .pNext = nullptr, + .srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT, + .dstAccessMask = VK_ACCESS_TRANSFER_READ_BIT, + .srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, + .dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, + .buffer = buffer, + .offset = base_offset, + .size = (span_end - span_start) * query_size, + }; + cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_TRANSFER_BIT, + VK_PIPELINE_STAGE_TRANSFER_BIT, 0, + copy_query_pool_barrier); + }); + offsets[bank_id] = {span_start, base_offset}; + base_offset += (span_end - span_start) * SamplesQueryBank::QUERY_SIZE; + } + } // Convert queries bool has_multi_queries = false; diff --git a/src/video_core/renderer_vulkan/vk_rasterizer.cpp b/src/video_core/renderer_vulkan/vk_rasterizer.cpp index 8c9df2d7ae..ad9b99ca8f 100644 --- a/src/video_core/renderer_vulkan/vk_rasterizer.cpp +++ b/src/video_core/renderer_vulkan/vk_rasterizer.cpp @@ -2281,7 +2281,13 @@ void RasterizerVulkan::FlushWork() { } return std::nullopt; }(); - budget = configured_budget.value_or(SubmissionPolicy::DefaultNativeBudget); + // AMD's driver pays much more per submission. Ocarina of Time on an RX 9070 XT: every 256 + // draws instead of 128 gave +10-12 FPS where it was already fast, every 512 a further + // 35 -> 50 FPS at one spot standing still. + const u32 default_budget = device.GetDriverID() == VK_DRIVER_ID_AMD_PROPRIETARY + ? SubmissionPolicy::AmdNativeBudget + : SubmissionPolicy::DefaultNativeBudget; + budget = configured_budget.value_or(default_budget); } u32 legacy_budget = 4096; if (!native && !android) { diff --git a/src/video_core/texture_cache/texture_cache.h b/src/video_core/texture_cache/texture_cache.h index 8fa7c1149e..3781879d48 100644 --- a/src/video_core/texture_cache/texture_cache.h +++ b/src/video_core/texture_cache/texture_cache.h @@ -264,8 +264,15 @@ template void TextureCache

::ApplyThresholdOverrides(bool log) { // On every GPU: past what a frame needs, more device memory only keeps the textures of earlier // scenes (title 1 filled 6.7 GB of an 8 GB card and draws its races in ~5 GB), so collect from - // MaxCriticalMemory on rather than near the card's budget. - critical_memory = (std::min)(critical_memory, MaxCriticalMemory); + // MaxCriticalMemory on rather than near the card's budget. Larger cards get 3/5 of their device + // memory: at 5 GiB on a 16 GB card, Ocarina of Time's Kokiri Forest kept the collector in its + // aggressive mode, re-creating ~10k images and re-uploading 13-16 GB every 22 s (with ~9 GiB: + // ~600 images and 1.5 GB). 8 GB cards keep 5 GiB. + u64 max_critical = MaxCriticalMemory; + if constexpr (HAS_DEVICE_MEMORY_INFO) { + max_critical = (std::max)(max_critical, runtime.GetDeviceLocalMemory() / 5 * 3); + } + critical_memory = (std::min)(critical_memory, max_critical); expected_memory = (std::min)(expected_memory, critical_memory - (std::min)(critical_memory, u64{768_MiB})); minimum_memory = diff --git a/src/video_core/textures/decoders.cpp b/src/video_core/textures/decoders.cpp index ef50be156b..0c0b1f085d 100644 --- a/src/video_core/textures/decoders.cpp +++ b/src/video_core/textures/decoders.cpp @@ -5,19 +5,126 @@ // SPDX-License-Identifier: GPL-2.0-or-later #include +#include #include +#include #include +#include +#include #include +#include +#include #include "common/alignment.h" #include "common/assert.h" #include "common/bit_util.h" #include "common/div_ceil.h" +#include "common/thread.h" #include "video_core/gpu.h" #include "video_core/textures/decoders.h" namespace Tegra::Texture { namespace { + +/// Large unswizzles (new textures while running through Ocarina of Time's Kokiri Forest: ~11% of +/// the split GPU back thread) are split by lines over helper threads plus the caller, which waits +/// for all of them: the result is the same, only sooner. A quarter of the host threads, at most 4 +/// (more competed with the guest cores and made busy spots drop harder). +class LinePool { +public: + static LinePool& Get() { + static LinePool pool; + return pool; + } + + [[nodiscard]] bool Enabled() const { + return !workers.empty(); + } + + /// Calls func(begin, end) over [0, count) in chunks; returns once every chunk is done. + void Run(u32 count, const std::function& func) { + std::unique_lock busy{run_mutex, std::try_to_lock}; + if (!busy || workers.empty()) { + func(0, count); + return; + } + { + std::scoped_lock lock{mutex}; + job = &func; + total = count; + chunk = (std::max)(8U, count / static_cast((workers.size() + 1) * 4)); + next.store(0, std::memory_order_relaxed); + pending = static_cast(workers.size()); + ++generation; + } + work_cv.notify_all(); + Work(func); + std::unique_lock lock{mutex}; + done_cv.wait(lock, [this] { return pending == 0; }); + job = nullptr; + } + +private: + LinePool() { + const u32 count = (std::min)(4U, std::thread::hardware_concurrency() / 4); + for (u32 i = 0; i < count; ++i) { + workers.emplace_back([this](std::stop_token stop) { Loop(stop); }); + } + } + + ~LinePool() { + for (auto& worker : workers) { + worker.request_stop(); + } + work_cv.notify_all(); + } + + void Work(const std::function& func) { + for (;;) { + const u32 begin = next.fetch_add(chunk, std::memory_order_relaxed); + if (begin >= total) { + return; + } + func(begin, (std::min)(begin + chunk, total)); + } + } + + void Loop(std::stop_token stop) { + Common::SetCurrentThreadName("Unswizzle"); + u64 seen = 0; + for (;;) { + const std::function* func; + { + std::unique_lock lock{mutex}; + work_cv.wait(lock, [&] { return stop.stop_requested() || generation != seen; }); + if (stop.stop_requested()) { + return; + } + seen = generation; + func = job; + } + Work(*func); + { + std::scoped_lock lock{mutex}; + --pending; + } + done_cv.notify_one(); + } + } + + std::mutex run_mutex; + std::mutex mutex; + std::condition_variable work_cv; + std::condition_variable done_cv; + const std::function* job{}; + u32 total{}; + u32 chunk{1}; + std::atomic next{}; + u32 pending{}; + u64 generation{}; + std::vector workers; +}; + constexpr u32 NativeGobOffset(u32 offset, bool native_tiling) { if (!native_tiling) return offset; const u32 difference = ((offset >> 5) ^ (offset >> 6)) & 1; @@ -63,11 +170,15 @@ void SwizzleImpl(std::span output, std::span input, u32 width, u32 const u32 block_depth_mask = (1U << block_depth) - 1; const u32 x_shift = GOB_SIZE_SHIFT + block_height + block_depth; - for (u32 slice = 0; slice < depth; ++slice) { + // Each (slice, line) writes its own bytes of the output: lines can go to other threads. + const auto lines = [&](u32 first, u32 end) { + for (u32 index = first; index < end; ++index) { + const u32 slice = index / height; + const u32 line = index % height; const u32 z = slice + origin_z; const u32 offset_z = (z >> block_depth) * slice_size + ((z & block_depth_mask) << (GOB_SIZE_SHIFT + block_height)); - for (u32 line = 0; line < height; ++line) { + { const u32 y = line + origin_y; const u32 swizzled_y = pdep(y); @@ -94,6 +205,15 @@ void SwizzleImpl(std::span output, std::span input, u32 width, u32 } } } + }; + const u32 total_lines = depth * height; + // From 1 MiB: splitting medium textures too (256 KiB) made busy spots drop harder. + constexpr u64 min_bytes = u64{1} << 20; + if (!TO_LINEAR && LinePool::Get().Enabled() && u64{pitch} * total_lines >= min_bytes) { + LinePool::Get().Run(total_lines, lines); + } else { + lines(0, total_lines); + } } template From 249eaa7d0d7f92248711e16b19a6c6758cbee884 Mon Sep 17 00:00:00 2001 From: DjouWish Date: Sat, 7 Nov 2026 18:07:38 +0100 Subject: [PATCH 002/105] Address review: semaphore writes, texture cache budget, pool shutdown - Drop the AMD default for native draw discard: a skipped native draw can write persistent resources; EDEN_NX2_ASYNC_DISCARD=1 stays opt-in (and evaluating the default per PipelineCache is moot). - puller: the early acquire also requires that no release to the semaphore is still queued for the back. The front notes the address of each puller WriteLong and each 3D report release/report it sends (GpuSplit::NoteSemaphoreWrite / SemaphoreWritePending). - texture_cache: derive the larger cap from the driver's memory budget; GetDeviceLocalMemory is capped at 7 GiB outside the aggressive VRAM mode, which left the cap at 5 GiB on 16 GB cards. - decoders: request the helper threads' stop under the pool mutex and join them in the destructor, so a worker cannot miss the wakeup. Co-Authored-By: Claude Opus 5.5 --- src/video_core/engines/maxwell_3d.cpp | 12 +++++++++ src/video_core/engines/puller.cpp | 9 +++++-- src/video_core/gpu_split.h | 26 ++++++++++++++++++++ src/video_core/texture_cache/texture_cache.h | 14 ++++++----- src/video_core/textures/decoders.cpp | 10 ++++++-- 5 files changed, 61 insertions(+), 10 deletions(-) diff --git a/src/video_core/engines/maxwell_3d.cpp b/src/video_core/engines/maxwell_3d.cpp index 29a9ceab3d..67e997820d 100644 --- a/src/video_core/engines/maxwell_3d.cpp +++ b/src/video_core/engines/maxwell_3d.cpp @@ -1213,6 +1213,18 @@ void Maxwell3D::FrontWrite(VideoCommon::GpuSplit& split, FrontState& fs, u32 met } else { split.Method(this, method, argument, is_last_call, nullptr); } + if (method == MAXWELL3D_REG_INDEX(report_semaphore.query)) { + // A release or report writes guest memory a semaphore acquire may wait on (Puller). + Regs::ReportSemaphore report{}; + report.query.raw = value; + const auto operation = report.query.operation.Value(); + if (operation == Regs::ReportSemaphore::Operation::Release || + operation == Regs::ReportSemaphore::Operation::ReportOnly) { + split.NoteSemaphoreWrite( + (GPUVAddr{fs.regs[MAXWELL3D_REG_INDEX(report_semaphore.address_high)]} << 32) | + fs.regs[MAXWELL3D_REG_INDEX(report_semaphore.address_low)]); + } + } } void Maxwell3D::FrontCallMethod(Core::System& system, VideoCommon::GpuSplit& split, u32 method, diff --git a/src/video_core/engines/puller.cpp b/src/video_core/engines/puller.cpp index 466943e8a8..ee64af431c 100644 --- a/src/video_core/engines/puller.cpp +++ b/src/video_core/engines/puller.cpp @@ -238,6 +238,9 @@ void Puller::ProcessSemaphoreTriggerMethod(DmaPusher& dma_pusher) { OnBack([rasterizer = dma_pusher.rasterizer, sequence_address, properties, payload] { rasterizer->Query(sequence_address, VideoCommon::QueryType::Payload, properties, payload, 0); }); + if (auto& split = VideoCommon::GpuSplit::Get(); split.Active()) { + split.NoteSemaphoreWrite(sequence_address); + } } else if (g_nx2_ampere) { // Switch 2 waits on semaphores the CPU (or another channel) releases, e.g. once it has // written the commands this channel runs next: suspend the channel until the value is @@ -258,11 +261,13 @@ void Puller::ProcessSemaphoreTriggerMethod(DmaPusher& dma_pusher) { // Most acquires wait on a value the CPU released already (73% of ~2800/s in Ocarina of // Time's Kokiri Forest), so the drain before the check only stalls both halves of the // split GPU thread. When the word in guest memory, on a page no GPU copy holds newer data - // for, already meets the value, take the acquire without draining (checked against the - // drained read on 573k acquires: never contradicted). + // for, already meets the value and no release to it is still queued for the back, take + // the acquire without draining (checked against the drained read on 573k acquires: never + // contradicted). if (auto& split = VideoCommon::GpuSplit::Get(); split.Active()) { if (const auto device = dma_pusher.memory_manager.GpuToCpuAddress(acquire.address); device && !Tegra::Nx2IsGpuWritten(*device) && + !split.SemaphoreWritePending(acquire.address) && acquire.Matches(dma_pusher.memory_manager.Read(acquire.address))) { // What the drain also did: the back gets what was sent (a release or syncpoint // the CPU waits on may be among it: without this the title hung at boot), and diff --git a/src/video_core/gpu_split.h b/src/video_core/gpu_split.h index 2e9250ff53..731c770aba 100644 --- a/src/video_core/gpu_split.h +++ b/src/video_core/gpu_split.h @@ -10,6 +10,7 @@ #include #include #include +#include #include "common/common_types.h" @@ -171,6 +172,30 @@ class GpuSplit { void NoteSync() { last_sync_position = write_position; } + /// A semaphore or report release to `address` was just sent (front thread only). + void NoteSemaphoreWrite(u64 address) { + if (pending_semaphore_writes.size() >= 4096) { + consumed_cache = consumed.load(std::memory_order_acquire); + std::erase_if(pending_semaphore_writes, + [this](const auto& entry) { return entry.second <= consumed_cache; }); + } + pending_semaphore_writes[address] = write_position; + } + /// Whether a release to `address` was sent and the back has not executed it yet (front only). + [[nodiscard]] bool SemaphoreWritePending(u64 address) { + const auto it = pending_semaphore_writes.find(address); + if (it == pending_semaphore_writes.end()) { + return false; + } + if (it->second > consumed_cache) { + consumed_cache = consumed.load(std::memory_order_acquire); + } + if (it->second <= consumed_cache) { + pending_semaphore_writes.erase(it); + return false; + } + return true; + } /// Waits until the back executed the work sent before the last NoteSync. Returns whether any /// of it was still queued. bool WaitForSync(); @@ -247,6 +272,7 @@ class GpuSplit { u64 write_position{}; u64 last_writer_position{}; u64 last_sync_position{}; + std::unordered_map pending_semaphore_writes; u64 published_local{}; u64 consumed_cache{}; /// Words between publishes: small right after a drain, while the back has nothing to do. diff --git a/src/video_core/texture_cache/texture_cache.h b/src/video_core/texture_cache/texture_cache.h index afde8bac9e..06e0a7fcee 100644 --- a/src/video_core/texture_cache/texture_cache.h +++ b/src/video_core/texture_cache/texture_cache.h @@ -268,13 +268,15 @@ template void TextureCache

::ApplyThresholdOverrides(bool log) { // On every GPU: past what a frame needs, more device memory only keeps the textures of earlier // scenes (title 1 filled 6.7 GB of an 8 GB card and draws its races in ~5 GB), so collect from - // MaxCriticalMemory on rather than near the card's budget. Larger cards get 3/5 of their device - // memory: at 5 GiB on a 16 GB card, Ocarina of Time's Kokiri Forest kept the collector in its - // aggressive mode, re-creating ~10k images and re-uploading 13-16 GB every 22 s (with ~9 GiB: - // ~600 images and 1.5 GB). 8 GB cards keep 5 GiB. + // MaxCriticalMemory on rather than near the card's budget. Larger cards get 3/5 of the driver's + // budget (VK_EXT_memory_budget; GetDeviceLocalMemory is capped at 7 GiB outside the aggressive + // VRAM mode): at 5 GiB on a 16 GB card, Ocarina of Time's Kokiri Forest kept the collector in + // its aggressive mode, re-creating ~10k images and re-uploading 13-16 GB every 22 s (with + // ~9 GiB: ~600 images and 1.5 GB). 8 GB cards keep 5 GiB. TickFrame still lowers the + // thresholds below the budget as it shrinks. u64 max_critical = MaxCriticalMemory; - if constexpr (HAS_DEVICE_MEMORY_INFO) { - max_critical = (std::max)(max_critical, runtime.GetDeviceLocalMemory() / 5 * 3); + if constexpr (requires { runtime.GetDeviceMemoryBudget(); }) { + max_critical = (std::max)(max_critical, runtime.GetDeviceMemoryBudget() / 5 * 3); } critical_memory = (std::min)(critical_memory, max_critical); expected_memory = diff --git a/src/video_core/textures/decoders.cpp b/src/video_core/textures/decoders.cpp index 0c0b1f085d..7d7bf60676 100644 --- a/src/video_core/textures/decoders.cpp +++ b/src/video_core/textures/decoders.cpp @@ -73,10 +73,16 @@ class LinePool { } ~LinePool() { - for (auto& worker : workers) { - worker.request_stop(); + { + // Under the mutex: a worker between its predicate check and its wait would otherwise + // miss the notification and never return to be joined. + std::scoped_lock lock{mutex}; + for (auto& worker : workers) { + worker.request_stop(); + } } work_cv.notify_all(); + workers.clear(); } void Work(const std::function& func) { From 07287cc219dd7d1ffd6160d61f48220a1e66cd4d Mon Sep 17 00:00:00 2001 From: DjouWish Date: Sat, 7 Nov 2026 18:48:23 +0100 Subject: [PATCH 003/105] puller: drop the early semaphore acquire from this PR Review found queued writes the shortcut does not see (SEM_EXECUTE releases, DMA engine clears, compute QMD releases). Covering them all would leave it almost never taken for ~1 FPS, so the acquire drains as upstream again; the pending-write tracking it needed goes too. Co-Authored-By: Claude Opus 5.5 --- src/video_core/engines/maxwell_3d.cpp | 12 ------------ src/video_core/engines/puller.cpp | 25 +------------------------ src/video_core/gpu_split.h | 26 -------------------------- 3 files changed, 1 insertion(+), 62 deletions(-) diff --git a/src/video_core/engines/maxwell_3d.cpp b/src/video_core/engines/maxwell_3d.cpp index 67e997820d..29a9ceab3d 100644 --- a/src/video_core/engines/maxwell_3d.cpp +++ b/src/video_core/engines/maxwell_3d.cpp @@ -1213,18 +1213,6 @@ void Maxwell3D::FrontWrite(VideoCommon::GpuSplit& split, FrontState& fs, u32 met } else { split.Method(this, method, argument, is_last_call, nullptr); } - if (method == MAXWELL3D_REG_INDEX(report_semaphore.query)) { - // A release or report writes guest memory a semaphore acquire may wait on (Puller). - Regs::ReportSemaphore report{}; - report.query.raw = value; - const auto operation = report.query.operation.Value(); - if (operation == Regs::ReportSemaphore::Operation::Release || - operation == Regs::ReportSemaphore::Operation::ReportOnly) { - split.NoteSemaphoreWrite( - (GPUVAddr{fs.regs[MAXWELL3D_REG_INDEX(report_semaphore.address_high)]} << 32) | - fs.regs[MAXWELL3D_REG_INDEX(report_semaphore.address_low)]); - } - } } void Maxwell3D::FrontCallMethod(Core::System& system, VideoCommon::GpuSplit& split, u32 method, diff --git a/src/video_core/engines/puller.cpp b/src/video_core/engines/puller.cpp index ee64af431c..8fb6a43f47 100644 --- a/src/video_core/engines/puller.cpp +++ b/src/video_core/engines/puller.cpp @@ -238,9 +238,6 @@ void Puller::ProcessSemaphoreTriggerMethod(DmaPusher& dma_pusher) { OnBack([rasterizer = dma_pusher.rasterizer, sequence_address, properties, payload] { rasterizer->Query(sequence_address, VideoCommon::QueryType::Payload, properties, payload, 0); }); - if (auto& split = VideoCommon::GpuSplit::Get(); split.Active()) { - split.NoteSemaphoreWrite(sequence_address); - } } else if (g_nx2_ampere) { // Switch 2 waits on semaphores the CPU (or another channel) releases, e.g. once it has // written the commands this channel runs next: suspend the channel until the value is @@ -258,28 +255,8 @@ void Puller::ProcessSemaphoreTriggerMethod(DmaPusher& dma_pusher) { return; } VideoCommon::Nx2Profile::Add(VideoCommon::Nx2Profile::Counter::AcquireChecks, 1); - // Most acquires wait on a value the CPU released already (73% of ~2800/s in Ocarina of - // Time's Kokiri Forest), so the drain before the check only stalls both halves of the - // split GPU thread. When the word in guest memory, on a page no GPU copy holds newer data - // for, already meets the value and no release to it is still queued for the back, take - // the acquire without draining (checked against the drained read on 573k acquires: never - // contradicted). - if (auto& split = VideoCommon::GpuSplit::Get(); split.Active()) { - if (const auto device = dma_pusher.memory_manager.GpuToCpuAddress(acquire.address); - device && !Tegra::Nx2IsGpuWritten(*device) && - !split.SemaphoreWritePending(acquire.address) && - acquire.Matches(dma_pusher.memory_manager.Read(acquire.address))) { - // What the drain also did: the back gets what was sent (a release or syncpoint - // the CPU waits on may be among it: without this the title hung at boot), and - // command lists the GPU wrote before this point wait for it when read. - split.Publish(); - split.NoteSync(); - return; - } - } DrainBack(); - const bool drained_match = acquire.Matches(ReadSemaphoreWord(dma_pusher, acquire.address)); - if (drained_match) { + if (acquire.Matches(ReadSemaphoreWord(dma_pusher, acquire.address))) { return; } // Registered before the fences run, so a release among them that meets the value counts diff --git a/src/video_core/gpu_split.h b/src/video_core/gpu_split.h index 731c770aba..2e9250ff53 100644 --- a/src/video_core/gpu_split.h +++ b/src/video_core/gpu_split.h @@ -10,7 +10,6 @@ #include #include #include -#include #include "common/common_types.h" @@ -172,30 +171,6 @@ class GpuSplit { void NoteSync() { last_sync_position = write_position; } - /// A semaphore or report release to `address` was just sent (front thread only). - void NoteSemaphoreWrite(u64 address) { - if (pending_semaphore_writes.size() >= 4096) { - consumed_cache = consumed.load(std::memory_order_acquire); - std::erase_if(pending_semaphore_writes, - [this](const auto& entry) { return entry.second <= consumed_cache; }); - } - pending_semaphore_writes[address] = write_position; - } - /// Whether a release to `address` was sent and the back has not executed it yet (front only). - [[nodiscard]] bool SemaphoreWritePending(u64 address) { - const auto it = pending_semaphore_writes.find(address); - if (it == pending_semaphore_writes.end()) { - return false; - } - if (it->second > consumed_cache) { - consumed_cache = consumed.load(std::memory_order_acquire); - } - if (it->second <= consumed_cache) { - pending_semaphore_writes.erase(it); - return false; - } - return true; - } /// Waits until the back executed the work sent before the last NoteSync. Returns whether any /// of it was still queued. bool WaitForSync(); @@ -272,7 +247,6 @@ class GpuSplit { u64 write_position{}; u64 last_writer_position{}; u64 last_sync_position{}; - std::unordered_map pending_semaphore_writes; u64 published_local{}; u64 consumed_cache{}; /// Words between publishes: small right after a drain, while the back has nothing to do. From 6e261395a6265db3d2c84ac29250bee72493e2e6 Mon Sep 17 00:00:00 2001 From: DjouWish Date: Sat, 7 Nov 2026 21:10:53 +0100 Subject: [PATCH 004/105] vk_query_cache: resolve AMD samples queries from their availability On AMD's driver the samples queries were copied without WAIT_BIT over slots preset to 1024, relying on the copy leaving unavailable slots untouched, which the Vulkan spec leaves undefined. Copy them with 64_BIT | WITH_AVAILABILITY instead and resolve each slot on the GPU (resolve_query_availability.comp, QueryAvailabilityResolvePass): its count when it had landed, 1024 samples (visible) otherwise. Approach from Savulska's test on #573. RX 9070 XT, same MSVC build: 50 -> 56.7 FPS under the Deku Tree, 24 -> 25.0 at the slow Kokiri spot, the same in motion. EDEN_NX2_QUERY_AVAILABILITY=0 keeps the preset method for A/B. Co-Authored-By: Claude Opus 5.5 --- src/video_core/host_shaders/CMakeLists.txt | 1 + .../resolve_query_availability.comp | 33 +++++++ .../renderer_vulkan/vk_compute_pass.cpp | 48 ++++++++++ .../renderer_vulkan/vk_compute_pass.h | 19 ++++ .../renderer_vulkan/vk_query_cache.cpp | 95 +++++++++++++++---- 5 files changed, 179 insertions(+), 17 deletions(-) create mode 100644 src/video_core/host_shaders/resolve_query_availability.comp diff --git a/src/video_core/host_shaders/CMakeLists.txt b/src/video_core/host_shaders/CMakeLists.txt index 6951a09e44..de0751364f 100644 --- a/src/video_core/host_shaders/CMakeLists.txt +++ b/src/video_core/host_shaders/CMakeLists.txt @@ -69,6 +69,7 @@ set(SHADER_FILES ${CMAKE_CURRENT_SOURCE_DIR}/queries_prefix_scan_sum.comp ${CMAKE_CURRENT_SOURCE_DIR}/queries_prefix_scan_sum_nosubgroups.comp ${CMAKE_CURRENT_SOURCE_DIR}/resolve_conditional_render.comp + ${CMAKE_CURRENT_SOURCE_DIR}/resolve_query_availability.comp ${CMAKE_CURRENT_SOURCE_DIR}/smaa_edge_detection.vert ${CMAKE_CURRENT_SOURCE_DIR}/smaa_edge_detection.frag ${CMAKE_CURRENT_SOURCE_DIR}/smaa_blending_weight_calculation.vert diff --git a/src/video_core/host_shaders/resolve_query_availability.comp b/src/video_core/host_shaders/resolve_query_availability.comp new file mode 100644 index 0000000000..92ef3256c1 --- /dev/null +++ b/src/video_core/host_shaders/resolve_query_availability.comp @@ -0,0 +1,33 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later + +// Samples query results copied with VK_QUERY_RESULT_64_BIT | VK_QUERY_RESULT_WITH_AVAILABILITY_BIT +// and no wait: each entry is the 64-bit count followed by a 64-bit availability word. A query +// whose result has not landed yet resolves to 1024 samples (visible) instead of whatever the copy +// wrote for it, which the Vulkan spec leaves undefined. + +#version 450 + +layout(local_size_x = 64) in; + +layout(std430, binding = 0) readonly buffer Copied { + uvec4 copied[]; // count low, count high, availability low, availability high +}; + +layout(std430, binding = 1) writeonly buffer Resolved { + uvec2 resolved[]; // 64-bit count +}; + +layout(push_constant) uniform PushConstants { + uint count; +}; + +void main() { + const uint index = gl_GlobalInvocationID.x; + if (index >= count) { + return; + } + const uvec4 entry = copied[index]; + const bool available = (entry.z | entry.w) != 0u; + resolved[index] = available ? entry.xy : uvec2(1024u, 0u); +} diff --git a/src/video_core/renderer_vulkan/vk_compute_pass.cpp b/src/video_core/renderer_vulkan/vk_compute_pass.cpp index 5902b7d8f5..f694aa4971 100644 --- a/src/video_core/renderer_vulkan/vk_compute_pass.cpp +++ b/src/video_core/renderer_vulkan/vk_compute_pass.cpp @@ -24,6 +24,7 @@ #include "video_core/host_shaders/queries_prefix_scan_sum_comp_spv.h" #include "video_core/host_shaders/queries_prefix_scan_sum_nosubgroups_comp_spv.h" #include "video_core/host_shaders/resolve_conditional_render_comp_spv.h" +#include "video_core/host_shaders/resolve_query_availability_comp_spv.h" #include "video_core/host_shaders/vulkan_quad_indexed_comp_spv.h" #include "video_core/host_shaders/vulkan_uint8_comp_spv.h" #include "video_core/host_shaders/vulkan_vertex_repack_comp_spv.h" @@ -539,6 +540,53 @@ void ConditionalRenderingResolvePass::Resolve(VkBuffer dst_buffer, VkBuffer src_ }); } +QueryAvailabilityResolvePass::QueryAvailabilityResolvePass( + const Device& device_, Scheduler& scheduler_, DescriptorPool& descriptor_pool_, + ComputePassDescriptorQueue& compute_pass_descriptor_queue_) + : ComputePass(device_, scheduler_, descriptor_pool_, INPUT_OUTPUT_DESCRIPTOR_SET_BINDINGS, + INPUT_OUTPUT_DESCRIPTOR_UPDATE_TEMPLATE, INPUT_OUTPUT_BANK_INFO, + COMPUTE_PUSH_CONSTANT_RANGE, RESOLVE_QUERY_AVAILABILITY_COMP_SPV), + scheduler{scheduler_}, compute_pass_descriptor_queue{compute_pass_descriptor_queue_} {} + +void QueryAvailabilityResolvePass::Resolve(VkBuffer dst_buffer, VkDeviceSize dst_offset, + VkBuffer src_buffer, VkDeviceSize src_offset, + u32 count) { + if (count == 0) { + return; + } + compute_pass_descriptor_queue.Acquire(scheduler, 2); + compute_pass_descriptor_queue.AddBuffer(src_buffer, src_offset, VkDeviceSize{count} * 16); + compute_pass_descriptor_queue.AddBuffer(dst_buffer, dst_offset, VkDeviceSize{count} * 8); + const void* const descriptor_data{compute_pass_descriptor_queue.UpdateData()}; + + scheduler.RequestOutsideRenderPassOperationContext(); + scheduler.Record([this, descriptor_data, count](vk::CommandBuffer cmdbuf) { + static constexpr VkMemoryBarrier read_barrier{ + .sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER, + .pNext = nullptr, + .srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT, + .dstAccessMask = VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT, + }; + static constexpr VkMemoryBarrier write_barrier{ + .sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER, + .pNext = nullptr, + .srcAccessMask = VK_ACCESS_SHADER_WRITE_BIT, + .dstAccessMask = VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_SHADER_READ_BIT, + }; + const VkDescriptorSet set = descriptor_allocator.Commit(); + device.GetLogical().UpdateDescriptorSet(set, *descriptor_template, descriptor_data); + cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, + 0, read_barrier); + cmdbuf.BindPipeline(VK_PIPELINE_BIND_POINT_COMPUTE, *pipeline); + cmdbuf.BindDescriptorSets(VK_PIPELINE_BIND_POINT_COMPUTE, *layout, 0, set, {}); + cmdbuf.PushConstants(*layout, VK_SHADER_STAGE_COMPUTE_BIT, count); + cmdbuf.Dispatch((count + 63) / 64, 1, 1); + cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, + VK_PIPELINE_STAGE_TRANSFER_BIT | VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, + 0, write_barrier); + }); +} + QueriesPrefixScanPass::QueriesPrefixScanPass( const Device& device_, Scheduler& scheduler_, DescriptorPool& descriptor_pool_, ComputePassDescriptorQueue& compute_pass_descriptor_queue_) diff --git a/src/video_core/renderer_vulkan/vk_compute_pass.h b/src/video_core/renderer_vulkan/vk_compute_pass.h index 9687bfff91..b1024e3562 100644 --- a/src/video_core/renderer_vulkan/vk_compute_pass.h +++ b/src/video_core/renderer_vulkan/vk_compute_pass.h @@ -120,6 +120,25 @@ class ConditionalRenderingResolvePass final : public ComputePass { ComputePassDescriptorQueue& compute_pass_descriptor_queue; }; +/// Resolves samples query results copied with availability (resolve_query_availability.comp): the +/// count when the query had landed, 1024 samples (visible) otherwise. +class QueryAvailabilityResolvePass final : public ComputePass { +public: + explicit QueryAvailabilityResolvePass(const Device& device_, Scheduler& scheduler_, + DescriptorPool& descriptor_pool_, + ComputePassDescriptorQueue& compute_pass_descriptor_queue_); + + /// `count` entries: 16 bytes each at `src_offset` in `src_buffer`, 8 bytes each at + /// `dst_offset` in `dst_buffer`. Orders the copy that wrote the source before it, and its own + /// writes before later transfer and shader reads. + void Resolve(VkBuffer dst_buffer, VkDeviceSize dst_offset, VkBuffer src_buffer, + VkDeviceSize src_offset, u32 count); + +private: + Scheduler& scheduler; + ComputePassDescriptorQueue& compute_pass_descriptor_queue; +}; + class QueriesPrefixScanPass final : public ComputePass { public: explicit QueriesPrefixScanPass(const Device& device_, Scheduler& scheduler_, diff --git a/src/video_core/renderer_vulkan/vk_query_cache.cpp b/src/video_core/renderer_vulkan/vk_query_cache.cpp index f8d626c7bb..bb70a04353 100644 --- a/src/video_core/renderer_vulkan/vk_query_cache.cpp +++ b/src/video_core/renderer_vulkan/vk_query_cache.cpp @@ -166,6 +166,10 @@ class SamplesStreamer : public BaseStreamer { amend_value = 0; accumulation_value = 0; queries_prefix_scan_pass.emplace(device, scheduler, descriptor_pool, compute_pass_descriptor_queue); + if (device.GetDriverID() == VK_DRIVER_ID_AMD_PROPRIETARY) { + availability_pass.emplace(device, scheduler, descriptor_pool, + compute_pass_descriptor_queue); + } const VkBufferCreateInfo buffer_ci = { .sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, @@ -317,33 +321,51 @@ class SamplesStreamer : public BaseStreamer { VkQueryPool query_pool = bank_pool.GetBank(bank_id).GetInnerPool(); auto& resolve_buffer = buffers[resolve_buffer_index]; // AMD's driver: even after a barrier on all prior work, a result can land after - // the copy (with WAIT_BIT each copy waited ~9.6 ms). Preset every synced slot to a - // "visible" count (1024 samples) and copy without waiting: Vulkan leaves the slots - // of unavailable queries unwritten, so a late result reads as visible (an occluded - // object drawn) instead of hidden (the scene vanished) or a stall (9 -> 22 FPS). - const bool preset_visible = device.GetDriverID() == VK_DRIVER_ID_AMD_PROPRIETARY; - const bool no_wait = preset_visible; + // the copy (with WAIT_BIT each copy waited ~9.6 ms). Copy without waiting, with each + // slot's availability, then resolve each slot on the GPU to its count when it had + // landed and to a "visible" 1024 samples otherwise: a late result reads as an + // occluded object drawn instead of hidden (the scene vanished) or a stall (9 -> 22 + // FPS in Kokiri Forest). The availability word is what decides, not what the copy + // writes for an unavailable query, which the Vulkan spec leaves undefined + // (approach from Savulska's test on #573). + // EDEN_NX2_QUERY_AVAILABILITY=0 (A/B): the earlier preset method instead, copying + // over slots preset to 1024 and relying on unavailable ones being left untouched. + static const bool use_availability = [] { + const char* const text = std::getenv("EDEN_NX2_QUERY_AVAILABILITY"); + return text == nullptr || text[0] != '0'; + }(); + const bool resolve_availability = availability_pass.has_value() && use_availability; + const bool preset_visible = availability_pass.has_value() && !use_availability; + const VkDeviceSize span_slots = span_end - span_start; + const VkDeviceSize availability_offset = base_offset * 2; + const VkBuffer availability_buffer = + resolve_availability ? AvailabilityBuffer(availability_offset + span_slots * 16) + : VkBuffer{}; scheduler.RequestOutsideRenderPassOperationContext(); scheduler.Record([merged, span_start, span_end, base_offset, query_pool, - buffer = *resolve_buffer, no_wait, - preset_visible](vk::CommandBuffer cmdbuf) { + buffer = *resolve_buffer, resolve_availability, availability_buffer, + availability_offset, preset_visible](vk::CommandBuffer cmdbuf) { constexpr size_t query_size = SamplesQueryBank::QUERY_SIZE; for (size_t i = 0; i + 1 < merged.size(); ++i) { const size_t gap_begin = merged[i].second; const size_t gap_end = merged[i + 1].first; cmdbuf.FillBuffer(buffer, base_offset + (gap_begin - span_start) * query_size, (gap_end - gap_begin) * query_size, 0); + if (resolve_availability) { + // Resolved as visible like any unavailable slot; nothing reads them. + cmdbuf.FillBuffer(availability_buffer, + availability_offset + (gap_begin - span_start) * 16, + (gap_end - gap_begin) * 16, 0); + } } if (preset_visible) { - constexpr u32 visible_samples = 1024; for (const auto& [start, end] : merged) { for (size_t slot = start; slot < end; ++slot) { const size_t offset = base_offset + (slot - span_start) * query_size; - cmdbuf.FillBuffer(buffer, offset, 4, visible_samples); + cmdbuf.FillBuffer(buffer, offset, 4, 1024); cmdbuf.FillBuffer(buffer, offset + 4, 4, 0); } } - // The copy below writes over these presets (write after write). static constexpr VkMemoryBarrier PRESET_BARRIER{ .sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER, .pNext = nullptr, @@ -354,12 +376,22 @@ class SamplesStreamer : public BaseStreamer { VK_PIPELINE_STAGE_TRANSFER_BIT, 0, PRESET_BARRIER); } for (const auto& [start, end] : merged) { - cmdbuf.CopyQueryPoolResults( - query_pool, static_cast(start), static_cast(end - start), - buffer, base_offset + (start - span_start) * query_size, query_size, - no_wait ? VkQueryResultFlags{VK_QUERY_RESULT_64_BIT} - : VkQueryResultFlags{VK_QUERY_RESULT_WAIT_BIT | - VK_QUERY_RESULT_64_BIT}); + if (resolve_availability) { + cmdbuf.CopyQueryPoolResults( + query_pool, static_cast(start), static_cast(end - start), + availability_buffer, availability_offset + (start - span_start) * 16, + 16, VK_QUERY_RESULT_64_BIT | VK_QUERY_RESULT_WITH_AVAILABILITY_BIT); + } else { + cmdbuf.CopyQueryPoolResults( + query_pool, static_cast(start), static_cast(end - start), + buffer, base_offset + (start - span_start) * query_size, query_size, + preset_visible ? VkQueryResultFlags{VK_QUERY_RESULT_64_BIT} + : VkQueryResultFlags{VK_QUERY_RESULT_WAIT_BIT | + VK_QUERY_RESULT_64_BIT}); + } + } + if (resolve_availability) { + return; // QueryAvailabilityResolvePass orders the copies before it. } const VkBufferMemoryBarrier copy_query_pool_barrier{ .sType = VK_STRUCTURE_TYPE_BUFFER_MEMORY_BARRIER, @@ -376,6 +408,10 @@ class SamplesStreamer : public BaseStreamer { VK_PIPELINE_STAGE_TRANSFER_BIT, 0, copy_query_pool_barrier); }); + if (resolve_availability) { + availability_pass->Resolve(*resolve_buffer, base_offset, availability_buffer, + availability_offset, static_cast(span_slots)); + } offsets[bank_id] = {span_start, base_offset}; base_offset += (span_end - span_start) * SamplesQueryBank::QUERY_SIZE; } @@ -716,6 +752,31 @@ class SamplesStreamer : public BaseStreamer { u64 bank_bound_resets{}; ///< Counts CloseCounter dropped once a query outgrew a bank. std::mutex flush_guard; std::optional queries_prefix_scan_pass; + /// AMD: results copied with availability (16 bytes a slot) before QueryAvailabilityResolvePass. + std::optional availability_pass; + std::vector availability_buffers; ///< The last is current; older ones stay alive + VkDeviceSize availability_capacity{}; + + /// The current availability buffer, grown to at least `size` bytes. + VkBuffer AvailabilityBuffer(VkDeviceSize size) { + if (size > availability_capacity) { + availability_capacity = (std::max)(size, availability_capacity * 2); + const VkBufferCreateInfo buffer_ci{ + .sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, + .pNext = nullptr, + .flags = 0, + .size = availability_capacity, + .usage = VK_BUFFER_USAGE_TRANSFER_DST_BIT | VK_BUFFER_USAGE_STORAGE_BUFFER_BIT, + .sharingMode = VK_SHARING_MODE_EXCLUSIVE, + .queueFamilyIndexCount = 0, + .pQueueFamilyIndices = nullptr, + }; + // Commands already recorded may still use the old one: it is kept, not destroyed. + availability_buffers.push_back( + memory_allocator.CreateBuffer(buffer_ci, MemoryUsage::DeviceLocal)); + } + return *availability_buffers.back(); + } }; // Transform feedback queries From 59d39a458e20a53b74dc0573de321e6c10a409f4 Mon Sep 17 00:00:00 2001 From: DjouWish Date: Sun, 8 Nov 2026 09:38:20 +0100 Subject: [PATCH 005/105] kepler_compute: don't shadow the state member (MSVC C4458 as error) Co-Authored-By: Claude Opus 5.5 --- src/video_core/engines/kepler_compute.cpp | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/video_core/engines/kepler_compute.cpp b/src/video_core/engines/kepler_compute.cpp index 66f2430580..68d4600c09 100644 --- a/src/video_core/engines/kepler_compute.cpp +++ b/src/video_core/engines/kepler_compute.cpp @@ -911,8 +911,8 @@ bool KeplerCompute::BypassOotNetwork(u64 program_id, u64 native_key) { if (!BypassEnabled(program_id)) { return false; } - auto* const state = rasterizer->OotNetworkBypass(); - if (!state) { + auto* const bypass = rasterizer->OotNetworkBypass(); + if (!bypass) { return false; } // The tonemap pass draws before the network every frame; until it has been seen (menus, the @@ -920,14 +920,14 @@ bool KeplerCompute::BypassOotNetwork(u64 program_id, u64 native_key) { // pass is being replaced, so a failed replacement brings the whole network back next frame. Maxwell3D::Regs::RenderTargetConfig tonemapped; { - std::scoped_lock lock{state->mutex}; - tonemapped = state->tonemapped; + std::scoped_lock lock{bypass->mutex}; + tonemapped = bypass->tonemapped; } if (tonemapped.Address() == 0) { return false; } if (native_key == PrepassProgramKey || native_key == NetworkProgramKey) { - return state->output_replaced.load(std::memory_order_relaxed); + return bypass->output_replaced.load(std::memory_order_relaxed); } if (native_key != OutputProgramKey) { return false; @@ -939,7 +939,7 @@ bool KeplerCompute::BypassOotNetwork(u64 program_id, u64 native_key) { tonemapped.format != Tegra::RenderTargetFormat::B10G11R11_FLOAT || tonemapped.tile_mode.is_pitch_linear || tonemapped.width != output->Width() || tonemapped.height != output->Height()) { - state->output_replaced.store(false, std::memory_order_relaxed); + bypass->output_replaced.store(false, std::memory_order_relaxed); static bool warned = false; if (!std::exchange(warned, true)) { LOG_WARNING(HW_GPU, @@ -984,7 +984,7 @@ bool KeplerCompute::BypassOotNetwork(u64 program_id, u64 native_key) { .src_y1 = height, }; const bool copied = rasterizer->AccelerateSurfaceCopy(source, destination, config); - state->output_replaced.store(copied, std::memory_order_relaxed); + bypass->output_replaced.store(copied, std::memory_order_relaxed); static bool logged = false; if (!std::exchange(logged, true)) { LOG_INFO(HW_GPU, From deb527e29ae87cb49c174c2fdbe1710438aba47a Mon Sep 17 00:00:00 2001 From: DjouWish Date: Sun, 8 Nov 2026 09:38:21 +0100 Subject: [PATCH 006/105] vk_query_cache: drop the rest of the preset method left by the merge Co-Authored-By: Claude Opus 5.5 --- .../renderer_vulkan/vk_query_cache.cpp | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/src/video_core/renderer_vulkan/vk_query_cache.cpp b/src/video_core/renderer_vulkan/vk_query_cache.cpp index 4c121be3a0..5a332a698b 100644 --- a/src/video_core/renderer_vulkan/vk_query_cache.cpp +++ b/src/video_core/renderer_vulkan/vk_query_cache.cpp @@ -362,22 +362,6 @@ class SamplesStreamer : public BaseStreamer { (gap_end - gap_begin) * 16, 0); } } - for (const auto& [start, end] : merged) { - for (size_t slot = start; slot < end; ++slot) { - const size_t offset = base_offset + (slot - span_start) * query_size; - cmdbuf.FillBuffer(buffer, offset, 4, 1024); - cmdbuf.FillBuffer(buffer, offset + 4, 4, 0); - } - } - static constexpr VkMemoryBarrier PRESET_BARRIER{ - .sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER, - .pNext = nullptr, - .srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT, - .dstAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT, - }; - cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_TRANSFER_BIT, - VK_PIPELINE_STAGE_TRANSFER_BIT, 0, PRESET_BARRIER); - } for (const auto& [start, end] : merged) { if (resolve_availability) { cmdbuf.CopyQueryPoolResults( From 0534753acef2fe0e13740cb8f1a3c0c58178f6ee Mon Sep 17 00:00:00 2001 From: DjouWish Date: Sun, 8 Nov 2026 09:54:06 +0100 Subject: [PATCH 007/105] vk_query_cache: resolve availability on AMD's proprietary driver unconditionally The A/B switch against #679's copy is gone: on an RX 9070 XT (2.0.416, 8 GB simulated) it gave 54 FPS under the Deku tree and a less steady run through Kokiri Forest, this copy 56-58 FPS and a steadier run. Co-Authored-By: Claude Opus 5.5 --- src/video_core/renderer_vulkan/vk_query_cache.cpp | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/src/video_core/renderer_vulkan/vk_query_cache.cpp b/src/video_core/renderer_vulkan/vk_query_cache.cpp index 5a332a698b..4245f038da 100644 --- a/src/video_core/renderer_vulkan/vk_query_cache.cpp +++ b/src/video_core/renderer_vulkan/vk_query_cache.cpp @@ -332,14 +332,9 @@ class SamplesStreamer : public BaseStreamer { // occluded object drawn instead of hidden (the scene vanished) or a stall (9 -> 22 // FPS in Kokiri Forest). The availability word is what decides, not what the copy // writes for an unavailable query, which the Vulkan spec leaves undefined - // (approach from Savulska's test on #573). - // EDEN_NX2_QUERY_AVAILABILITY=0 (local A/B only, not for the PR): #679's copy - // that waits on the newest query instead. - static const bool use_availability = [] { - const char* const text = std::getenv("EDEN_NX2_QUERY_AVAILABILITY"); - return text == nullptr || text[0] != '0'; - }(); - const bool resolve_availability = availability_pass.has_value() && use_availability; + // (approach from Savulska's test on #573). Waiting on the newest query instead + // (#679) gave 54 FPS under the Deku tree on an RX 9070 XT, this 56-58. + const bool resolve_availability = availability_pass.has_value(); const VkDeviceSize span_slots = span_end - span_start; const VkDeviceSize availability_offset = base_offset * 2; const VkBuffer availability_buffer = From a2178a63718938e409d2bdd6f4806392a2cc4ece Mon Sep 17 00:00:00 2001 From: DjouWish Date: Sun, 8 Nov 2026 10:43:24 +0100 Subject: [PATCH 008/105] vk_query_cache: keep gap slots at zero through the availability resolve The prefix scan of multi-slot queries sums every slot of the resolve buffer, gaps between synced runs included; resolved as unavailable they read as 1024. Mark them available with a zero count instead (CodeRabbit review on #573). Co-Authored-By: Claude Opus 5.5 --- src/video_core/renderer_vulkan/vk_query_cache.cpp | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/video_core/renderer_vulkan/vk_query_cache.cpp b/src/video_core/renderer_vulkan/vk_query_cache.cpp index 4245f038da..b342fd1b33 100644 --- a/src/video_core/renderer_vulkan/vk_query_cache.cpp +++ b/src/video_core/renderer_vulkan/vk_query_cache.cpp @@ -351,10 +351,14 @@ class SamplesStreamer : public BaseStreamer { cmdbuf.FillBuffer(buffer, base_offset + (gap_begin - span_start) * query_size, (gap_end - gap_begin) * query_size, 0); if (resolve_availability) { - // Resolved as visible like any unavailable slot; nothing reads them. - cmdbuf.FillBuffer(availability_buffer, - availability_offset + (gap_begin - span_start) * 16, - (gap_end - gap_begin) * 16, 0); + // Available with a zero count, so the resolve keeps them at zero: the + // prefix scan of multi-slot queries sums every slot, gaps included. + for (size_t slot = gap_begin; slot < gap_end; ++slot) { + const VkDeviceSize offset = + availability_offset + (slot - span_start) * 16; + cmdbuf.FillBuffer(availability_buffer, offset, 8, 0); + cmdbuf.FillBuffer(availability_buffer, offset + 8, 8, 1); + } } } for (const auto& [start, end] : merged) { From fc42430f51841a8dda685c8c5780a0894e732f12 Mon Sep 17 00:00:00 2001 From: jpkto <255363096+jpkto@users.noreply.github.com> Date: Sat, 7 Nov 2026 18:06:45 +0100 Subject: [PATCH 009/105] TEST: log every 10 s how long guest threads wait for the GPU In Hyrule Field Ocarina of Time reads GPU-written memory about 80 times a second (the "CPU read of GPU-written memory" count), and every such read that GetFlushArea can't serve makes the reading thread wait for a flush on the GPU thread. This measures how much of each frame those waits take, to tell whether they hold the frame rate down. One line, "GPU waits in 10.0 s:", always on: - frames presented, and the time between presents (average, longest); - reactive flushes: count, total and longest wait, per frame, by the cache that held the data (texture or buffer) and by guest core, split into queue (until the GPU thread starts the flush), flush (applying the collected CPU writes, then the downloads: the time spent in Scheduler::Finish waiting for the host GPU, how many flushes had nothing to download and how many bytes they copied back) and wake; - the four areas that waited longest and, for buffer pages, the GPU writes that marked them in that window (what wrote them, the range, the largest one); - the other sync operations guest threads wait for (CPU writes over GPU-written buffers, WaitForIdle); - guest reads and writes that took the rasterizer path, by core, with the time of their cache lookups (GetFlushArea, OnCPUWrite), which take the buffer and texture cache locks the GPU thread holds while it draws; - how busy the GPU thread and the split's back thread were, every Scheduler::Finish with its time, and every wait of FlushQueryResults for pending query results (macro DMA reads of report regions, #215). Cost: a counter only its own core writes per rasterizer-path read or write; two clock reads more per GetFlushArea or OnCPUWrite call; one load per buffer write mark (four compares while pages are watched); two clock reads and an atomic add per GPU thread idle period, per Scheduler::Finish and per query result wait. The clock is the wall clock's time stamp counter (steady_clock where it is not invariant), the one EDEN_NX2_GPU_PROFILE uses. Co-Authored-By: Claude Opus 5.5 --- src/core/hle/service/nx2_gsv.h | 2 + src/core/memory.cpp | 6 + src/video_core/CMakeLists.txt | 1 + src/video_core/buffer_cache/buffer_cache.h | 34 +- .../buffer_cache/buffer_cache_base.h | 5 +- src/video_core/gpu.cpp | 25 +- src/video_core/gpu_split.cpp | 2 + src/video_core/gpu_thread.cpp | 5 + src/video_core/gpu_wait_stats.h | 600 ++++++++++++++++++ src/video_core/rasterizer_download_area.h | 8 + .../renderer_vulkan/vk_rasterizer.cpp | 5 + .../renderer_vulkan/vk_scheduler.cpp | 3 + src/video_core/texture_cache/texture_cache.h | 2 + 13 files changed, 688 insertions(+), 10 deletions(-) create mode 100644 src/video_core/gpu_wait_stats.h diff --git a/src/core/hle/service/nx2_gsv.h b/src/core/hle/service/nx2_gsv.h index f968e2f6ec..8ebadcf313 100644 --- a/src/core/hle/service/nx2_gsv.h +++ b/src/core/hle/service/nx2_gsv.h @@ -39,6 +39,7 @@ #include "core/frontend/framebuffer_layout.h" #include "core/perf_stats.h" #include "video_core/gpu.h" +#include "video_core/gpu_wait_stats.h" #include "video_core/framebuffer_config.h" #include "core/core_timing.h" #include "core/hle/kernel/k_client_session.h" @@ -1757,6 +1758,7 @@ class INx2Vi2Session final : public ServiceFramework { m_vsync->trace_present.store(m_presents, std::memory_order_relaxed); } const u64 present = Core::g_nx2_present_count.fetch_add(1, std::memory_order_relaxed) + 1; + VideoCommon::GpuWaitStats::NotePresent(); if (Core::g_nx2_present_time_services.load(std::memory_order_relaxed) != 0) { const u64 present_ns = std::chrono::duration_cast( std::chrono::steady_clock::now().time_since_epoch()) diff --git a/src/core/memory.cpp b/src/core/memory.cpp index 82fac1059c..5724301b12 100644 --- a/src/core/memory.cpp +++ b/src/core/memory.cpp @@ -35,6 +35,7 @@ #include "core/hle/kernel/k_process.h" #include "core/memory.h" #include "video_core/gpu.h" +#include "video_core/gpu_wait_stats.h" #include "video_core/host1x/gpu_device_memory_manager.h" #include "video_core/host1x/host1x.h" #include "video_core/rasterizer_download_area.h" @@ -891,6 +892,7 @@ struct Memory::Impl { return; } const size_t core = system.GetCurrentHostThreadID(); + VideoCommon::GpuWaitStats::CountRasterizerRead(core); if (core >= Core::Hardware::NUM_CPU_CORES) { // A service thread has no slot in the arrays below, and it does not use a slot of a // core. It keeps no area, so it asks on each read. @@ -943,6 +945,7 @@ struct Memory::Impl { sys_core_guard.unlock(); } }; + VideoCommon::GpuWaitStats::CountRasterizerWrite(core); gpu_device_memory->ApplyOpOnPointer(p, scratch_buffers[core], [&](DAddr address) { auto& current_area = rasterizer_write_areas[core]; PAddr subaddress = address >> YUZU_PAGEBITS; @@ -952,8 +955,11 @@ struct Memory::Impl { // we should let the compiler just do it's thing... if (current_area.last_address != subaddress) { // Short circuit the need to check for address/size + const u64 check_start = VideoCommon::GpuWaitStats::Now(); auto const do_collection = (address != 0 && size != 0) && system.GPU().OnCPUWrite(address, size); + VideoCommon::GpuWaitStats::AddWriteCheck(core, VideoCommon::GpuWaitStats::Now() - + check_start); if (do_collection) { current_area.last_address = subaddress; } else { diff --git a/src/video_core/CMakeLists.txt b/src/video_core/CMakeLists.txt index 816500db62..4813673d96 100644 --- a/src/video_core/CMakeLists.txt +++ b/src/video_core/CMakeLists.txt @@ -106,6 +106,7 @@ add_library(video_core STATIC gpu_split.h gpu_thread.cpp gpu_thread.h + gpu_wait_stats.h guest_memory.h invalidation_accumulator.h memory_manager.cpp diff --git a/src/video_core/buffer_cache/buffer_cache.h b/src/video_core/buffer_cache/buffer_cache.h index bf5aaabbba..744c365feb 100644 --- a/src/video_core/buffer_cache/buffer_cache.h +++ b/src/video_core/buffer_cache/buffer_cache.h @@ -387,6 +387,12 @@ void BufferCache

::DownloadRangesMemory(std::span> if (staged.empty() && mapped.empty()) { return; } + for (const Download& download : staged) { + GpuWaitStats::AddDownloadBytes(download.copy.size); + } + for (const Download& download : mapped) { + GpuWaitStats::AddDownloadBytes(download.copy.size); + } const Common::RangeSet cpu_written = PendingCpuWriteRanges(); if (staged.empty()) { if constexpr (has_mapped) { @@ -482,6 +488,7 @@ bool BufferCache

::DMACopy(GPUVAddr src_address, GPUVAddr dest_address, u64 am const bool has_new_downloads = tmp_intervals.size() != 0; for (const auto& pair : tmp_intervals) { gpu_modified_ranges.Add(pair.first, pair.second); + GpuWaitStats::NoteGpuWrite(pair.first, pair.second, GpuWaitStats::WriteSite::DmaCopy); if (Tegra::g_nx2_ampere.load(std::memory_order_relaxed)) { Tegra::Nx2MarkGpuWritten(pair.first, pair.second); Nx2NoteListWrite(buffer_b, pair.first, pair.second); @@ -556,7 +563,8 @@ std::pair BufferCache

::ObtainCPUBuffer( switch (post_op) { case ObtainBufferOperation::MarkAsWritten: - MarkWrittenBuffer(buffer_id, device_addr, size); + MarkWrittenBuffer(buffer_id, device_addr, size, false, + GpuWaitStats::WriteSite::ImageToBuffer); buffer.MarkUsage(buffer.Offset(device_addr), size); break; case ObtainBufferOperation::DiscardWrite: { @@ -1564,7 +1572,8 @@ bool BufferCache

::BindMultiRangeStorage(const Binding& binding, bool is_writt const u32 offset = buffer.Offset(segment.device_addr); buffer.MarkUsage(offset, segment.size); if (is_written) { - MarkWrittenBuffer(segment.buffer_id, segment.device_addr, segment.size); + MarkWrittenBuffer(segment.buffer_id, segment.device_addr, segment.size, false, + GpuWaitStats::WriteSite::MultiRangeStorage); } runtime.PushMultiRangeSource(buffer, offset, segment.size); } @@ -1592,7 +1601,8 @@ void BufferCache

::BindHostGraphicsStorageBuffers(size_t stage) { buffer.MarkUsage(offset, size); if (is_written) { - MarkWrittenBuffer(binding.buffer_id, binding.device_addr, size); + MarkWrittenBuffer(binding.buffer_id, binding.device_addr, size, false, + GpuWaitStats::WriteSite::GraphicsStorage); } if constexpr (NEEDS_BIND_STORAGE_INDEX) { @@ -1614,7 +1624,8 @@ void BufferCache

::BindHostGraphicsTextureBuffers(size_t stage) { const bool is_written = ((channel_state->written_texture_buffers[stage] >> index) & 1) != 0; if (is_written) { - MarkWrittenBuffer(binding.buffer_id, binding.device_addr, size); + MarkWrittenBuffer(binding.buffer_id, binding.device_addr, size, false, + GpuWaitStats::WriteSite::GraphicsTexel); } const u32 offset = buffer.Offset(binding.device_addr); @@ -1652,7 +1663,8 @@ void BufferCache

::BindHostTransformFeedbackBuffers() { TouchBuffer(buffer, binding.buffer_id); size = binding.size; SynchronizeBuffer(buffer, binding.device_addr, size); - MarkWrittenBuffer(binding.buffer_id, binding.device_addr, size); + MarkWrittenBuffer(binding.buffer_id, binding.device_addr, size, false, + GpuWaitStats::WriteSite::TransformFeedback); offset = buffer.Offset(binding.device_addr); buffer.MarkUsage(offset, size); host_buffer = &buffer; @@ -1739,7 +1751,8 @@ void BufferCache

::BindHostComputeStorageBuffers() { buffer.MarkUsage(offset, size); if (is_written) { - MarkWrittenBuffer(binding.buffer_id, binding.device_addr, size, true); + MarkWrittenBuffer(binding.buffer_id, binding.device_addr, size, true, + GpuWaitStats::WriteSite::ComputeStorage); } if constexpr (NEEDS_BIND_STORAGE_INDEX) { @@ -1762,7 +1775,8 @@ void BufferCache

::BindHostComputeTextureBuffers() { const bool is_written = ((channel_state->written_compute_texture_buffers >> index) & 1) != 0; if (is_written) { - MarkWrittenBuffer(binding.buffer_id, binding.device_addr, size); + MarkWrittenBuffer(binding.buffer_id, binding.device_addr, size, false, + GpuWaitStats::WriteSite::ComputeTexel); } const u32 offset = buffer.Offset(binding.device_addr); @@ -2041,7 +2055,9 @@ void BufferCache

::UpdateComputeTextureBuffers() { } template -void BufferCache

::MarkWrittenBuffer(BufferId buffer_id, DAddr device_addr, u32 size, bool needs_sync) { +void BufferCache

::MarkWrittenBuffer(BufferId buffer_id, DAddr device_addr, u32 size, + bool needs_sync, GpuWaitStats::WriteSite site) { + GpuWaitStats::NoteGpuWrite(device_addr, size, site); if constexpr (!IS_OPENGL) { if (needs_sync) { Buffer& buffer = slot_buffers[buffer_id]; @@ -2727,6 +2743,7 @@ void BufferCache

::DownloadBuffersMemory(std::span buffer_ids) if (total_size_bytes == 0) { return; } + GpuWaitStats::AddDownloadBytes(total_size_bytes); auto download_staging = runtime.DownloadStagingBuffer(total_size_bytes); for (Download& download : downloads) { download.copy.dst_offset += download_staging.offset; @@ -2776,6 +2793,7 @@ void BufferCache

::DownloadBufferMemory(Buffer& buffer, DAddr device_addr, u64 if (total_size_bytes == 0) { return; } + GpuWaitStats::AddDownloadBytes(total_size_bytes); const Common::RangeSet cpu_written = PendingCpuWriteRanges(); if constexpr (requires { buffer.Mapped(); }) { diff --git a/src/video_core/buffer_cache/buffer_cache_base.h b/src/video_core/buffer_cache/buffer_cache_base.h index 2afb361c40..7a1b7e57d5 100644 --- a/src/video_core/buffer_cache/buffer_cache_base.h +++ b/src/video_core/buffer_cache/buffer_cache_base.h @@ -43,6 +43,7 @@ #include "video_core/engines/maxwell_3d.h" #include "video_core/engines/kepler_compute.h" #include "video_core/engines/maxwell_3d.h" +#include "video_core/gpu_wait_stats.h" #include "video_core/memory_manager.h" #include "video_core/nx2_download_trace.h" #include "video_core/surface.h" @@ -553,7 +554,9 @@ class BufferCache : public VideoCommon::ChannelSetupCachesReadRasterizer()->GetFlushArea(addr, size); + const u64 request = Stats::Now(); + Stats::AddAreaCheck(core, request - check_start); if (raster_area.preemtive) { return raster_area; } @@ -252,13 +261,24 @@ struct GPU::Impl { raster_area.start_address, raster_area.end_address, reads.load(std::memory_order_relaxed)); } - const u64 fence = RequestSyncOperation([this, raster_area]() { + // Filled on the GPU thread while this thread waits for it. + Stats::FlushTiming timing{}; + const u64 fence = RequestSyncOperation([this, raster_area, &timing]() { + timing.start = Stats::Now(); + const Stats::HostWaits host_before = Stats::t_host_waits; + const u64 bytes_before = Stats::t_download_bytes; // Apply collected guest writes before downloading GPU data over guest memory. InvalidateGPUCache(); + timing.applied = Stats::Now(); renderer->ReadRasterizer()->FlushRegion(raster_area.start_address, raster_area.end_address - raster_area.start_address); + timing.host_ns = Stats::t_host_waits.ns - host_before.ns; + timing.finishes = Stats::t_host_waits.count - host_before.count; + timing.bytes = Stats::t_download_bytes - bytes_before; + timing.end = Stats::Now(); }); gpu_thread.TickGPU(is_async); WaitForSyncOperation(fence); + Stats::RecordReactive(raster_area, core, request, timing, Stats::Now()); return raster_area; } @@ -267,12 +287,15 @@ struct GPU::Impl { if (preserve_gpu_writes) { VideoCore::RasterizerInterface* rasterizer = renderer->ReadRasterizer(); if (!rasterizer->TryCacheInvalidation(addr, size)) { + const u64 request = VideoCommon::GpuWaitStats::Now(); const u64 fence = RequestSyncOperation([rasterizer, addr, size] { rasterizer->FlushRegion(addr, size, VideoCommon::CacheType::BufferCache); rasterizer->OnCacheInvalidation(addr, size); }); gpu_thread.TickGPU(is_async); WaitForSyncOperation(fence); + VideoCommon::GpuWaitStats::RecordSync(VideoCommon::GpuWaitStats::Site::Invalidate, + VideoCommon::GpuWaitStats::Now() - request); } return; } diff --git a/src/video_core/gpu_split.cpp b/src/video_core/gpu_split.cpp index c13781131a..447ea6c4a2 100644 --- a/src/video_core/gpu_split.cpp +++ b/src/video_core/gpu_split.cpp @@ -25,6 +25,7 @@ #include "video_core/engines/engine_interface.h" #include "video_core/engines/maxwell_3d.h" #include "video_core/gpu_split.h" +#include "video_core/gpu_wait_stats.h" #include "video_core/memory_manager.h" #include "video_core/nx2.h" #include "video_core/nx2_profile.h" @@ -425,6 +426,7 @@ void GpuSplit::BackLoop(std::stop_token stop_token) { u64 available = published.load(std::memory_order_acquire); if (available == read_position) { Nx2Profile::Scope scope{Nx2Profile::Zone::SplitWait}; + const GpuWaitStats::IdleScope idle{GpuWaitStats::g_back_idle_ns}; // The front usually sends more within microseconds: spin briefly before sleeping. for (u32 spins = 0; spins < 8192 && available == read_position && !hook_requested.load(std::memory_order_relaxed); diff --git a/src/video_core/gpu_thread.cpp b/src/video_core/gpu_thread.cpp index 8f25448101..baa6f64513 100644 --- a/src/video_core/gpu_thread.cpp +++ b/src/video_core/gpu_thread.cpp @@ -23,6 +23,7 @@ #include "video_core/gpu.h" #include "video_core/gpu_thread.h" #include "video_core/gpu_split.h" +#include "video_core/gpu_wait_stats.h" #include "video_core/nx2_profile.h" #include "video_core/host1x/host1x.h" #include "video_core/renderer_base.h" @@ -72,9 +73,11 @@ void ThreadManager::StartThread(VideoCore::RendererBase& renderer, Core::Fronten auto last_progress = std::chrono::steady_clock::now(); namespace Profile = VideoCommon::Nx2Profile; auto& split = VideoCommon::GpuSplit::Get(); + namespace WaitStats = VideoCommon::GpuWaitStats; while (!stop_token.stop_requested()) { Profile::Report(); split.MaybeStart(system); + WaitStats::MaybeReport(split.Active()); if (scheduler.HasPendingChannels()) { // Suspended channels wait on semaphores the CPU releases: poll them meanwhile. { @@ -88,6 +91,7 @@ void ThreadManager::StartThread(VideoCore::RendererBase& renderer, Core::Fronten // present waited ~17 ms a frame in Kokiri village, with 50 us ~1 ms. A channel // still waiting after 2 s (menus, loading) goes back to 1 ms sleeps. Profile::Scope scope{Profile::Zone::PendingSleep}; + const WaitStats::IdleScope idle{WaitStats::g_front_poll_ns}; const auto waited = std::chrono::steady_clock::now() - last_progress; if (waited < std::chrono::microseconds{500}) { std::this_thread::yield(); @@ -100,6 +104,7 @@ void ThreadManager::StartThread(VideoCore::RendererBase& renderer, Core::Fronten } } else { Profile::Scope scope{Profile::Zone::Idle}; + const WaitStats::IdleScope idle{WaitStats::g_front_idle_ns}; state.queue.PopWait(next, stop_token); } if (stop_token.stop_requested()) { diff --git a/src/video_core/gpu_wait_stats.h b/src/video_core/gpu_wait_stats.h new file mode 100644 index 0000000000..70656ffb56 --- /dev/null +++ b/src/video_core/gpu_wait_stats.h @@ -0,0 +1,600 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later + +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "common/common_types.h" +#include "common/cpu_features.h" +#include "common/logging.h" +#include "core/hardware_properties.h" +#include "video_core/rasterizer_download_area.h" + +// TEST: how long guest threads wait for the GPU. Always on: one line every 10 s ("GPU waits"). +// +// A guest read of GPU-written memory that GetFlushArea can't serve (reactive flushing) queues a +// flush on the GPU thread and waits for it. Each wait is timed from the request to the wake-up and +// split into: queue (until the GPU thread starts the flush), flush (applying the collected CPU +// writes, then the downloads; "host GPU" is the time in Vulkan's Scheduler::Finish, waiting for +// the host GPU) and wake (until the waiting thread runs again). Also counted: guest reads and +// writes that take the rasterizer path, by core, with the time of their cache lookups +// (GetFlushArea, OnCPUWrite: both take the cache locks the GPU thread holds while it draws); the +// other sync operations guest threads wait for; the GPU thread's (and the split's back thread's) +// idle time; every Scheduler::Finish and every wait for pending query results (FlushQueryResults); +// the time between presents; and, for the pages that waited longest, the GPU writes that marked +// them. +namespace VideoCommon::GpuWaitStats { + +/// One slot per guest core, the last one for every other thread. +inline constexpr size_t CoreSlots = Core::Hardware::NUM_CPU_CORES + 1; +/// Pages whose GPU writers are recorded (the longest waits of the last window). +inline constexpr size_t WatchSlots = 4; +/// Distinct flush areas totaled per window. +inline constexpr size_t MaxAreas = 64; +inline constexpr u64 WindowNs = 10'000'000'000ULL; +inline constexpr u64 PageSize = 0x1000; + +/// Steady nanoseconds: the time stamp counter where the wall clock reads it directly, else +/// steady_clock (as Nx2Profile::Now). Cheap enough for the per-read counters. +inline u64 Now() { + return static_cast(Common::g_wall_clock.GetTimeNS().count()); +} + +inline void AtomicMax(std::atomic& value, u64 candidate) { + u64 current = value.load(std::memory_order_relaxed); + while (candidate > current && + !value.compare_exchange_weak(current, candidate, std::memory_order_relaxed)) { + } +} + +/// Increments a counter only one thread writes (no locked instruction). +inline void BumpOwned(std::atomic& value, u64 amount = 1) { + value.store(value.load(std::memory_order_relaxed) + amount, std::memory_order_relaxed); +} + +/// What marked buffer memory GPU-written (BufferCache::MarkWrittenBuffer and DMACopy). +enum class WriteSite : u8 { + Other, + GraphicsStorage, + GraphicsTexel, + ComputeStorage, + ComputeTexel, + MultiRangeStorage, + TransformFeedback, + ImageToBuffer, ///< ObtainBuffer with MarkAsWritten: DMA copies of images into buffers + DmaCopy, + Count, +}; + +inline constexpr std::array(WriteSite::Count)> write_site_names{ + "other", + "graphics storage", + "graphics texel buffer", + "compute storage", + "compute texel buffer", + "multi-range storage", + "transform feedback", + "image to buffer", + "DMA copy", +}; + +inline constexpr std::array source_names{"other", "texture", "buffer"}; + +/// Sync operations other than reactive flushes that a guest thread waits for. +enum class Site : u8 { + Invalidate, ///< A CPU write over GPU-written buffer memory (InvalidateRegion, preserved) + Idle, ///< GPU::WaitForIdle + Count, +}; + +/// Filled by the GPU thread while the guest thread waits for it. +struct FlushTiming { + u64 start{}; ///< The GPU thread started the flush + u64 applied{}; ///< It applied the CPU writes collected so far + u64 end{}; + u64 host_ns{}; ///< Of the flush: time in Scheduler::Finish + u64 finishes{}; + u64 bytes{}; ///< Downloaded by the flush +}; + +struct HostWaits { + u64 ns{}; + u64 count{}; +}; + +/// This thread's Scheduler::Finish totals, so a flush can tell its own host waits. +inline thread_local HostWaits t_host_waits{}; +inline std::atomic g_host_wait_ns{}; +inline std::atomic g_host_waits{}; + +/// Scheduler::Finish took `ns` (submit, then wait for the host GPU). +inline void AddHostWait(u64 ns) { + t_host_waits.ns += ns; + ++t_host_waits.count; + g_host_wait_ns.fetch_add(ns, std::memory_order_relaxed); + g_host_waits.fetch_add(1, std::memory_order_relaxed); +} + +/// Bytes this thread's buffer and texture cache downloads copied back (read as a difference by a +/// reactive flush, like t_host_waits). +inline thread_local u64 t_download_bytes{}; + +inline void AddDownloadBytes(u64 bytes) { + t_download_bytes += bytes; +} + +/// RasterizerVulkan::FlushQueryResults waited `ns` for pending query results (macro DMA reads of +/// report regions: the GPU thread waits for the host GPU to reach a new fence). +inline std::atomic g_query_wait_ns{}; +inline std::atomic g_query_waits{}; + +inline void AddQueryWait(u64 ns) { + g_query_wait_ns.fetch_add(ns, std::memory_order_relaxed); + g_query_waits.fetch_add(1, std::memory_order_relaxed); +} + +struct alignas(64) CoreCounter { + std::atomic value{}; +}; +/// Guest reads that took the rasterizer path (Memory HandleRasterizerDownload), by core. +inline std::array g_rasterizer_reads{}; + +inline void CountRasterizerRead(size_t core) { + if (core < CoreSlots - 1) { + BumpOwned(g_rasterizer_reads[core].value); + } else { + g_rasterizer_reads[CoreSlots - 1].value.fetch_add(1, std::memory_order_relaxed); + } +} + +/// Guest writes that took the rasterizer path (Memory HandleRasterizerWrite), and their +/// GPU::OnCPUWrite calls (the buffer and texture cache locks) with time, by core. As in Memory, +/// the last core's slot also takes the other threads, under Memory's guard: one writer at a time. +inline std::array g_rasterizer_writes{}; +inline std::array g_write_checks{}; +inline std::array g_write_check_ns{}; + +inline void CountRasterizerWrite(size_t core) { + if (core < g_rasterizer_writes.size()) { + BumpOwned(g_rasterizer_writes[core].value); + } +} + +inline void AddWriteCheck(size_t core, u64 ns) { + if (core < g_write_checks.size()) { + BumpOwned(g_write_checks[core].value); + BumpOwned(g_write_check_ns[core].value, ns); + } +} + +/// GPU::OnCPURead's GetFlushArea calls (served or not) and their time (two cache locks), by core +/// (GetCurrentHostThreadID; the last slot takes every other thread). +inline std::array g_area_checks{}; +inline std::array g_area_check_ns{}; + +inline void AddAreaCheck(size_t core, u64 ns) { + if (core < CoreSlots - 1) { + BumpOwned(g_area_checks[core].value); + BumpOwned(g_area_check_ns[core].value, ns); + } else { + g_area_checks[CoreSlots - 1].value.fetch_add(1, std::memory_order_relaxed); + g_area_check_ns[CoreSlots - 1].value.fetch_add(ns, std::memory_order_relaxed); + } +} + +/// Idle time of the GPU thread (waiting for commands; polling suspended channels) and of the +/// split's back thread (waiting for the front). +inline std::atomic g_front_idle_ns{}; +inline std::atomic g_front_poll_ns{}; +inline std::atomic g_back_idle_ns{}; + +class IdleScope { +public: + explicit IdleScope(std::atomic& sink_) : sink{sink_}, start{Now()} {} + ~IdleScope() { + sink.fetch_add(Now() - start, std::memory_order_relaxed); + } + IdleScope(const IdleScope&) = delete; + IdleScope& operator=(const IdleScope&) = delete; + +private: + std::atomic& sink; + u64 start; +}; + +struct SyncTotals { + std::atomic count{}; + std::atomic ns{}; +}; +inline std::array(Site::Count)> g_syncs{}; + +inline void RecordSync(Site site, u64 ns) { + auto& totals = g_syncs[static_cast(site)]; + totals.count.fetch_add(1, std::memory_order_relaxed); + totals.ns.fetch_add(ns, std::memory_order_relaxed); +} + +/// Time between presents (nx2:vi present). +inline std::atomic g_last_present{}; +inline std::atomic g_presents{}; +inline std::atomic g_present_ns{}; +inline std::atomic g_present_max_ns{}; + +inline void NotePresent() { + const u64 now = Now(); + const u64 last = g_last_present.exchange(now, std::memory_order_relaxed); + if (last == 0 || now <= last) { + return; + } + g_presents.fetch_add(1, std::memory_order_relaxed); + g_present_ns.fetch_add(now - last, std::memory_order_relaxed); + AtomicMax(g_present_max_ns, now - last); +} + +/// GPU writes over a watched page: written by the buffer cache under its lock, read by the report. +struct alignas(64) Watch { + std::atomic page{}; ///< 0: unused + std::atomic writes{}; + std::atomic last_start{}; + std::atomic last_size{}; + std::atomic largest{}; + std::atomic last_site{}; +}; +inline std::array g_watches{}; +inline std::atomic g_watching{}; + +/// The buffer cache marked [addr, addr + size) GPU-written. +inline void NoteGpuWrite(u64 addr, u64 size, WriteSite site) { + if (g_watching.load(std::memory_order_relaxed) == 0) { + return; + } + for (Watch& watch : g_watches) { + const u64 page = watch.page.load(std::memory_order_relaxed); + if (page == 0 || page >= addr + size || addr >= page + PageSize) { + continue; + } + BumpOwned(watch.writes); + watch.last_start.store(addr, std::memory_order_relaxed); + watch.last_size.store(size, std::memory_order_relaxed); + watch.last_site.store(static_cast(site), std::memory_order_relaxed); + if (size > watch.largest.load(std::memory_order_relaxed)) { + watch.largest.store(size, std::memory_order_relaxed); + } + } +} + +struct AreaTotals { + u64 start{}; + u64 size{}; + u64 count{}; + u64 wait_ns{}; + u64 max_ns{}; + VideoCore::DownloadAreaSource source{}; +}; + +struct ReactiveTotals { + u64 count{}; + u64 wait_ns{}; + u64 max_ns{}; + u64 queue_ns{}; + u64 apply_ns{}; + u64 flush_ns{}; + u64 host_ns{}; + u64 finishes{}; + u64 without_finish{}; + u64 bytes{}; + u64 max_bytes{}; + u64 wake_ns{}; + u64 other_areas{}; ///< Flushes of areas past MaxAreas + std::array count_by_source{}; + std::array ns_by_source{}; + std::array ns_by_core{}; + std::vector areas; +}; +inline std::mutex g_reactive_mutex; +inline ReactiveTotals g_reactive; + +/// A guest thread (`core`, GetCurrentHostThreadID) waited from `request` to `woke` for the flush +/// of `area`, which the GPU thread ran as `timing` says. +inline void RecordReactive(const VideoCore::RasterizerDownloadArea& area, size_t core, u64 request, + const FlushTiming& timing, u64 woke) { + const u64 wait = woke - request; + const size_t source = (std::min)(static_cast(area.source), 2); + std::scoped_lock lock{g_reactive_mutex}; + ReactiveTotals& totals = g_reactive; + ++totals.count; + totals.wait_ns += wait; + totals.max_ns = (std::max)(totals.max_ns, wait); + if (timing.start >= request && timing.end >= timing.start && woke >= timing.end) { + totals.queue_ns += timing.start - request; + totals.apply_ns += timing.applied - timing.start; + totals.flush_ns += timing.end - timing.start; + totals.wake_ns += woke - timing.end; + } + totals.host_ns += timing.host_ns; + totals.finishes += timing.finishes; + totals.without_finish += timing.finishes == 0 ? 1 : 0; + totals.bytes += timing.bytes; + totals.max_bytes = (std::max)(totals.max_bytes, timing.bytes); + ++totals.count_by_source[source]; + totals.ns_by_source[source] += wait; + totals.ns_by_core[(std::min)(core, CoreSlots - 1)] += wait; + const auto it = + std::find_if(totals.areas.begin(), totals.areas.end(), [&area](const AreaTotals& entry) { + return entry.start == area.start_address; + }); + AreaTotals* entry = it != totals.areas.end() ? &*it : nullptr; + if (entry == nullptr) { + if (totals.areas.size() >= MaxAreas) { + ++totals.other_areas; + return; + } + entry = &totals.areas.emplace_back(); + entry->start = area.start_address; + entry->source = area.source; + } + entry->size = (std::max)(entry->size, area.end_address - area.start_address); + ++entry->count; + entry->wait_ns += wait; + entry->max_ns = (std::max)(entry->max_ns, wait); +} + +/// Report state: the cumulative counters at the start of the window (GPU thread only). +struct ReportState { + u64 window_start{}; + std::array reads{}; + std::array writes{}; + std::array write_checks{}; + std::array write_check_ns{}; + u64 area_checks{}; + u64 area_check_ns{}; + u64 front_idle{}; + u64 front_poll{}; + u64 back_idle{}; + u64 host_waits{}; + u64 host_wait_ns{}; + u64 query_waits{}; + u64 query_wait_ns{}; + std::array(Site::Count)> sync_count{}; + std::array(Site::Count)> sync_ns{}; + u64 presents{}; + u64 present_ns{}; +}; +inline ReportState g_report; + +/// Takes the cumulative counters' current values; returns the old ones. +inline ReportState Snapshot(u64 now) { + ReportState state; + state.window_start = now; + for (size_t i = 0; i < CoreSlots; ++i) { + state.reads[i] = g_rasterizer_reads[i].value.load(std::memory_order_relaxed); + } + for (size_t i = 0; i < state.writes.size(); ++i) { + state.writes[i] = g_rasterizer_writes[i].value.load(std::memory_order_relaxed); + state.write_checks[i] = g_write_checks[i].value.load(std::memory_order_relaxed); + state.write_check_ns[i] = g_write_check_ns[i].value.load(std::memory_order_relaxed); + } + for (size_t i = 0; i < CoreSlots; ++i) { + state.area_checks += g_area_checks[i].value.load(std::memory_order_relaxed); + state.area_check_ns += g_area_check_ns[i].value.load(std::memory_order_relaxed); + } + state.front_idle = g_front_idle_ns.load(std::memory_order_relaxed); + state.front_poll = g_front_poll_ns.load(std::memory_order_relaxed); + state.back_idle = g_back_idle_ns.load(std::memory_order_relaxed); + state.host_waits = g_host_waits.load(std::memory_order_relaxed); + state.host_wait_ns = g_host_wait_ns.load(std::memory_order_relaxed); + state.query_waits = g_query_waits.load(std::memory_order_relaxed); + state.query_wait_ns = g_query_wait_ns.load(std::memory_order_relaxed); + for (size_t i = 0; i < state.sync_count.size(); ++i) { + state.sync_count[i] = g_syncs[i].count.load(std::memory_order_relaxed); + state.sync_ns[i] = g_syncs[i].ns.load(std::memory_order_relaxed); + } + state.presents = g_presents.load(std::memory_order_relaxed); + state.present_ns = g_present_ns.load(std::memory_order_relaxed); + return std::exchange(g_report, state); +} + +inline double Ms(u64 ns) { + return static_cast(ns) / 1e6; +} + +/// Logs the window's totals every 10 s. Called by the GPU thread between commands, so its own +/// idle time is complete; `split_active` says whether the back thread's counts apply. +inline void MaybeReport(bool split_active) { + const u64 now = Now(); + if (g_report.window_start == 0) { + (void)Snapshot(now); + return; + } + if (now - g_report.window_start < WindowNs) { + return; + } + const ReportState old = Snapshot(now); + const ReportState& cur = g_report; + const u64 window = now - old.window_start; + ReactiveTotals reactive; + { + std::scoped_lock lock{g_reactive_mutex}; + reactive = std::move(g_reactive); + g_reactive = {}; + } + const u64 presents = cur.presents - old.presents; + const u64 present_ns = cur.present_ns - old.present_ns; + const u64 present_max = g_present_max_ns.exchange(0, std::memory_order_relaxed); + + std::string line = fmt::format("GPU waits in {:.1f} s:", static_cast(window) / 1e9); + if (presents != 0) { + line += fmt::format(" {} frames ({:.1f}/s, {:.2f} ms each, max {:.2f} ms) |", presents, + static_cast(presents) * 1e9 / static_cast(window), + Ms(present_ns) / static_cast(presents), Ms(present_max)); + } else { + line += " no presents |"; + } + line += + fmt::format(" reactive flushes {} waited {:.1f} ms ({:.1f}% of the time", reactive.count, + Ms(reactive.wait_ns), + static_cast(reactive.wait_ns) * 100.0 / static_cast(window)); + if (presents != 0) { + line += fmt::format(", {:.2f} ms and {:.2f} flushes a frame", + Ms(reactive.wait_ns) / static_cast(presents), + static_cast(reactive.count) / static_cast(presents)); + } + line += fmt::format("), max {:.2f} ms", Ms(reactive.max_ns)); + // Ends a list of ", "-separated items: drops the separator after the last one. + const auto end_list = [&line] { + if (line.size() >= 1 && line.back() == ',') { + line.pop_back(); + } + }; + if (reactive.count != 0) { + line += "; by cache:"; + for (size_t i = 0; i < source_names.size(); ++i) { + if (reactive.count_by_source[i] != 0) { + line += fmt::format(" {} {} {:.1f} ms,", source_names[i], + reactive.count_by_source[i], Ms(reactive.ns_by_source[i])); + } + } + end_list(); + line += fmt::format(" | queue {:.1f} ms, flush {:.1f} ms (CPU writes {:.1f}, host GPU " + "{:.1f} in {} finishes, {} flushes without one; downloaded {} KiB, " + "at most {} KiB), wake {:.1f} ms | by core:", + Ms(reactive.queue_ns), Ms(reactive.flush_ns), Ms(reactive.apply_ns), + Ms(reactive.host_ns), reactive.finishes, reactive.without_finish, + reactive.bytes / 1024, reactive.max_bytes / 1024, Ms(reactive.wake_ns)); + for (size_t i = 0; i < CoreSlots; ++i) { + if (reactive.ns_by_core[i] == 0) { + continue; + } + line += i + 1 < CoreSlots + ? fmt::format(" {} {:.1f} ms,", i, Ms(reactive.ns_by_core[i])) + : fmt::format(" other {:.1f} ms,", Ms(reactive.ns_by_core[i])); + } + end_list(); + } + // The longest waits, with the GPU writes seen over them (watched since the last report). + std::sort(reactive.areas.begin(), reactive.areas.end(), + [](const AreaTotals& a, const AreaTotals& b) { return a.wait_ns > b.wait_ns; }); + line += " | areas:"; + const size_t shown = (std::min)(reactive.areas.size(), WatchSlots); + if (shown == 0) { + line += " none"; + } + for (size_t i = 0; i < shown; ++i) { + const AreaTotals& area = reactive.areas[i]; + line += fmt::format(" {:#x}+{:#x} {} {}x {:.1f} ms (max {:.2f})", area.start, area.size, + source_names[(std::min)(static_cast(area.source), 2)], + area.count, Ms(area.wait_ns), Ms(area.max_ns)); + for (const Watch& watch : g_watches) { + if (watch.page.load(std::memory_order_relaxed) != area.start) { + continue; + } + const u64 writes = watch.writes.load(std::memory_order_relaxed); + if (writes == 0) { + line += " no GPU writes seen"; + } else { + const u32 site = (std::min)(watch.last_site.load(std::memory_order_relaxed), + static_cast(WriteSite::Count) - 1); + line += fmt::format(" GPU-written {}x, last by {} {:#x}+{:#x} (largest {:#x})", + writes, write_site_names[site], + watch.last_start.load(std::memory_order_relaxed), + watch.last_size.load(std::memory_order_relaxed), + watch.largest.load(std::memory_order_relaxed)); + } + } + line += ','; + } + if (reactive.areas.size() > shown || reactive.other_areas != 0) { + line += + fmt::format(" {} more areas,", reactive.areas.size() - shown + reactive.other_areas); + } + end_list(); + // Watch the longest waits' buffer pages during the next window. + g_watching.store(0, std::memory_order_relaxed); + u32 watching = 0; + for (size_t i = 0; i < WatchSlots; ++i) { + Watch& watch = g_watches[i]; + u64 page = 0; + if (i < shown && reactive.areas[i].source == VideoCore::DownloadAreaSource::BufferCache) { + page = reactive.areas[i].start; + } + watch.page.store(0, std::memory_order_relaxed); + watch.writes.store(0, std::memory_order_relaxed); + watch.last_start.store(0, std::memory_order_relaxed); + watch.last_size.store(0, std::memory_order_relaxed); + watch.largest.store(0, std::memory_order_relaxed); + watch.last_site.store(0, std::memory_order_relaxed); + watch.page.store(page, std::memory_order_relaxed); + watching += page != 0 ? 1 : 0; + } + g_watching.store(watching, std::memory_order_relaxed); + + line += " | other waits:"; + constexpr std::array(Site::Count)> site_names{"invalidate", + "idle"}; + for (size_t i = 0; i < site_names.size(); ++i) { + line += + fmt::format(" {} {}x {:.1f} ms,", site_names[i], cur.sync_count[i] - old.sync_count[i], + Ms(cur.sync_ns[i] - old.sync_ns[i])); + } + end_list(); + line += " | rasterizer-path reads by core:"; + u64 all_reads = 0; + for (size_t i = 0; i < CoreSlots; ++i) { + const u64 reads = cur.reads[i] - old.reads[i]; + all_reads += reads; + if (reads != 0) { + line += i + 1 < CoreSlots ? fmt::format(" {} {},", i, reads) + : fmt::format(" other {},", reads); + } + } + if (all_reads == 0) { + line += " none"; + } + end_list(); + line += fmt::format("; flush area checks {} in {:.1f} ms", cur.area_checks - old.area_checks, + Ms(cur.area_check_ns - old.area_check_ns)); + line += " | rasterizer-path writes by core (OnCPUWrite calls, time):"; + u64 all_writes = 0; + for (size_t i = 0; i < cur.writes.size(); ++i) { + const u64 writes = cur.writes[i] - old.writes[i]; + all_writes += writes; + if (writes != 0) { + line += fmt::format(" {} {} ({}, {:.1f} ms),", i, writes, + cur.write_checks[i] - old.write_checks[i], + Ms(cur.write_check_ns[i] - old.write_check_ns[i])); + } + } + if (all_writes == 0) { + line += " none"; + } + end_list(); + const auto percent = [window](u64 ns) { + return static_cast(ns) * 100.0 / static_cast(window); + }; + const u64 front_idle = cur.front_idle - old.front_idle; + const u64 front_poll = cur.front_poll - old.front_poll; + line += fmt::format(" | GPU thread busy {:.1f}% (idle {:.1f}%, polling suspended channels " + "{:.1f}%)", + 100.0 - percent(front_idle + front_poll), percent(front_idle), + percent(front_poll)); + if (split_active) { + line += fmt::format(", back thread busy {:.1f}%", + 100.0 - percent(cur.back_idle - old.back_idle)); + } + line += + fmt::format(" | Scheduler::Finish {}x {:.1f} ms; query result waits {}x {:.1f} ms", + cur.host_waits - old.host_waits, Ms(cur.host_wait_ns - old.host_wait_ns), + cur.query_waits - old.query_waits, Ms(cur.query_wait_ns - old.query_wait_ns)); + LOG_INFO(HW_GPU, "{}", line); +} + +} // namespace VideoCommon::GpuWaitStats diff --git a/src/video_core/rasterizer_download_area.h b/src/video_core/rasterizer_download_area.h index d28826043d..c5d6cf17ee 100644 --- a/src/video_core/rasterizer_download_area.h +++ b/src/video_core/rasterizer_download_area.h @@ -7,10 +7,18 @@ namespace VideoCore { +/// The cache that reported a flush area (GetFlushArea), for statistics (GpuWaitStats). +enum class DownloadAreaSource : u8 { + None, + TextureCache, + BufferCache, +}; + struct RasterizerDownloadArea { VAddr start_address; VAddr end_address; bool preemtive; + DownloadAreaSource source{}; }; } // namespace VideoCore diff --git a/src/video_core/renderer_vulkan/vk_rasterizer.cpp b/src/video_core/renderer_vulkan/vk_rasterizer.cpp index 7bbdc0c0cd..f5bfcafaa8 100644 --- a/src/video_core/renderer_vulkan/vk_rasterizer.cpp +++ b/src/video_core/renderer_vulkan/vk_rasterizer.cpp @@ -35,6 +35,7 @@ #include "common/settings.h" #include "video_core/buffer_cache/buffer_cache.h" #include "video_core/gpu_logging/gpu_logging.h" +#include "video_core/gpu_wait_stats.h" #include "video_core/control/channel_state.h" #include "video_core/native_query_trace.h" #include "video_core/native_indirect_reset.h" @@ -2029,6 +2030,7 @@ VideoCore::RasterizerDownloadArea RasterizerVulkan::GetFlushArea(DAddr addr, u64 std::scoped_lock lock{texture_cache.mutex}; auto area = texture_cache.GetFlushArea(addr, size); if (area) { + area->source = VideoCore::DownloadAreaSource::TextureCache; return *area; } } @@ -2036,6 +2038,7 @@ VideoCore::RasterizerDownloadArea RasterizerVulkan::GetFlushArea(DAddr addr, u64 std::scoped_lock lock{buffer_cache.mutex}; auto area = buffer_cache.GetFlushArea(addr, size); if (area) { + area->source = VideoCore::DownloadAreaSource::BufferCache; return *area; } } @@ -2224,6 +2227,7 @@ void RasterizerVulkan::FlushQueryResults(DAddr addr, u64 size) { // Below High GPU accuracy a fence runs its own callback at once, so wait for an operation // queued behind the pending report writes instead. On the split's back the fence's // submission may be deferred, and the fencing thread would wait for this thread to submit it. + const u64 wait_start = VideoCommon::GpuWaitStats::Now(); std::mutex wait_mutex; std::condition_variable wait_cv; bool written{}; @@ -2236,6 +2240,7 @@ void RasterizerVulkan::FlushQueryResults(DAddr addr, u64 size) { scheduler.FlushDeferred(); std::unique_lock lock{wait_mutex}; wait_cv.wait(lock, [&written] { return written; }); + VideoCommon::GpuWaitStats::AddQueryWait(VideoCommon::GpuWaitStats::Now() - wait_start); } void RasterizerVulkan::FlushAndInvalidateRegion(DAddr addr, u64 size, diff --git a/src/video_core/renderer_vulkan/vk_scheduler.cpp b/src/video_core/renderer_vulkan/vk_scheduler.cpp index 19bbcd9680..d112c36187 100644 --- a/src/video_core/renderer_vulkan/vk_scheduler.cpp +++ b/src/video_core/renderer_vulkan/vk_scheduler.cpp @@ -25,6 +25,7 @@ #include "common/dkb_cpu_trace.h" #include "common/thread.h" #include "video_core/gpu_logging/gpu_logging.h" +#include "video_core/gpu_wait_stats.h" #include "video_core/nx2_profile.h" #include "video_core/renderer_vulkan/vk_command_pool.h" #include "video_core/renderer_vulkan/vk_graphics_pipeline.h" @@ -248,9 +249,11 @@ void Scheduler::Finish(VkSemaphore signal_semaphore, VkSemaphore wait_semaphore, const u64 presubmit_tick = CurrentTick(); const bool profile = VideoCommon::Nx2Profile::Enabled(); const u64 start = profile ? VideoCommon::Nx2Profile::Now() : 0; + const u64 wait_start = VideoCommon::GpuWaitStats::Now(); SubmitExecution(signal_semaphore, wait_semaphore); Wait(presubmit_tick); AllocateNewContext(); + VideoCommon::GpuWaitStats::AddHostWait(VideoCommon::GpuWaitStats::Now() - wait_start); last_flush = std::chrono::steady_clock::now(); flush_pending = false; if (profile) { diff --git a/src/video_core/texture_cache/texture_cache.h b/src/video_core/texture_cache/texture_cache.h index 680986f5b2..aab81f1c3a 100644 --- a/src/video_core/texture_cache/texture_cache.h +++ b/src/video_core/texture_cache/texture_cache.h @@ -27,6 +27,7 @@ #include "video_core/control/channel_state.h" #include "video_core/dirty_flags.h" #include "video_core/engines/kepler_compute.h" +#include "video_core/gpu_wait_stats.h" #include "video_core/guest_memory.h" #include "video_core/host1x/gpu_device_memory_manager.h" #include "video_core/nx2.h" @@ -1067,6 +1068,7 @@ void TextureCache

::DownloadMemory(DAddr cpu_addr, size_t size) { auto copies = FullDownloadCopies(image.info); image.DownloadMemory(map, copies); pending_bytes += image.unswizzled_size_bytes; + GpuWaitStats::AddDownloadBytes(image.unswizzled_size_bytes); pending.push_back(Pending{image_id, map, std::move(copies)}); } flush_pending(); From d86bffda46dd1b473513a7b114a155d8a115c862 Mon Sep 17 00:00:00 2001 From: jpkto <255363096+jpkto@users.noreply.github.com> Date: Sat, 7 Nov 2026 19:32:10 +0100 Subject: [PATCH 010/105] video_core: serve a guest read's flush on the split's back after its current entry A guest thread that reads GPU-written memory queues a sync operation and waits for it. The operation ran at the next draw on the back, or when the tick sent through the GPU thread's queue got there: behind every command list the front still had, and then behind everything it had handed the back. In Ocarina's Hyrule Field the front was busy 99.9% of the time and these waits spent 1201.6 ms of 10 s in that queue, 1.17 ms a flush, before the flush itself started (test38). The guest synchronized with the GPU work it reads before it read, and with the split every signal it can see (syncpoints, semaphore releases) runs on the back after the work before it. So the back can run the flush at its next entry boundary. GPU::OnCPURead asks it to (GpuSplit::RequestTick): the back runs GPU::TickWork after the entry it is on, or wakes and runs it at once when it waits for work. Without the split nothing changes, the tick through the queue still runs it there. Both renderers also tick after compute dispatches, as they do after draws, so compute work doesn't hold a flush until the next draw. The split test checks that a request wakes an idle back, runs between two entries, and does nothing before the back runs. Co-Authored-By: Claude Opus 5.5 --- src/tests/video_core/maxwell_fast_method.cpp | 55 +++++++++++++++++++ src/video_core/gpu.cpp | 9 +++ src/video_core/gpu_split.cpp | 15 ++++- src/video_core/gpu_split.h | 30 ++++++++++ .../renderer_opengl/gl_rasterizer.cpp | 5 ++ .../renderer_vulkan/vk_rasterizer.cpp | 5 ++ 6 files changed, 116 insertions(+), 3 deletions(-) diff --git a/src/tests/video_core/maxwell_fast_method.cpp b/src/tests/video_core/maxwell_fast_method.cpp index 71f69ae48f..34c18ba097 100644 --- a/src/tests/video_core/maxwell_fast_method.cpp +++ b/src/tests/video_core/maxwell_fast_method.cpp @@ -3,6 +3,7 @@ #include #include +#include #include #include #include @@ -749,6 +750,60 @@ TEST_CASE("Split consumption does not complete deferred downstream callbacks", REQUIRE(output == std::array{1, 2, 3}); } +TEST_CASE("Split back runs a requested tick while it waits for work and between entries", + "[video_core][gpu_split_dependencies]") { + using Access = VideoCommon::GpuSplitTestAccess; + Fixture f; + f.system.Initialize(); + VideoCommon::GpuSplit split; + std::atomic ticks{}; + std::atomic entered{}; + std::atomic release{}; + u32 ticks_at_second{~0U}; + split.SetTickHook([](void* counter) { static_cast*>(counter)->fetch_add(1); }, + &ticks); + // Before the back runs, a request does nothing, nor is it kept for later. + split.RequestTick(); + Access::Start(split, f.system); + SCOPE_EXIT { + split.Stop(); + }; + // Runs before Stop: a failed check must not leave the back in the entry below. + SCOPE_EXIT { + release = true; + }; + const auto wait_until = [](const auto& done) { + const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds{5}; + while (!done() && std::chrono::steady_clock::now() < deadline) { + std::this_thread::sleep_for(std::chrono::milliseconds{1}); + } + return done(); + }; + REQUIRE(wait_until([&] { return split.BackRunning(); })); + // The back has nothing to do and goes to sleep: a request wakes it and runs at once. + std::this_thread::sleep_for(std::chrono::milliseconds{20}); + REQUIRE(ticks.load() == 0); + split.RequestTick(); + REQUIRE(wait_until([&] { return ticks.load() == 1; })); + // The back runs an entry: the request runs once it ends, before the next entry. + split.Defer([&] { + entered = true; + while (!release.load()) { + std::this_thread::yield(); + } + }); + split.Defer([&] { ticks_at_second = ticks.load(); }); + split.Publish(); + REQUIRE(wait_until([&] { return entered.load(); })); + split.RequestTick(); + std::this_thread::sleep_for(std::chrono::milliseconds{5}); + REQUIRE(ticks.load() == 1); + release = true; + split.Drain(VideoCommon::DrainSite::Other); + REQUIRE(ticks_at_second == 2); + REQUIRE(ticks.load() == 2); +} + TEST_CASE("Split restart resets writer positions without replaying old releases", "[video_core][gpu_split_dependencies]") { using Access = VideoCommon::GpuSplitTestAccess; diff --git a/src/video_core/gpu.cpp b/src/video_core/gpu.cpp index 717dc8fd06..135160132a 100644 --- a/src/video_core/gpu.cpp +++ b/src/video_core/gpu.cpp @@ -33,6 +33,7 @@ #include "video_core/engines/maxwell_dma.h" #include "video_core/gpu.h" #include "video_core/pending_maps.h" +#include "video_core/gpu_split.h" #include "video_core/gpu_thread.h" #include "video_core/gpu_timestamp.h" #include "video_core/gpu_wait_stats.h" @@ -192,11 +193,16 @@ struct GPU::Impl { /// core timing events. void Start() { Settings::UpdateGPUAccuracy(); + // With the GPU thread split, its back runs a sync operation a guest thread waits for as + // soon as it asks (OnCPURead). + VideoCommon::GpuSplit::Get().SetTickHook( + [](void* argument) { static_cast(argument)->TickWork(); }, this); gpu_thread.StartThread(*renderer, renderer->Context(), scheduler); } void NotifyShutdown() { gpu_thread.NotifyShutdown(); + VideoCommon::GpuSplit::Get().SetTickHook(nullptr, nullptr); std::unique_lock lk{sync_mutex}; shutting_down.store(true, std::memory_order::relaxed); sync_cv.notify_all(); @@ -276,6 +282,9 @@ struct GPU::Impl { timing.bytes = Stats::t_download_bytes - bytes_before; timing.end = Stats::Now(); }); + // The split's back runs it after the entry it is on. The tick sent through the GPU + // thread's queue still runs it without the split. + VideoCommon::GpuSplit::Get().RequestTick(); gpu_thread.TickGPU(is_async); WaitForSyncOperation(fence); Stats::RecordReactive(raster_area, core, request, timing, Stats::Now()); diff --git a/src/video_core/gpu_split.cpp b/src/video_core/gpu_split.cpp index 447ea6c4a2..c30b10a139 100644 --- a/src/video_core/gpu_split.cpp +++ b/src/video_core/gpu_split.cpp @@ -152,6 +152,7 @@ void GpuSplit::Stop() { consumed.store(0, std::memory_order_relaxed); back_waiting.store(false, std::memory_order_relaxed); front_waiting.store(false, std::memory_order_relaxed); + tick_requested.store(false, std::memory_order_relaxed); } u64* GpuSplit::ReserveSlow(size_t words) { @@ -423,13 +424,15 @@ void GpuSplit::BackLoop(std::stop_token stop_token) { u64 read_position = 0; while (!stop_token.stop_requested()) { Nx2Profile::Report("GPU back thread"); + MaybeTick(); u64 available = published.load(std::memory_order_acquire); if (available == read_position) { Nx2Profile::Scope scope{Nx2Profile::Zone::SplitWait}; const GpuWaitStats::IdleScope idle{GpuWaitStats::g_back_idle_ns}; // The front usually sends more within microseconds: spin briefly before sleeping. for (u32 spins = 0; spins < 8192 && available == read_position && - !hook_requested.load(std::memory_order_relaxed); + !hook_requested.load(std::memory_order_relaxed) && + !tick_requested.load(std::memory_order_relaxed); ++spins) { Pause(); available = published.load(std::memory_order_acquire); @@ -445,9 +448,11 @@ void GpuSplit::BackLoop(std::stop_token stop_token) { back_waiting.store(true, std::memory_order_seq_cst); const u32 seen = wake.load(std::memory_order_seq_cst); available = published.load(std::memory_order_seq_cst); - // A request made after the hook ran (RequestIdleHook) runs it before sleeping. + // A request made after the hook ran (RequestIdleHook) runs it before sleeping, and + // a requested tick runs at the top of the loop. if (available == read_position && !stop_token.stop_requested() && - !hook_requested.load(std::memory_order_seq_cst)) { + !hook_requested.load(std::memory_order_seq_cst) && + !tick_requested.load(std::memory_order_seq_cst)) { wake.wait(seen, std::memory_order_seq_cst); } back_waiting.store(false, std::memory_order_relaxed); @@ -465,6 +470,10 @@ void GpuSplit::BackLoop(std::stop_token stop_token) { hook_requested.store(false, std::memory_order_seq_cst); idle_hook(idle_argument); } + // A guest thread waits for a flush: run it now, not after the draws and compute work + // still queued here, or after everything the front holds (the guest synchronized with + // the GPU work it reads before it read, so that work ran on this thread already). + MaybeTick(); // Hand ring space back now and then while working through a long batch, and at once // when the front waits for this position (WaitForWriters). if ((read_position - last_store >= 4096 || diff --git a/src/video_core/gpu_split.h b/src/video_core/gpu_split.h index 2e9250ff53..d7f642434a 100644 --- a/src/video_core/gpu_split.h +++ b/src/video_core/gpu_split.h @@ -91,6 +91,25 @@ class GpuSplit { wake.fetch_add(1, std::memory_order_seq_cst); wake.notify_one(); } + /// Run on the back when a thread asks for it (RequestTick): the GPU's sync operations. + void SetTickHook(void (*hook)(void*), void* argument) { + tick_hook = hook; + tick_argument = argument; + } + /// Any thread, after queueing a sync operation it waits for: has the back run the tick hook + /// after the entry it is running, or at once if it waits for work. Otherwise the operation runs + /// after the back's next draw, or after the front sent everything it had queued before it. + void RequestTick() { + if (!back_running.load(std::memory_order_acquire)) { + return; + } + // The back checks tick_requested after setting back_waiting: one of the two sees the other. + tick_requested.store(true, std::memory_order_seq_cst); + if (back_waiting.load(std::memory_order_seq_cst)) { + wake.fetch_add(1, std::memory_order_seq_cst); + wake.notify_one(); + } + } /// Starts the back thread once the split is wanted (EDEN_NX2_SPLIT=1, a Switch 2 title, /// commands read without cache flushes). Called by the front between commands. @@ -237,6 +256,14 @@ class GpuSplit { void BackLoop(std::stop_token stop_token); /// Runs the entry at `entry` (of the `words` published before the ring's end), returns its size. size_t Execute(const u64* entry, size_t words); + /// The back, between entries: runs the tick hook if a thread asked for it. + void MaybeTick() { + if (tick_requested.load(std::memory_order_relaxed) && + tick_requested.exchange(false, std::memory_order_acq_rel) && tick_hook != nullptr) + [[unlikely]] { + tick_hook(tick_argument); + } + } Core::System* system{}; bool active{}; @@ -260,7 +287,10 @@ class GpuSplit { static inline thread_local bool on_back_thread{}; void (*idle_hook)(void*){}; void* idle_argument{}; + void (*tick_hook)(void*){}; + void* tick_argument{}; alignas(64) std::atomic hook_requested{}; + std::atomic tick_requested{}; std::atomic back_running{}; // 128 bytes: x86 spatial prefetchers pull 64-byte lines in pairs. back_waiting and wake are diff --git a/src/video_core/renderer_opengl/gl_rasterizer.cpp b/src/video_core/renderer_opengl/gl_rasterizer.cpp index 5ca8e5a0ad..529301bc01 100644 --- a/src/video_core/renderer_opengl/gl_rasterizer.cpp +++ b/src/video_core/renderer_opengl/gl_rasterizer.cpp @@ -396,6 +396,11 @@ void RasterizerOpenGL::DrawTexture() { } void RasterizerOpenGL::DispatchCompute() { + // As after a draw: run the sync operations threads wait for, such as a guest read's flush, + // rather than only at the next draw. + SCOPE_EXIT { + gpu.TickWork(); + }; gpu_memory->FlushCaching(); ComputePipeline* const pipeline{shader_cache.CurrentComputePipeline()}; if (!pipeline) { diff --git a/src/video_core/renderer_vulkan/vk_rasterizer.cpp b/src/video_core/renderer_vulkan/vk_rasterizer.cpp index f5bfcafaa8..69fd788aa5 100644 --- a/src/video_core/renderer_vulkan/vk_rasterizer.cpp +++ b/src/video_core/renderer_vulkan/vk_rasterizer.cpp @@ -1735,6 +1735,11 @@ void RasterizerVulkan::Clear(u32 layer_count) { } void RasterizerVulkan::DispatchCompute() { + // As after a draw: run the sync operations threads wait for, such as a guest read's flush, + // rather than only at the next draw. + SCOPE_EXIT { + gpu.TickWork(); + }; VideoCommon::Nx2Profile::Scope profile_scope{VideoCommon::Nx2Profile::Zone::Compute}; FlushWork(); gpu_memory->FlushCaching(); From 6229487c8c6c6c8e486d06daa1207a3a401fb8f3 Mon Sep 17 00:00:00 2001 From: jpkto <255363096+jpkto@users.noreply.github.com> Date: Sat, 7 Nov 2026 20:27:59 +0100 Subject: [PATCH 011/105] video_core: wait for a command list's writers only up to the last host sync Before the split's front fetches a command list on pages no GPU work has written yet, it waited until its back had run every compute dispatch and copy sent so far (DmaPusher::Step, dma_pusher.cpp:205; GpuSplit:: WaitForWriters), in case one of them writes the list first: the GPU-written page bits (nx2.h:36) learn of a write only when the back records it. In Ocarina of Time's Hyrule Field (EDEN_NX2_GPU_PROFILE, about 50 fps) these were about 7,500 waits a second at 19 us each, 142 ms of every second of the front, which runs in near lockstep with the back. Nearly all of them were for lists the CPU wrote: the lists Ocarina's culling writes every frame are on pages already marked, which take the copy path (ReadGpuWrittenCommands, dma_pusher.cpp:616) and still wait for their writers there (line 642). Only about 220 lists a second take it. The hardware gives a fetch no such ordering. The PBDMA fetches a GP entry's commands ahead of the engines that run the work before it; only a host-level wait in between (a GP entry's sync bit, a host WFI, a syncpoint acquire) makes that work complete before the fetch. NoteSync marks those (dma_pusher.cpp:138, puller.cpp:201 and 395), and semaphore acquires drain the split anyway (puller.cpp:258). Lists on pages that draws write already wait only for the work before the last of them (BufferCache::Nx2ReadListCopy, buffer_cache.h:2208: NeedsSync, then WaitForSync), and draws were never waited for here (IsMemoryWriter, dma_pusher.cpp:37: compute and copy methods only). So wait only for the compute and copy work sent before the last host-level wait: GpuSplit::WaitForSyncedWriters waits for the earlier of the last writer and the last sync. A list on fresh pages that work sent after the last sync writes is read as the CPU submitted it, as the hardware could. The split test checks that a list fetch waits for a writer before the last sync, not for a writer after it, and not for other work before it. Co-Authored-By: Claude Opus 5.5 --- src/tests/video_core/maxwell_fast_method.cpp | 87 ++++++++++++++++++++ src/video_core/dma_pusher.cpp | 6 +- src/video_core/gpu_split.cpp | 5 ++ src/video_core/gpu_split.h | 4 + 4 files changed, 101 insertions(+), 1 deletion(-) diff --git a/src/tests/video_core/maxwell_fast_method.cpp b/src/tests/video_core/maxwell_fast_method.cpp index 34c18ba097..84b179d9f2 100644 --- a/src/tests/video_core/maxwell_fast_method.cpp +++ b/src/tests/video_core/maxwell_fast_method.cpp @@ -55,6 +55,11 @@ struct GpuSplitTestAccess { return GpuSplit::RingWords - 1; } + /// The front waits for the back to reach a position (WaitFor). + static bool FrontWaits(const GpuSplit& split) { + return split.wait_target.load() != ~u64{0}; + } + static u64 Execute(GpuSplit& split, u64 position, u64 available) { const size_t index = position & GpuSplit::RingMask; const size_t size = split.Execute( @@ -804,6 +809,88 @@ TEST_CASE("Split back runs a requested tick while it waits for work and between REQUIRE(ticks.load() == 2); } +TEST_CASE("Split command list fetches wait only for the writers before the last host sync", + "[video_core][gpu_split_dependencies]") { + using Access = VideoCommon::GpuSplitTestAccess; + Fixture f; + f.system.Initialize(); + VideoCommon::GpuSplit split; + std::array, 3> release{}; + std::atomic done{}; + bool waited{}; + std::thread waiter; + Access::Start(split, f.system); + SCOPE_EXIT { + split.Stop(); + }; + // Runs before Stop: a failed check must not leave the back in an entry or the waiter waiting. + SCOPE_EXIT { + for (auto& flag : release) { + flag = true; + } + if (waiter.joinable()) { + waiter.join(); + } + }; + // An entry the back stays in until its flag is set. + const auto blocking = [&split](std::atomic& flag) { + split.Defer([&flag] { + while (!flag.load()) { + std::this_thread::yield(); + } + }); + }; + const auto finishes = [&done] { + const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds{5}; + while (!done.load() && std::chrono::steady_clock::now() < deadline) { + std::this_thread::sleep_for(std::chrono::milliseconds{1}); + } + return done.load(); + }; + + // A writer sent after the last host-level wait: the fetch does not wait for it. + split.NoteSync(); + blocking(release[0]); + split.NoteWriter(); + split.Publish(); + REQUIRE_FALSE(split.WaitForSyncedWriters()); + + // A wait after it orders that writer before the fetch, but not the writer sent after the wait. + split.NoteSync(); + blocking(release[1]); + split.NoteWriter(); + split.Publish(); + waiter = std::thread([&] { + waited = split.WaitForSyncedWriters(); + done = true; + }); + // The back reports its position at the target only if the front waits before it passes it. + const auto wait_deadline = std::chrono::steady_clock::now() + std::chrono::seconds{5}; + while (!Access::FrontWaits(split) && !done.load() && + std::chrono::steady_clock::now() < wait_deadline) { + std::this_thread::yield(); + } + std::this_thread::sleep_for(std::chrono::milliseconds{20}); + REQUIRE_FALSE(done.load()); + release[0] = true; + REQUIRE(finishes()); + waiter.join(); + REQUIRE(waited); + release[1] = true; + split.Drain(VideoCommon::DrainSite::Other); + + // Other work before the wait is not a writer: the fetch does not wait for it either. + split.Defer([] {}); + split.NoteWriter(); + split.Drain(VideoCommon::DrainSite::Other); + blocking(release[2]); + split.NoteSync(); + split.Publish(); + REQUIRE_FALSE(split.WaitForSyncedWriters()); + release[2] = true; + split.Drain(VideoCommon::DrainSite::Other); +} + TEST_CASE("Split restart resets writer positions without replaying old releases", "[video_core][gpu_split_dependencies]") { using Access = VideoCommon::GpuSplitTestAccess; diff --git a/src/video_core/dma_pusher.cpp b/src/video_core/dma_pusher.cpp index 87da3e1634..5fd39343e5 100644 --- a/src/video_core/dma_pusher.cpp +++ b/src/video_core/dma_pusher.cpp @@ -198,8 +198,12 @@ bool DmaPusher::Step() { // (the split only starts with unsafe reads; the settings can change mid-game). const bool use_safe = !VideoCommon::GpuSplit::Get().Active() && (Settings::IsDMALevelDefault() ? Settings::IsGPULevelHigh() : Settings::IsDMALevelSafe()); + // Pages no GPU work has written yet may still get the list from compute or copy work + // queued before it. The GPU fetches the list after that work only past a host-level wait + // (NoteSync): later work is not ordered before the fetch, and the list does not wait for + // it (nor for draws, which write memory too; draw-written lists wait for the last sync). if (!gpu_written && VideoCommon::GpuSplit::Get().Active() && - VideoCommon::GpuSplit::Get().WaitForWriters()) { + VideoCommon::GpuSplit::Get().WaitForSyncedWriters()) { gpu_written = IsGpuWrittenCommands(dma_state.dma_get, size_t{header.size} * sizeof(u32)); } diff --git a/src/video_core/gpu_split.cpp b/src/video_core/gpu_split.cpp index c30b10a139..27238dc03e 100644 --- a/src/video_core/gpu_split.cpp +++ b/src/video_core/gpu_split.cpp @@ -228,6 +228,11 @@ bool GpuSplit::WaitForSync() { return WaitFor(std::min(last_sync_position, write_position), DrainSite::ListSync); } +bool GpuSplit::WaitForSyncedWriters() { + return WaitFor(std::min({last_writer_position, last_sync_position, write_position}), + DrainSite::ListWriters); +} + bool GpuSplit::WaitFor(u64 target, DrainSite site) { consumed_cache = consumed.load(std::memory_order_acquire); if (consumed_cache >= target) { diff --git a/src/video_core/gpu_split.h b/src/video_core/gpu_split.h index d7f642434a..e152f91356 100644 --- a/src/video_core/gpu_split.h +++ b/src/video_core/gpu_split.h @@ -193,6 +193,10 @@ class GpuSplit { /// Waits until the back executed the work sent before the last NoteSync. Returns whether any /// of it was still queued. bool WaitForSync(); + /// Waits until the back executed the compute and copy work sent before the last NoteSync: + /// the writers a command list fetched now is ordered after. Returns whether any of it was + /// still queued. + bool WaitForSyncedWriters(); private: friend struct GpuSplitTestAccess; From 4a143da9577e2159669b8fa627ab0daee7e0494e Mon Sep 17 00:00:00 2001 From: jpkto <255363096+jpkto@users.noreply.github.com> Date: Sat, 7 Nov 2026 20:34:09 +0100 Subject: [PATCH 012/105] TEST: count what the split's front would miss without its waits (EDEN_NX2_DRAIN_CHECK) Opt-in: with EDEN_NX2_DRAIN_CHECK unset, each site only tests a constant. With it set, the GPU thread logs one line every 10 s: Drain checks in 10.0 s: command lists that did not wait for writers sent after the last sync N, GPU-written by those writers M | acquires at addresses the GPU never released A (met before and after the drain ..., only after ..., only before ..., neither ...), at ones it released B (...) | polls of suspended acquires at addresses never released C (...), at released ones D (...) - Command lists on pages no GPU work had written that did not wait for the compute and copy work sent after the last host-level wait (the previous commit): at the list's place, the back checks whether that work made the list's pages GPU-written (DmaPusher::CheckUnwaitedWriters). M must stay 0: a list it counts was read as the CPU submitted it although work before it rewrote it. - Semaphore acquires and the polls of suspended ones (Puller): whether the value met the acquire before the front drained the split as well as after, by whether the GPU had run a release at that address before (any engine: the rasterizers' Query with a payload). Skipping the drain only at addresses the GPU never released, "only before" there is what an acquire would get wrong, "only after" what would wait longer, and "met before and after" the drains it would save. The existing diagnostics are unchanged. Rebased onto upstream: EDEN_NX2_DRAIN_CHECK is a plain flag read again by VideoCore::CreateGPU, as upstream reads its diagnostic switches since #714, so it works from nx2_switches.txt too, not only from the environment. Co-Authored-By: Claude Opus 5.5 --- src/tests/video_core/maxwell_fast_method.cpp | 34 +++++ src/video_core/CMakeLists.txt | 1 + src/video_core/dma_pusher.cpp | 22 +++ src/video_core/dma_pusher.h | 3 + src/video_core/engines/puller.cpp | 27 +++- src/video_core/gpu_split.cpp | 5 + src/video_core/gpu_split.h | 2 + src/video_core/gpu_thread.cpp | 2 + src/video_core/nx2_drain_check.h | 131 ++++++++++++++++++ .../renderer_opengl/gl_rasterizer.cpp | 7 + .../renderer_vulkan/vk_rasterizer.cpp | 7 + src/video_core/video_core.cpp | 2 + 12 files changed, 242 insertions(+), 1 deletion(-) create mode 100644 src/video_core/nx2_drain_check.h diff --git a/src/tests/video_core/maxwell_fast_method.cpp b/src/tests/video_core/maxwell_fast_method.cpp index 84b179d9f2..c4597f56ba 100644 --- a/src/tests/video_core/maxwell_fast_method.cpp +++ b/src/tests/video_core/maxwell_fast_method.cpp @@ -891,6 +891,40 @@ TEST_CASE("Split command list fetches wait only for the writers before the last split.Drain(VideoCommon::DrainSite::Other); } +TEST_CASE("Split tells whether compute or copy work it was sent is still queued", + "[video_core][gpu_split_dependencies]") { + using Access = VideoCommon::GpuSplitTestAccess; + Fixture f; + f.system.Initialize(); + VideoCommon::GpuSplit split; + std::atomic release{}; + Access::Start(split, f.system); + SCOPE_EXIT { + split.Stop(); + }; + // Runs before Stop: a failed check must not leave the back in the entry below. + SCOPE_EXIT { + release = true; + }; + REQUIRE_FALSE(split.WritersPending()); + split.Defer([&release] { + while (!release.load()) { + std::this_thread::yield(); + } + }); + split.NoteWriter(); + split.Publish(); + REQUIRE(split.WritersPending()); + release = true; + split.Drain(VideoCommon::DrainSite::Other); + REQUIRE_FALSE(split.WritersPending()); + // Other work does not count. + split.Defer([] {}); + split.Publish(); + REQUIRE_FALSE(split.WritersPending()); + split.Drain(VideoCommon::DrainSite::Other); +} + TEST_CASE("Split restart resets writer positions without replaying old releases", "[video_core][gpu_split_dependencies]") { using Access = VideoCommon::GpuSplitTestAccess; diff --git a/src/video_core/CMakeLists.txt b/src/video_core/CMakeLists.txt index 4813673d96..f85ad42cbe 100644 --- a/src/video_core/CMakeLists.txt +++ b/src/video_core/CMakeLists.txt @@ -118,6 +118,7 @@ add_library(video_core STATIC nx2_cmdpage_diag.h nx2_debug_control.cpp nx2_debug_control.h + nx2_drain_check.h nx2_profile.h nx2_perf.h nx2_download_trace.h diff --git a/src/video_core/dma_pusher.cpp b/src/video_core/dma_pusher.cpp index 5fd39343e5..eb9d8415d7 100644 --- a/src/video_core/dma_pusher.cpp +++ b/src/video_core/dma_pusher.cpp @@ -27,6 +27,7 @@ #include "video_core/memory_manager.h" #include "video_core/nx2.h" #include "video_core/nx2_cmdpage_diag.h" +#include "video_core/nx2_drain_check.h" #include "video_core/rasterizer_interface.h" namespace Tegra { @@ -207,6 +208,9 @@ bool DmaPusher::Step() { gpu_written = IsGpuWrittenCommands(dma_state.dma_get, size_t{header.size} * sizeof(u32)); } + if (VideoCommon::Nx2DrainCheck::g_enabled && !gpu_written) [[unlikely]] { + CheckUnwaitedWriters(dma_state.dma_get, size_t{header.size} * sizeof(u32)); + } // Resume and GPU-written entries need live words, even after a flush cleared dirty state. const bool submitted = (command_list.snapshot_suspensions & 1) == 0 && command_list.snapshot_suspensions == @@ -603,6 +607,24 @@ bool DmaPusher::IsGpuWrittenCommands(GPUVAddr address, size_t size) { return false; } +void DmaPusher::CheckUnwaitedWriters(GPUVAddr address, size_t size) { + auto& split = VideoCommon::GpuSplit::Get(); + if (!split.Active() || !split.WritersPending()) { + return; + } + // The list did not wait for compute or copy work sent after the last sync. At the list's + // place, the back has run that work: did it make the list's pages GPU-written? + VideoCommon::Nx2DrainCheck::NoteListUnwaited(); + split.Defer([memory = &memory_manager, address, size] { + bool written = false; + for (GPUVAddr page = address & ~GPUVAddr{0xfff}; page < address + size; page += 0x1000) { + const auto device = memory->GpuToCpuAddress(page); + written = written || (device && Tegra::Nx2IsGpuWritten(*device)); + } + VideoCommon::Nx2DrainCheck::NoteListChecked(written); + }); +} + bool DmaPusher::ReadGpuWrittenCommands(GPUVAddr address, u32 count) { // The pages are marked as command list pages: from the next write on, the buffer cache copies // them to host memory right after the dispatch or copy that writes them and submits. The diff --git a/src/video_core/dma_pusher.h b/src/video_core/dma_pusher.h index 94ada95691..0a59e65dc7 100644 --- a/src/video_core/dma_pusher.h +++ b/src/video_core/dma_pusher.h @@ -217,6 +217,9 @@ class DmaPusher final { void RecordParsedEntry(const CommandListHeader& header, std::span commands); /// Whether the entry lies on device pages the GPU has written (NX2). bool IsGpuWrittenCommands(GPUVAddr address, size_t size); + /// EDEN_NX2_DRAIN_CHECK: checks an entry on pages no GPU work had written against the compute + /// and copy work it did not wait for (Nx2DrainCheck). + void CheckUnwaitedWriters(GPUVAddr address, size_t size); /// Copies the list's words from memory again, as of suspension count `now`. void RefreshSnapshot(CommandList& list, u64 now); /// Copies the list's deferred entries (CommandList::deferred_entries) from memory. diff --git a/src/video_core/engines/puller.cpp b/src/video_core/engines/puller.cpp index 8fb6a43f47..32426f2277 100644 --- a/src/video_core/engines/puller.cpp +++ b/src/video_core/engines/puller.cpp @@ -24,6 +24,7 @@ #include "video_core/gpu_split.h" #include "video_core/memory_manager.h" #include "video_core/nx2.h" +#include "video_core/nx2_drain_check.h" #include "video_core/rasterizer_interface.h" namespace Tegra::Engines { @@ -62,6 +63,15 @@ void DrainBack() { VideoCommon::GpuSplit::Get().Drain(VideoCommon::DrainSite::Acquire); } +// EDEN_NX2_DRAIN_CHECK: whether an acquire (or a suspended one's poll) met its value before the +// drain as well as after it, and whether the GPU had released a semaphore at the address. +void NoteAcquireCheck(DmaPusher& dma_pusher, GPUVAddr address, bool poll, bool met_before, + bool met_after) { + const auto device = dma_pusher.memory_manager.GpuToCpuAddress(address); + const bool released = device && VideoCommon::Nx2DrainCheck::WasReleased(*device); + VideoCommon::Nx2DrainCheck::NoteAcquire(poll, released, met_before, met_after); +} + // The fence manager records into the scheduler and downloads the CPU's writes over GPU data: // the back's state, which a drained back still uses from its idle hook (it submits). void ReleaseFences(DmaPusher& dma_pusher) { @@ -89,10 +99,18 @@ bool Puller::CanResumeAcquire(DmaPusher& dma_pusher) { if (!pending_acquire) { return true; } + const bool check = VideoCommon::Nx2DrainCheck::g_enabled; + const bool met_before = + check && + pending_acquire->Matches(dma_pusher.memory_manager.Read(pending_acquire->address)); // The release may still be queued for the split GPU thread's back half (and the caches // ReadSemaphoreWord asks are its state). DrainBack(); const u32 current = ReadSemaphoreWord(dma_pusher, pending_acquire->address); + if (check) [[unlikely]] { + NoteAcquireCheck(dma_pusher, pending_acquire->address, true, met_before, + pending_acquire->Matches(current)); + } if (Nx2SemaphoreWaiters::Get().Latched(this)) { if (static std::atomic rescued{}; !pending_acquire->Matches(current) && rescued.fetch_add(1) < 64) { @@ -255,8 +273,15 @@ void Puller::ProcessSemaphoreTriggerMethod(DmaPusher& dma_pusher) { return; } VideoCommon::Nx2Profile::Add(VideoCommon::Nx2Profile::Counter::AcquireChecks, 1); + const bool check = VideoCommon::Nx2DrainCheck::g_enabled; + const bool met_before = + check && acquire.Matches(dma_pusher.memory_manager.Read(acquire.address)); DrainBack(); - if (acquire.Matches(ReadSemaphoreWord(dma_pusher, acquire.address))) { + const bool met = acquire.Matches(ReadSemaphoreWord(dma_pusher, acquire.address)); + if (check) [[unlikely]] { + NoteAcquireCheck(dma_pusher, acquire.address, false, met_before, met); + } + if (met) { return; } // Registered before the fences run, so a release among them that meets the value counts diff --git a/src/video_core/gpu_split.cpp b/src/video_core/gpu_split.cpp index 27238dc03e..e148aba4fe 100644 --- a/src/video_core/gpu_split.cpp +++ b/src/video_core/gpu_split.cpp @@ -233,6 +233,11 @@ bool GpuSplit::WaitForSyncedWriters() { DrainSite::ListWriters); } +bool GpuSplit::WritersPending() const { + return consumed.load(std::memory_order_acquire) < + std::min(last_writer_position, write_position); +} + bool GpuSplit::WaitFor(u64 target, DrainSite site) { consumed_cache = consumed.load(std::memory_order_acquire); if (consumed_cache >= target) { diff --git a/src/video_core/gpu_split.h b/src/video_core/gpu_split.h index e152f91356..3e31cd668f 100644 --- a/src/video_core/gpu_split.h +++ b/src/video_core/gpu_split.h @@ -197,6 +197,8 @@ class GpuSplit { /// the writers a command list fetched now is ordered after. Returns whether any of it was /// still queued. bool WaitForSyncedWriters(); + /// Whether compute or copy work sent so far is still queued (EDEN_NX2_DRAIN_CHECK). + [[nodiscard]] bool WritersPending() const; private: friend struct GpuSplitTestAccess; diff --git a/src/video_core/gpu_thread.cpp b/src/video_core/gpu_thread.cpp index baa6f64513..8c2e5973d8 100644 --- a/src/video_core/gpu_thread.cpp +++ b/src/video_core/gpu_thread.cpp @@ -24,6 +24,7 @@ #include "video_core/gpu_thread.h" #include "video_core/gpu_split.h" #include "video_core/gpu_wait_stats.h" +#include "video_core/nx2_drain_check.h" #include "video_core/nx2_profile.h" #include "video_core/host1x/host1x.h" #include "video_core/renderer_base.h" @@ -78,6 +79,7 @@ void ThreadManager::StartThread(VideoCore::RendererBase& renderer, Core::Fronten Profile::Report(); split.MaybeStart(system); WaitStats::MaybeReport(split.Active()); + VideoCommon::Nx2DrainCheck::MaybeReport(); if (scheduler.HasPendingChannels()) { // Suspended channels wait on semaphores the CPU releases: poll them meanwhile. { diff --git a/src/video_core/nx2_drain_check.h b/src/video_core/nx2_drain_check.h new file mode 100644 index 0000000000..ec79c4665f --- /dev/null +++ b/src/video_core/nx2_drain_check.h @@ -0,0 +1,131 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later + +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "common/common_types.h" +#include "common/logging.h" + +// EDEN_NX2_DRAIN_CHECK: what the split GPU thread's front would see without some of its waits for +// the back, counted without changing what runs and logged every 10 s by the front: +// - command lists on pages no GPU work had written that did not wait for the compute and copy +// work sent after the last host-level wait (DmaPusher::Step): how many of them that work made +// GPU-written, checked on the back at the list's place. It must stay 0; +// - semaphore acquires, and the polls of suspended ones (Puller): whether the value met the +// acquire before the front drained the split and after, by whether the GPU had released a +// semaphore at that address before (any engine, RasterizerInterface::Query with a payload). +namespace VideoCommon::Nx2DrainCheck { + +// A plain flag: the front checks it at every command list and acquire. +inline bool g_enabled = std::getenv("EDEN_NX2_DRAIN_CHECK") != nullptr; + +/// Called by VideoCore::CreateGPU: static initialization runs before the frontend loads +/// nx2_switches.txt, so a switch set only there takes effect from that point on. +inline void ReadSwitch() { + g_enabled = std::getenv("EDEN_NX2_DRAIN_CHECK") != nullptr; +} + +struct State { + std::atomic lists_unwaited{}; + std::atomic lists_written{}; + /// Indexed by poll * 8 + released * 4 + met before the drain * 2 + met after it. + std::array, 16> acquires{}; + std::mutex released_mutex; + std::unordered_set released; ///< Device addresses of the semaphores the GPU released + std::chrono::steady_clock::time_point last_report{}; ///< The front's +}; + +inline State& Get() { + static State state; + return state; +} + +/// The back runs a semaphore release at `device_address`. +inline void NoteRelease(u64 device_address) { + State& state = Get(); + std::scoped_lock lock{state.released_mutex}; + state.released.insert(device_address & ~u64{3}); +} + +[[nodiscard]] inline bool WasReleased(u64 device_address) { + State& state = Get(); + std::scoped_lock lock{state.released_mutex}; + return state.released.contains(device_address & ~u64{3}); +} + +/// An acquire, or the poll of a suspended one (`poll`), at a semaphore the GPU had `released`: +/// whether the value met it before the drain and after. +inline void NoteAcquire(bool poll, bool released, bool met_before, bool met_after) { + const size_t index = + (poll ? 8U : 0U) + (released ? 4U : 0U) + (met_before ? 2U : 0U) + (met_after ? 1U : 0U); + Get().acquires[index].fetch_add(1, std::memory_order_relaxed); +} + +/// The front fetches a list without waiting for writers still queued after the last sync. +inline void NoteListUnwaited() { + Get().lists_unwaited.fetch_add(1, std::memory_order_relaxed); +} + +/// The back reached that list: whether the work before it made the list's pages GPU-written. +inline void NoteListChecked(bool written) { + if (written) { + Get().lists_written.fetch_add(1, std::memory_order_relaxed); + } +} + +/// The front, between commands. +inline void MaybeReport() { + if (!g_enabled) { + return; + } + State& state = Get(); + const auto now = std::chrono::steady_clock::now(); + if (state.last_report == std::chrono::steady_clock::time_point{}) { + state.last_report = now; + return; + } + const double seconds = std::chrono::duration(now - state.last_report).count(); + if (seconds < 10.0) { + return; + } + state.last_report = now; + const auto take = [](std::atomic& value) { + return value.exchange(0, std::memory_order_relaxed); + }; + const auto group = [&](bool poll, bool released) { + const size_t base = (poll ? 8U : 0U) + (released ? 4U : 0U); + const u64 neither = take(state.acquires[base]); + const u64 only_after = take(state.acquires[base + 1]); + const u64 only_before = take(state.acquires[base + 2]); + const u64 both = take(state.acquires[base + 3]); + return fmt::format("{} (met before and after the drain {}, only after {}, only before {}, " + "neither {})", + neither + only_after + only_before + both, both, only_after, only_before, + neither); + }; + const u64 unwaited = take(state.lists_unwaited); + const u64 written = take(state.lists_written); + const std::string acquires_never = group(false, false); + const std::string acquires_released = group(false, true); + const std::string polls_never = group(true, false); + const std::string polls_released = group(true, true); + LOG_INFO(HW_GPU, + "Drain checks in {:.1f} s: command lists that did not wait for writers sent after the " + "last sync {}, GPU-written by those writers {} | acquires at addresses the GPU never " + "released {}, at ones it released {} | polls of suspended acquires at addresses never " + "released {}, at released ones {}", + seconds, unwaited, written, acquires_never, acquires_released, polls_never, + polls_released); +} + +} // namespace VideoCommon::Nx2DrainCheck diff --git a/src/video_core/renderer_opengl/gl_rasterizer.cpp b/src/video_core/renderer_opengl/gl_rasterizer.cpp index 529301bc01..aadd5d708d 100644 --- a/src/video_core/renderer_opengl/gl_rasterizer.cpp +++ b/src/video_core/renderer_opengl/gl_rasterizer.cpp @@ -23,6 +23,7 @@ #include "video_core/engines/kepler_compute.h" #include "video_core/engines/maxwell_3d.h" #include "video_core/memory_manager.h" +#include "video_core/nx2_drain_check.h" #include "video_core/renderer_opengl/gl_device.h" #include "video_core/renderer_opengl/gl_query_cache.h" #include "video_core/renderer_opengl/gl_rasterizer.h" @@ -446,6 +447,12 @@ void RasterizerOpenGL::ResetCounter(VideoCommon::QueryType type) { void RasterizerOpenGL::Query(GPUVAddr gpu_addr, VideoCommon::QueryType type, VideoCommon::QueryPropertiesFlags flags, u32 payload, u32 subreport) { + if (VideoCommon::Nx2DrainCheck::g_enabled && type == VideoCommon::QueryType::Payload) + [[unlikely]] { + if (const auto released = gpu_memory->GpuToCpuAddress(gpu_addr)) { + VideoCommon::Nx2DrainCheck::NoteRelease(*released); + } + } const auto query_cache_type = MaxwellToVideoCoreQuery(type); if (!query_cache_type.has_value()) { return QueryFallback(gpu_addr, type, flags, payload, subreport); diff --git a/src/video_core/renderer_vulkan/vk_rasterizer.cpp b/src/video_core/renderer_vulkan/vk_rasterizer.cpp index 69fd788aa5..0a3e6affd8 100644 --- a/src/video_core/renderer_vulkan/vk_rasterizer.cpp +++ b/src/video_core/renderer_vulkan/vk_rasterizer.cpp @@ -73,6 +73,7 @@ #include "video_core/memory_manager.h" #include "video_core/nx2.h" #include "video_core/nx2_debug_control.h" +#include "video_core/nx2_drain_check.h" #include "video_core/nx2_profile.h" namespace Vulkan { @@ -1934,6 +1935,12 @@ void RasterizerVulkan::ResetCounter(VideoCommon::QueryType type) { void RasterizerVulkan::Query(GPUVAddr gpu_addr, VideoCommon::QueryType type, VideoCommon::QueryPropertiesFlags flags, u32 payload, u32 subreport) { + if (VideoCommon::Nx2DrainCheck::g_enabled && type == VideoCommon::QueryType::Payload) + [[unlikely]] { + if (const auto released = gpu_memory->GpuToCpuAddress(gpu_addr)) { + VideoCommon::Nx2DrainCheck::NoteRelease(*released); + } + } query_cache.CounterReport(gpu_addr, type, flags, payload, subreport); } diff --git a/src/video_core/video_core.cpp b/src/video_core/video_core.cpp index c6d08414ce..734522e708 100644 --- a/src/video_core/video_core.cpp +++ b/src/video_core/video_core.cpp @@ -15,6 +15,7 @@ #include "video_core/host1x/host1x.h" #include "video_core/nx2_cmdpage_diag.h" #include "video_core/nx2_download_trace.h" +#include "video_core/nx2_drain_check.h" #include "video_core/nx2_profile.h" #include "video_core/renderer_base.h" #include "video_core/renderer_null/renderer_null.h" @@ -54,6 +55,7 @@ void CreateGPU(std::optional& gpu, Core::Frontend::EmuWindow& emu_wi VideoCommon::Nx2Profile::ReadSwitch(); Tegra::Nx2CmdPageDiag::ReadSwitch(); VideoCommon::Nx2DownloadTrace::ReadSwitch(); + VideoCommon::Nx2DrainCheck::ReadSwitch(); const auto nvdec_value = Settings::values.nvdec_emulation.GetValue(); const bool use_nvdec = nvdec_value != Settings::NvdecEmulation::Off; From fb4d4e47a7285c01dc202fc73d3f95029ac71267 Mon Sep 17 00:00:00 2001 From: jpkto <255363096+jpkto@users.noreply.github.com> Date: Sat, 7 Nov 2026 21:09:13 +0100 Subject: [PATCH 013/105] video_core: check Switch 2 semaphore acquires without draining the split when they can Every Switch 2 semaphore acquire drained the split before it read the word (Puller::ProcessSemaphoreTriggerMethod), and so did every poll of a suspended one (CanResumeAcquire): the front waited until the back had run everything queued, in case a release among it changes the word. In Ocarina of Time's Hyrule Field (EDEN_NX2_GPU_PROFILE, about 50 fps) that was about 7,100 drains a second at 34 us, 239 ms of every second of the front. Counted without changing anything (EDEN_NX2_DRAIN_CHECK, per 10 s): of about 48,000 acquires, 25,000 met their value before the drain as well as after it and none before but not after; of 145,000 polls, the drain changed the outcome of about 10. Now a greater-or-equal acquire is checked first without the drain. The words Switch 2 syncs on only advance (NVN2 even releases QMD completions as maxima), so a value that meets one now still does once the back ran the queued work. Only a release the front sent to the word that the back may not have run yet can move it: the front now records those as it sends them (GpuSplit::NoteSemaphoreWrite, from the puller's releases, the 3D and compute engines' report semaphores and the copy engine's), and the last one decides. A counter report or a reduction, whose value the front can't tell, sends the acquire down the drain path as before. - Met: the acquire passes without waiting for the back. When it is met by a release still queued, NoteSync makes what the front reads after it (GPU-written command lists) wait for the work sent before the release. - Not met: an acquire drains as before. A suspended one polls again later without draining: until the back runs a release, nothing sent to the word meets it, and the back's release latches the waiter or reaches memory on its own. After 2 s a poll drains once to report the wait. Words the buffer cache may hold newer than memory (Nx2MayBeGpuWritten), equal and mask acquires, High GPU accuracy and delayed fences (where releases reach memory later) and macros run on the back keep the drain. The split test checks the record of queued writes. Co-Authored-By: Claude Opus 5.5 --- src/tests/video_core/maxwell_fast_method.cpp | 49 ++++++++++ src/video_core/dma_pusher.cpp | 39 ++++++++ src/video_core/dma_pusher.h | 5 + src/video_core/engines/kepler_compute.cpp | 33 +++++++ src/video_core/engines/kepler_compute.h | 1 + src/video_core/engines/maxwell_3d.cpp | 28 ++++++ src/video_core/engines/maxwell_3d.h | 1 + src/video_core/engines/maxwell_dma.h | 5 + src/video_core/engines/puller.cpp | 97 ++++++++++++++++++++ src/video_core/engines/puller.h | 2 + src/video_core/gpu_split.cpp | 24 +++++ src/video_core/gpu_split.h | 25 ++++- 12 files changed, 307 insertions(+), 2 deletions(-) diff --git a/src/tests/video_core/maxwell_fast_method.cpp b/src/tests/video_core/maxwell_fast_method.cpp index c4597f56ba..ac901a322e 100644 --- a/src/tests/video_core/maxwell_fast_method.cpp +++ b/src/tests/video_core/maxwell_fast_method.cpp @@ -925,6 +925,55 @@ TEST_CASE("Split tells whether compute or copy work it was sent is still queued" split.Drain(VideoCommon::DrainSite::Other); } +TEST_CASE("Split remembers the semaphore writes its back has not run yet", + "[video_core][gpu_split_dependencies]") { + using Access = VideoCommon::GpuSplitTestAccess; + Fixture f; + f.system.Initialize(); + VideoCommon::GpuSplit split; + std::atomic release{}; + Access::Start(split, f.system); + SCOPE_EXIT { + split.Stop(); + }; + // Runs before Stop: a failed check must not leave the back in the entry below. + SCOPE_EXIT { + release = true; + }; + // Releases behind an entry the back is still in. + split.Defer([&release] { + while (!release.load()) { + std::this_thread::yield(); + } + }); + split.Defer([] {}); + split.NoteSemaphoreWrite(0x1000, 5, true); + split.Defer([] {}); + split.NoteSemaphoreWrite(0x2002, 7, false); + split.Publish(); + auto queued = split.QueuedSemaphoreWriteAt(0x1000); + REQUIRE(queued.has_value()); + REQUIRE(queued->known); + REQUIRE(queued->payload == 5); + // By 4-byte word: the second write is at 0x2000. No write was sent to 0x1004. + queued = split.QueuedSemaphoreWriteAt(0x2000); + REQUIRE(queued.has_value()); + REQUIRE_FALSE(queued->known); + REQUIRE_FALSE(split.QueuedSemaphoreWriteAt(0x1004).has_value()); + // The last write sent to a word decides. + split.Defer([] {}); + split.NoteSemaphoreWrite(0x1000, 9, true); + split.Publish(); + queued = split.QueuedSemaphoreWriteAt(0x1000); + REQUIRE(queued.has_value()); + REQUIRE(queued->payload == 9); + // Once the back ran them, the words hold their values in memory. + release = true; + split.Drain(VideoCommon::DrainSite::Other); + REQUIRE_FALSE(split.QueuedSemaphoreWriteAt(0x1000).has_value()); + REQUIRE_FALSE(split.QueuedSemaphoreWriteAt(0x2000).has_value()); +} + TEST_CASE("Split restart resets writer positions without replaying old releases", "[video_core][gpu_split_dependencies]") { using Access = VideoCommon::GpuSplitTestAccess; diff --git a/src/video_core/dma_pusher.cpp b/src/video_core/dma_pusher.cpp index eb9d8415d7..e914bdcb09 100644 --- a/src/video_core/dma_pusher.cpp +++ b/src/video_core/dma_pusher.cpp @@ -6,6 +6,7 @@ #include #include +#include #include #include #include @@ -21,6 +22,7 @@ #include "video_core/control/channel_state.h" #include "video_core/engines/kepler_compute.h" #include "video_core/engines/maxwell_3d.h" +#include "video_core/engines/maxwell_dma.h" #include "video_core/gpu.h" #include "video_core/gpu_split.h" #include "video_core/guest_memory.h" @@ -497,6 +499,9 @@ void DmaPusher::CallMethod(u32 argument) { } split.Method(subchannel, dma_state.method, argument, dma_state.is_last_call, WantsSegment() ? &segment : nullptr); + if (engine_type == Engines::EngineTypes::MaxwellDMA) { + NoteDmaSemaphore(dma_state.method, argument); + } return; } if (!subchannel->execution_mask[dma_state.method]) { @@ -555,6 +560,11 @@ void DmaPusher::CallMultiMethod(const u32* base_start, u32 num_methods) { } split.MultiMethod(subchannel, dma_state.method, base_start, num_methods, dma_state.method_count, WantsSegment() ? &segment : nullptr); + if (engine_type == Engines::EngineTypes::MaxwellDMA) { + for (u32 i = 0; i < num_methods; ++i) { + NoteDmaSemaphore(dma_state.method, base_start[i]); + } + } return; } VideoCommon::Nx2Profile::Scope method_scope{VideoCommon::Nx2Profile::Zone::Method}; @@ -564,6 +574,35 @@ void DmaPusher::CallMultiMethod(const u32* base_start, u32 num_methods) { } } +void DmaPusher::NoteDmaSemaphore(u32 method, u32 argument) { + using Engines::MaxwellDMA; + if (method >= MaxwellDMA::SemaphoreMethod && method < MaxwellDMA::SemaphoreMethod + 3) { + dma_semaphore[method - MaxwellDMA::SemaphoreMethod] = argument; + return; + } + if (method != MaxwellDMA::LaunchDmaMethod) { + return; + } + // As MaxwellDMA::ReleaseSemaphore writes it. + using SemaphoreType = MaxwellDMA::LaunchDMA::SemaphoreType; + const auto type = std::bit_cast(argument).semaphore_type.Value(); + if (type != SemaphoreType::RELEASE_ONE_WORD_SEMAPHORE && + type != SemaphoreType::RELEASE_FOUR_WORD_SEMAPHORE) { + return; + } + const GPUVAddr address = MaxwellDMA::PackedGPUVAddr{dma_semaphore[0], dma_semaphore[1]}; + const auto device = memory_manager.GpuToCpuAddress(address); + if (!device) { + return; + } + auto& split = VideoCommon::GpuSplit::Get(); + split.NoteSemaphoreWrite(*device, dma_semaphore[2], true); + if (type == SemaphoreType::RELEASE_FOUR_WORD_SEMAPHORE) { + // The payload as 64 bits (a timestamp follows). + split.NoteSemaphoreWrite(*device + 4, 0, true); + } +} + void DmaPusher::RefreshSnapshot(CommandList& list, u64 now) { list.snapshot_suspensions = now; for (size_t i = 0; i < list.command_lists.size() && i < list.submitted_offsets.size(); ++i) { diff --git a/src/video_core/dma_pusher.h b/src/video_core/dma_pusher.h index 0a59e65dc7..227a97cdcd 100644 --- a/src/video_core/dma_pusher.h +++ b/src/video_core/dma_pusher.h @@ -220,6 +220,9 @@ class DmaPusher final { /// EDEN_NX2_DRAIN_CHECK: checks an entry on pages no GPU work had written against the compute /// and copy work it did not wait for (Nx2DrainCheck). void CheckUnwaitedWriters(GPUVAddr address, size_t size); + /// A copy engine method sent to the split's back: a launch that releases its semaphore tells + /// the front what it writes (GpuSplit::NoteSemaphoreWrite). + void NoteDmaSemaphore(u32 method, u32 argument); /// Copies the list's words from memory again, as of suspension count `now`. void RefreshSnapshot(CommandList& list, u64 now); /// Copies the list's deferred entries (CommandList::deferred_entries) from memory. @@ -303,6 +306,8 @@ class DmaPusher final { std::array subchannels{}; std::array subchannel_type; + /// The copy engine's semaphore address (high, low) and payload as sent (NoteDmaSemaphore). + std::array dma_semaphore{}; Engines::Puller puller; std::mutex sync_mutex; diff --git a/src/video_core/engines/kepler_compute.cpp b/src/video_core/engines/kepler_compute.cpp index b4992625a1..311f6dbb53 100644 --- a/src/video_core/engines/kepler_compute.cpp +++ b/src/video_core/engines/kepler_compute.cpp @@ -1526,6 +1526,31 @@ void KeplerCompute::FrontCallMethod(Core::System& system, VideoCommon::GpuSplit& } FrontApply(fs, method, argument); split.Method(this, method, argument, is_last_call, segment); + if (method == 0x6c3) { + FrontNoteReport(split, fs, argument); + } +} + +void KeplerCompute::FrontNoteReport(VideoCommon::GpuSplit& split, const FrontState& fs, + u32 control) { + // SET_REPORT_SEMAPHORE_D was sent: what ProcessNativeReportSemaphore will write on the back. + const u32 operation = control & 3; + if (operation != 0 && operation != 2) { + return; + } + const GPUVAddr address = + (GPUVAddr{fs.registers[0x6c0] & 0x1ffffff} << 32) | fs.registers[0x6c1]; + const auto device = memory_manager.GpuToCpuAddress(address); + if (!device) { + return; + } + // A reduction combines the payload with the word's value then. + const bool reduction = ((control >> 3) & 1) != 0; + split.NoteSemaphoreWrite(*device, fs.registers[0x6c2], !reduction); + if (!reduction && ((control >> 28) & 1) == 0) { + // 64 bits (a timestamp follows). + split.NoteSemaphoreWrite(*device + 4, 0, true); + } } void KeplerCompute::FrontCallMultiMethod(Core::System& system, VideoCommon::GpuSplit& split, @@ -1544,6 +1569,11 @@ void KeplerCompute::FrontCallMultiMethod(Core::System& system, VideoCommon::GpuS } } split.MultiMethod(this, method, base_start, amount, methods_pending, segment); + if (method == 0x6c3) { + for (u32 i = 0; i < amount; ++i) { + FrontNoteReport(split, fs, base_start[i]); + } + } } void KeplerCompute::FrontRunMacro(Core::System& system, VideoCommon::GpuSplit& split, @@ -1565,6 +1595,9 @@ void KeplerCompute::FrontRunMacro(Core::System& system, VideoCommon::GpuSplit& s } FrontApply(fs, address, value); split.MacroWrite(this, address, value); + if (address == 0x6c3) { + FrontNoteReport(split, fs, value); + } return true; }; #ifdef ARCHITECTURE_x86_64 diff --git a/src/video_core/engines/kepler_compute.h b/src/video_core/engines/kepler_compute.h index 158d472c0d..6e99ffc239 100644 --- a/src/video_core/engines/kepler_compute.h +++ b/src/video_core/engines/kepler_compute.h @@ -274,6 +274,7 @@ class KeplerCompute final : public EngineInterface { FrontState& Front(VideoCommon::GpuSplit& split); void ReleaseFront(); void FrontApply(FrontState& state, u32 method, u32 argument); + void FrontNoteReport(VideoCommon::GpuSplit& split, const FrontState& state, u32 control); MacroEngine& FrontMacroEngine(FrontState& state); std::span FrontMacroPositions(FrontState& state); void FrontRunMacro(Core::System& system, VideoCommon::GpuSplit& split, FrontState& state); diff --git a/src/video_core/engines/maxwell_3d.cpp b/src/video_core/engines/maxwell_3d.cpp index ff97710f95..142c8573cd 100644 --- a/src/video_core/engines/maxwell_3d.cpp +++ b/src/video_core/engines/maxwell_3d.cpp @@ -1210,6 +1210,31 @@ void Maxwell3D::FrontWrite(VideoCommon::GpuSplit& split, FrontState& fs, u32 met } else { split.Method(this, method, argument, is_last_call, nullptr); } + if (method == MAXWELL3D_REG_INDEX(report_semaphore.query)) { + FrontNoteReport(split, fs); + } +} + +void Maxwell3D::FrontNoteReport(VideoCommon::GpuSplit& split, const FrontState& fs) { + // What ProcessQueryGet will write on the back: a payload report writes the payload, a counter + // what the GPU counted. + Regs::ReportSemaphore report; + std::memcpy(&report, &fs.regs[MAXWELL3D_REG_INDEX(report_semaphore)], sizeof(report)); + const auto operation = report.query.operation.Value(); + if (operation != Regs::ReportSemaphore::Operation::Release && + operation != Regs::ReportSemaphore::Operation::ReportOnly) { + return; + } + const auto device = memory_manager.GpuToCpuAddress(report.Address()); + if (!device) { + return; + } + const bool payload = report.query.report == Regs::ReportSemaphore::Report::Payload; + split.NoteSemaphoreWrite(*device, report.payload, payload); + if (report.query.short_query == 0) { + // 64 bits (a timestamp follows). + split.NoteSemaphoreWrite(*device + 4, 0, payload); + } } void Maxwell3D::FrontCallMethod(Core::System& system, VideoCommon::GpuSplit& split, u32 method, @@ -1317,6 +1342,9 @@ void Maxwell3D::FrontCallMultiMethod(Core::System& system, VideoCommon::GpuSplit } } split.MultiMethod(this, method, base_start, amount, methods_pending, nullptr); + if (method == MAXWELL3D_REG_INDEX(report_semaphore.query) && amount != 0) { + FrontNoteReport(split, fs); + } } void Maxwell3D::FrontRunMacro(Core::System& system, VideoCommon::GpuSplit& split, diff --git a/src/video_core/engines/maxwell_3d.h b/src/video_core/engines/maxwell_3d.h index 55d3ab69ce..1fda4a9f2b 100644 --- a/src/video_core/engines/maxwell_3d.h +++ b/src/video_core/engines/maxwell_3d.h @@ -3409,6 +3409,7 @@ class Maxwell3D final : public EngineInterface { bool FrontSkipsLoopWrite(FrontState& state, u32 method, u32 value, bool shadow_kept); void FrontDmaRead(VideoCommon::GpuSplit& split, FrontState& state, u32 macro, u32 method, u32 count); + void FrontNoteReport(VideoCommon::GpuSplit& split, const FrontState& state); void FrontVerify(VideoCommon::GpuSplit& split, FrontState& state); void FinishFrontMacro(Core::System& system); std::unique_ptr front; diff --git a/src/video_core/engines/maxwell_dma.h b/src/video_core/engines/maxwell_dma.h index 9cf2813d31..82ac274598 100644 --- a/src/video_core/engines/maxwell_dma.h +++ b/src/video_core/engines/maxwell_dma.h @@ -113,6 +113,9 @@ class MaxwellDMA final : public EngineInterface { PackedGPUVAddr address; u32 payload; }; + /// The methods of Regs::semaphore (address high and low, payload) and Regs::launch_dma. + static constexpr u32 SemaphoreMethod = 0x240 / sizeof(u32); + static constexpr u32 LaunchDmaMethod = 0x300 / sizeof(u32); static_assert(sizeof(Semaphore) == 12); struct RenderEnable { @@ -328,6 +331,8 @@ class MaxwellDMA final : public EngineInterface { ASSERT_REG_POSITION(src_params, 0x728); ASSERT_REG_POSITION(pm_trigger_end, 0x1114); #undef ASSERT_REG_POSITION + static_assert(SemaphoreMethod * sizeof(u32) == offsetof(Regs, semaphore)); + static_assert(LaunchDmaMethod * sizeof(u32) == offsetof(Regs, launch_dma)); }; } // namespace Tegra::Engines diff --git a/src/video_core/engines/puller.cpp b/src/video_core/engines/puller.cpp index 32426f2277..5b21c41612 100644 --- a/src/video_core/engines/puller.cpp +++ b/src/video_core/engines/puller.cpp @@ -80,6 +80,64 @@ void ReleaseFences(DmaPusher& dma_pusher) { VideoCommon::DrainSite::FenceRelease); } +// A release was just sent to the back: tell the front what it writes (CheckWithoutDrain). +void NoteRelease(DmaPusher& dma_pusher, GPUVAddr address, u32 payload, bool four_words) { + auto& split = VideoCommon::GpuSplit::Get(); + if (!split.Active()) { + return; + } + const auto device = dma_pusher.memory_manager.GpuToCpuAddress(address); + if (!device) { + return; + } + split.NoteSemaphoreWrite(*device, payload, true); + if (four_words) { + // The payload as 64 bits (a timestamp follows). + split.NoteSemaphoreWrite(*device + 4, 0, true); + } +} + +// Whether the back writes a payload release to memory when it runs it (CounterReport): below +// High GPU accuracy, with fences that run their operations at once (FenceManager::SignalFence). +bool ReleasesLandAtOnce() { + const bool delay_fence = + Settings::IsGPUFenceBehaviorDefault() + ? Settings::IsGPULevelHigh() + : Settings::IsGPUFenceBehaviorBalanced() || Settings::IsGPUFenceBehaviorAccurate(); + return !delay_fence && !Settings::IsGPULevelHigh(); +} + +enum class AcquireCheck { + Met, ///< By the word in memory + MetByQueued, ///< By a release sent to the word that the back has not run yet + NotMet, ///< Nothing sent to the word can meet it yet + Drain, ///< Drain the split and read the word as before +}; + +// Checks a greater-or-equal acquire without draining the split. The words Switch 2 syncs on only +// advance (NVN2 even releases QMD completions as maxima, KeplerCompute::ProcessNativeMethod), so +// a value that meets one now still does once the back ran the queued work. Only the last release +// the front sent to the word decides instead, when the back may not have run it yet. +AcquireCheck CheckWithoutDrain(DmaPusher& dma_pusher, const Puller::PendingAcquire& acquire) { + auto& split = VideoCommon::GpuSplit::Get(); + if (!acquire.WaitsToReach() || !split.Active() || !split.FrontMacros() || + !ReleasesLandAtOnce()) { + return AcquireCheck::Drain; + } + const auto device = dma_pusher.memory_manager.GpuToCpuAddress(acquire.address); + // A word the buffer cache may hold newer than memory is read through the back. + if (!device || Nx2MayBeGpuWritten(*device, sizeof(u32))) { + return AcquireCheck::Drain; + } + if (const auto queued = split.QueuedSemaphoreWriteAt(*device)) { + return queued->known && acquire.Matches(queued->payload) ? AcquireCheck::MetByQueued + : AcquireCheck::Drain; + } + return acquire.Matches(dma_pusher.memory_manager.Read(acquire.address)) + ? AcquireCheck::Met + : AcquireCheck::NotMet; +} + } // Anonymous namespace bool Puller::PendingAcquire::Matches(u32 current) const { @@ -95,10 +153,36 @@ bool Puller::PendingAcquire::Matches(u32 current) const { } } +bool Puller::PendingAcquire::WaitsToReach() const { + return static_cast(operation) == GpuSemaphoreOperation::AcquireGequal; +} + bool Puller::CanResumeAcquire(DmaPusher& dma_pusher) { if (!pending_acquire) { return true; } + switch (CheckWithoutDrain(dma_pusher, *pending_acquire)) { + case AcquireCheck::MetByQueued: + // What the front reads after the acquire waits for the work sent before the release. + VideoCommon::GpuSplit::Get().NoteSync(); + return true; + case AcquireCheck::Met: + return true; + case AcquireCheck::NotMet: + // A release the back ran latches the waiter. Otherwise poll again later, without waiting + // for the back: until it runs one, nothing sent to the word meets it. After 2 s, the + // drain below reports the wait. + if (Nx2SemaphoreWaiters::Get().Latched(this)) { + return true; + } + if (pending_acquire->reported || + std::chrono::steady_clock::now() - pending_acquire->since <= std::chrono::seconds{2}) { + return false; + } + break; + case AcquireCheck::Drain: + break; + } const bool check = VideoCommon::Nx2DrainCheck::g_enabled; const bool met_before = check && @@ -256,6 +340,8 @@ void Puller::ProcessSemaphoreTriggerMethod(DmaPusher& dma_pusher) { OnBack([rasterizer = dma_pusher.rasterizer, sequence_address, properties, payload] { rasterizer->Query(sequence_address, VideoCommon::QueryType::Payload, properties, payload, 0); }); + NoteRelease(dma_pusher, sequence_address, payload, + properties == VideoCommon::QueryPropertiesFlags::HasTimeout); } else if (g_nx2_ampere) { // Switch 2 waits on semaphores the CPU (or another channel) releases, e.g. once it has // written the commands this channel runs next: suspend the channel until the value is @@ -273,6 +359,15 @@ void Puller::ProcessSemaphoreTriggerMethod(DmaPusher& dma_pusher) { return; } VideoCommon::Nx2Profile::Add(VideoCommon::Nx2Profile::Counter::AcquireChecks, 1); + if (const AcquireCheck without_drain = CheckWithoutDrain(dma_pusher, acquire); + without_drain == AcquireCheck::Met || without_drain == AcquireCheck::MetByQueued) { + if (without_drain == AcquireCheck::MetByQueued) { + // What the front reads after the acquire waits for the work sent before the + // release (GpuSplit::NoteSync). + VideoCommon::GpuSplit::Get().NoteSync(); + } + return; + } const bool check = VideoCommon::Nx2DrainCheck::g_enabled; const bool met_before = check && acquire.Matches(dma_pusher.memory_manager.Read(acquire.address)); @@ -337,6 +432,7 @@ void Puller::ProcessSemaphoreRelease(DmaPusher& dma_pusher) { rasterizer->Query(sequence_address, VideoCommon::QueryType::Payload, VideoCommon::QueryPropertiesFlags::IsAFence, payload, 0); }); + NoteRelease(dma_pusher, sequence_address, payload, false); } void Puller::ProcessSemaphoreAcquire(DmaPusher& dma_pusher) { @@ -366,6 +462,7 @@ void Puller::ProcessSemaphoreExecute(DmaPusher& dma_pusher, u32 argument) { rasterizer->Query(address, VideoCommon::QueryType::Payload, VideoCommon::QueryPropertiesFlags::IsAFence, payload, 0); }); + NoteRelease(dma_pusher, address, payload, false); return; } // Acquires: check once after flushing pending fences; never block the GPU thread on them. diff --git a/src/video_core/engines/puller.h b/src/video_core/engines/puller.h index 2b9e2c5df3..fe6874d789 100644 --- a/src/video_core/engines/puller.h +++ b/src/video_core/engines/puller.h @@ -81,6 +81,8 @@ class Puller final { bool reported; [[nodiscard]] bool Matches(u32 current) const; + /// Whether it waits for the word to reach `value` (greater or equal). + [[nodiscard]] bool WaitsToReach() const; }; [[nodiscard]] bool HasPendingAcquire() const { diff --git a/src/video_core/gpu_split.cpp b/src/video_core/gpu_split.cpp index e148aba4fe..f7ee05a978 100644 --- a/src/video_core/gpu_split.cpp +++ b/src/video_core/gpu_split.cpp @@ -142,6 +142,8 @@ void GpuSplit::Stop() { write_position = 0; last_writer_position = 0; last_sync_position = 0; + semaphore_writes.clear(); + semaphore_writes_limit = 1024; published_local = 0; consumed_cache = 0; publish_every = PublishEvery; @@ -233,6 +235,28 @@ bool GpuSplit::WaitForSyncedWriters() { DrainSite::ListWriters); } +void GpuSplit::NoteSemaphoreWrite(u64 device_address, u32 payload, bool known) { + if (semaphore_writes.size() >= semaphore_writes_limit) { + // The words of releases the back ran hold their values in memory. + const u64 done = consumed.load(std::memory_order_acquire); + std::erase_if(semaphore_writes, + [done](const auto& entry) { return entry.second.position <= done; }); + semaphore_writes_limit = std::max(1024, semaphore_writes.size() * 2); + } + semaphore_writes.insert_or_assign(device_address & ~u64{3}, + SemaphoreWrite{write_position, payload, known}); +} + +std::optional GpuSplit::QueuedSemaphoreWriteAt( + u64 device_address) const { + const auto it = semaphore_writes.find(device_address & ~u64{3}); + if (it == semaphore_writes.end() || + it->second.position <= consumed.load(std::memory_order_acquire)) { + return std::nullopt; + } + return QueuedSemaphoreWrite{it->second.payload, it->second.known}; +} + bool GpuSplit::WritersPending() const { return consumed.load(std::memory_order_acquire) < std::min(last_writer_position, write_position); diff --git a/src/video_core/gpu_split.h b/src/video_core/gpu_split.h index 3e31cd668f..f65de753f3 100644 --- a/src/video_core/gpu_split.h +++ b/src/video_core/gpu_split.h @@ -8,8 +8,10 @@ #include #include #include +#include #include #include +#include #include "common/common_types.h" @@ -185,8 +187,8 @@ class GpuSplit { /// Returns whether any of it was still queued. bool WaitForWriters(); /// A host-level wait was just sent (a syncpoint acquire, a host WFI, a GP entry with its sync - /// bit; semaphore acquires drain): the guest's GPU work before it is complete before the - /// commands after it are fetched. + /// bit; semaphore acquires drain, or note it when a release still queued meets them): the + /// guest's GPU work before it is complete before the commands after it are fetched. void NoteSync() { last_sync_position = write_position; } @@ -197,6 +199,17 @@ class GpuSplit { /// the writers a command list fetched now is ordered after. Returns whether any of it was /// still queued. bool WaitForSyncedWriters(); + /// A semaphore release was just sent: once the back ran it, the word at `device_address` holds + /// `payload`, or a value the front can't tell (`known` false: a reduction, a counter report). + void NoteSemaphoreWrite(u64 device_address, u32 payload, bool known); + struct QueuedSemaphoreWrite { + u32 payload; + bool known; + }; + /// The last write sent to the semaphore word at `device_address`, if the back may not have run + /// it yet. + [[nodiscard]] std::optional QueuedSemaphoreWriteAt( + u64 device_address) const; /// Whether compute or copy work sent so far is still queued (EDEN_NX2_DRAIN_CHECK). [[nodiscard]] bool WritersPending() const; @@ -280,6 +293,14 @@ class GpuSplit { u64 write_position{}; u64 last_writer_position{}; u64 last_sync_position{}; + struct SemaphoreWrite { + u64 position; ///< write_position once it was sent + u32 payload; + bool known; + }; + /// The front's NoteSemaphoreWrite, by word address. + std::unordered_map semaphore_writes; + size_t semaphore_writes_limit = 1024; ///< Forget what the back ran past this many u64 published_local{}; u64 consumed_cache{}; /// Words between publishes: small right after a drain, while the back has nothing to do. From 34659468f21bf6c006740a7dbab91137bc7ff7a8 Mon Sep 17 00:00:00 2001 From: jpkto <255363096+jpkto@users.noreply.github.com> Date: Sat, 7 Nov 2026 21:11:09 +0100 Subject: [PATCH 014/105] TEST: verify acquires decided without the drain (EDEN_NX2_DRAIN_CHECK) With EDEN_NX2_DRAIN_CHECK set, every acquire or poll of a suspended one that the previous commit decides without draining the split is checked against the drain path anyway: the front drains, reads the word as before (and, for a poll, asks whether a release latched the waiter). The 10 s "Drain checks" line gains: without the drain: acquires passed N (by a release still queued Q), not met after the drain W; polls resumed N (by a release still queued Q), not met after the drain W, kept waiting K (met after the drain M) W must stay 0: a pass the drained word does not meet. Such a decision is corrected in this mode (it takes the drain path), so the run stays right while it counts. M are polls that resume one poll later than a drain would have resumed them. Unset, each site tests a constant. Co-Authored-By: Claude Opus 5.5 --- src/video_core/engines/puller.cpp | 37 +++++++++++++++++++++++- src/video_core/nx2_drain_check.h | 48 +++++++++++++++++++++++++++++-- 2 files changed, 81 insertions(+), 4 deletions(-) diff --git a/src/video_core/engines/puller.cpp b/src/video_core/engines/puller.cpp index 5b21c41612..b0eafabb93 100644 --- a/src/video_core/engines/puller.cpp +++ b/src/video_core/engines/puller.cpp @@ -138,6 +138,27 @@ AcquireCheck CheckWithoutDrain(DmaPusher& dma_pusher, const Puller::PendingAcqui : AcquireCheck::NotMet; } +// EDEN_NX2_DRAIN_CHECK: what the drain path decides instead, to verify a decision made without it +// (a release the back ran latches `waiter`, a suspended acquire's). +bool MetAfterDrain(DmaPusher& dma_pusher, const Puller::PendingAcquire& acquire, + const void* waiter) { + DrainBack(); + return acquire.Matches(ReadSemaphoreWord(dma_pusher, acquire.address)) || + (waiter != nullptr && Nx2SemaphoreWaiters::Get().Latched(waiter)); +} + +// Whether a pass without the drain stands. Verified, a pass the drained word contradicts takes +// the drain path instead. +bool PassStands(DmaPusher& dma_pusher, const Puller::PendingAcquire& acquire, bool queued, + const void* waiter) { + if (!VideoCommon::Nx2DrainCheck::g_enabled) { + return true; + } + const bool met = MetAfterDrain(dma_pusher, acquire, waiter); + VideoCommon::Nx2DrainCheck::NotePass(waiter != nullptr, queued, met); + return met; +} + } // Anonymous namespace bool Puller::PendingAcquire::Matches(u32 current) const { @@ -163,10 +184,16 @@ bool Puller::CanResumeAcquire(DmaPusher& dma_pusher) { } switch (CheckWithoutDrain(dma_pusher, *pending_acquire)) { case AcquireCheck::MetByQueued: + if (!PassStands(dma_pusher, *pending_acquire, true, this)) { + break; + } // What the front reads after the acquire waits for the work sent before the release. VideoCommon::GpuSplit::Get().NoteSync(); return true; case AcquireCheck::Met: + if (!PassStands(dma_pusher, *pending_acquire, false, this)) { + break; + } return true; case AcquireCheck::NotMet: // A release the back ran latches the waiter. Otherwise poll again later, without waiting @@ -175,6 +202,13 @@ bool Puller::CanResumeAcquire(DmaPusher& dma_pusher) { if (Nx2SemaphoreWaiters::Get().Latched(this)) { return true; } + if (VideoCommon::Nx2DrainCheck::g_enabled) [[unlikely]] { + const bool met = MetAfterDrain(dma_pusher, *pending_acquire, this); + VideoCommon::Nx2DrainCheck::NoteKeptWaiting(met); + if (met) { + return true; + } + } if (pending_acquire->reported || std::chrono::steady_clock::now() - pending_acquire->since <= std::chrono::seconds{2}) { return false; @@ -360,7 +394,8 @@ void Puller::ProcessSemaphoreTriggerMethod(DmaPusher& dma_pusher) { } VideoCommon::Nx2Profile::Add(VideoCommon::Nx2Profile::Counter::AcquireChecks, 1); if (const AcquireCheck without_drain = CheckWithoutDrain(dma_pusher, acquire); - without_drain == AcquireCheck::Met || without_drain == AcquireCheck::MetByQueued) { + (without_drain == AcquireCheck::Met || without_drain == AcquireCheck::MetByQueued) && + PassStands(dma_pusher, acquire, without_drain == AcquireCheck::MetByQueued, nullptr)) { if (without_drain == AcquireCheck::MetByQueued) { // What the front reads after the acquire waits for the work sent before the // release (GpuSplit::NoteSync). diff --git a/src/video_core/nx2_drain_check.h b/src/video_core/nx2_drain_check.h index ec79c4665f..85b63bab81 100644 --- a/src/video_core/nx2_drain_check.h +++ b/src/video_core/nx2_drain_check.h @@ -23,7 +23,10 @@ // GPU-written, checked on the back at the list's place. It must stay 0; // - semaphore acquires, and the polls of suspended ones (Puller): whether the value met the // acquire before the front drained the split and after, by whether the GPU had released a -// semaphore at that address before (any engine, RasterizerInterface::Query with a payload). +// semaphore at that address before (any engine, RasterizerInterface::Query with a payload); +// - acquires and polls decided without the drain (Puller CheckWithoutDrain), verified: the front +// drains and reads the word as before anyway, and counts the decisions the drain contradicts. +// A pass the drained word does not meet must stay 0; such a decision is corrected. namespace VideoCommon::Nx2DrainCheck { // A plain flag: the front checks it at every command list and acquire. @@ -40,6 +43,12 @@ struct State { std::atomic lists_written{}; /// Indexed by poll * 8 + released * 4 + met before the drain * 2 + met after it. std::array, 16> acquires{}; + /// Passes without the drain, by poll * 2 + met by a release still queued, and those the + /// drained word did not meet. + std::array, 4> passes{}; + std::array, 4> wrong_passes{}; + std::atomic polls_kept{}; ///< Polls that kept waiting without the drain + std::atomic polls_kept_met{}; ///< Of those, met after the drain std::mutex released_mutex; std::unordered_set released; ///< Device addresses of the semaphores the GPU released std::chrono::steady_clock::time_point last_report{}; ///< The front's @@ -71,6 +80,26 @@ inline void NoteAcquire(bool poll, bool released, bool met_before, bool met_afte Get().acquires[index].fetch_add(1, std::memory_order_relaxed); } +/// An acquire (or a poll, `poll`) passed without the drain, met by a release still queued +/// (`queued`) or by memory; whether the word read after a drain met it too. +inline void NotePass(bool poll, bool queued, bool met_after_drain) { + const size_t index = (poll ? 2U : 0U) + (queued ? 1U : 0U); + State& state = Get(); + state.passes[index].fetch_add(1, std::memory_order_relaxed); + if (!met_after_drain) { + state.wrong_passes[index].fetch_add(1, std::memory_order_relaxed); + } +} + +/// A poll kept waiting without the drain; whether the word read after a drain met it. +inline void NoteKeptWaiting(bool met_after_drain) { + State& state = Get(); + state.polls_kept.fetch_add(1, std::memory_order_relaxed); + if (met_after_drain) { + state.polls_kept_met.fetch_add(1, std::memory_order_relaxed); + } +} + /// The front fetches a list without waiting for writers still queued after the last sync. inline void NoteListUnwaited() { Get().lists_unwaited.fetch_add(1, std::memory_order_relaxed); @@ -113,19 +142,32 @@ inline void MaybeReport() { neither + only_after + only_before + both, both, only_after, only_before, neither); }; + const auto passes = [&](bool poll) { + const size_t base = poll ? 2U : 0U; + const u64 by_memory = take(state.passes[base]); + const u64 by_queued = take(state.passes[base + 1]); + const u64 wrong = take(state.wrong_passes[base]) + take(state.wrong_passes[base + 1]); + return fmt::format("{} (by a release still queued {}), not met after the drain {}", + by_memory + by_queued, by_queued, wrong); + }; const u64 unwaited = take(state.lists_unwaited); const u64 written = take(state.lists_written); const std::string acquires_never = group(false, false); const std::string acquires_released = group(false, true); const std::string polls_never = group(true, false); const std::string polls_released = group(true, true); + const std::string acquire_passes = passes(false); + const std::string poll_passes = passes(true); + const u64 kept = take(state.polls_kept); + const u64 kept_met = take(state.polls_kept_met); LOG_INFO(HW_GPU, "Drain checks in {:.1f} s: command lists that did not wait for writers sent after the " "last sync {}, GPU-written by those writers {} | acquires at addresses the GPU never " "released {}, at ones it released {} | polls of suspended acquires at addresses never " - "released {}, at released ones {}", + "released {}, at released ones {} | without the drain: acquires passed {}; polls " + "resumed {}, kept waiting {} (met after the drain {})", seconds, unwaited, written, acquires_never, acquires_released, polls_never, - polls_released); + polls_released, acquire_passes, poll_passes, kept, kept_met); } } // namespace VideoCommon::Nx2DrainCheck From 1d9b6ef31d7c271fcc955094cf37806478e2e3c6 Mon Sep 17 00:00:00 2001 From: jpkto <255363096+jpkto@users.noreply.github.com> Date: Sat, 7 Nov 2026 21:50:56 +0100 Subject: [PATCH 015/105] video_core: wait only for the work that writes what a macro DMA reads When a macro on the split's front reads memory (MME_DMA_READ and MME_DMA_READ_FIFOED, FrontDmaRead), the front drained the whole split (maxwell_3d.cpp:1431 before this change), had the back download what the buffer cache holds of the words (line 1449) and write out the reports the query cache holds there (line 1456), then read them. In Ocarina of Time's Hyrule Field (EDEN_NX2_GPU_PROFILE, about 50 fps) that was 25,500 reads a second, all by macro 0xe2c (24 bytes, a ReportSemaphore::Compare: the render enable condition of two reports), at 9.6 us each: 246 ms of every second of the front, which runs in near lockstep with the back. The words were written long before almost every time: occlusion reports a frame earlier, status reports some objects back. EDEN_NX2_CMDPAGE_DIAG found 0.3% of the reads changed by the drain. What a drained read returns is guest memory once the back ran every entry before it, plus what those two caches write out. Only entries that write guest memory there, make it GPU-written or leave reports for the query cache can change that. The 3D engine's front sends nearly all entries and knows where its own writes go, so it notes them as it sends them (FrontNoteWrites, maxwell_3d.cpp:1429): - constant buffer data at the constant buffer address and offset, a report or semaphore release at the report address (CounterReport writes its value and timestamp, now or once the host result is in), and a pitch inline upload over its lines are placed at their addresses; - a draw, or any entry the back runs while one may be pending in the draw manager (an instance loop's deferred draw is issued by the next register write, maxwell_3d.cpp:361; FrontState::draw_pending, line 1045), a block linear upload, a render enable condition read from memory (ProcessQueryCondition reads it through every cache, line 864) and a syncpoint increment (macros read syncpoints through memory too, memory_manager.cpp:306) count as writes anywhere; - the other executable methods change registers and host state only, and state writes the back sinks run nothing yet. Every other entry, from the other engines, the puller or deferred calls, counts as a write anywhere unless its sender says otherwise (GpuSplit::Writes). GpuSplit::WaitForWrites (gpu_split.cpp:301) waits for the last entry that may write anywhere and the last one placed in the words read. It places writes by the device memory the GPU addresses map to, so the same memory under another GPU address is found: GPU maps drain the split before later commands (GPU::MapInOrder, gpu_thread.cpp:128), and the unmaps it is sent are deferred calls. Past those entries no queued work can make the words GPU-written or add reports there, so the caches hold now what they would after a drain. The front checks them where it can without the back: the buffer cache as before (IsMemoryDirty), and the query cache under its lock for a report whose value or timestamp reaches into the words (HoldsQueries, query_cache_base.h:96). When either holds them, it drains and has them written out as before; otherwise it reads the words at once. The fencing thread writes only reports the cache holds, and those reads drain. The split test checks that a macro's DMA read on the front goes ahead of other queued work and of writes elsewhere, waits for a write of its words under another GPU address and sees it, waits for work that does not note its writes, and where writes are placed. Co-Authored-By: Claude Opus 5.5 --- src/tests/video_core/maxwell_fast_method.cpp | 216 ++++++++++++++++++ src/video_core/engines/maxwell_3d.cpp | 139 ++++++++--- src/video_core/engines/maxwell_3d.h | 2 + src/video_core/gpu_split.cpp | 71 +++++- src/video_core/gpu_split.h | 72 +++++- src/video_core/query_cache/query_cache_base.h | 28 +++ src/video_core/query_dma.h | 17 ++ src/video_core/rasterizer_interface.h | 8 + .../renderer_null/null_rasterizer.h | 4 + .../renderer_vulkan/vk_rasterizer.cpp | 4 + .../renderer_vulkan/vk_rasterizer.h | 1 + 11 files changed, 527 insertions(+), 35 deletions(-) diff --git a/src/tests/video_core/maxwell_fast_method.cpp b/src/tests/video_core/maxwell_fast_method.cpp index ac901a322e..1cf786d4bf 100644 --- a/src/tests/video_core/maxwell_fast_method.cpp +++ b/src/tests/video_core/maxwell_fast_method.cpp @@ -66,6 +66,18 @@ struct GpuSplitTestAccess { &split.ring[index], std::min(available - position, GpuSplit::RingWords - index)); return position + (size != 0 ? size : GpuSplit::RingWords - index); } + + /// The position a read waits for the back to reach (WaitForWrites). + static u64 WritesTarget(const GpuSplit& split, const Tegra::MemoryManager& memory, + GPUVAddr address, u64 size) { + return split.WritesTarget(memory, address, size); + } + + /// Whether writes to two device addresses share a slot of the noted write positions. + static bool ShareWriteSlot(DAddr a, DAddr b) { + return GpuSplit::WriteSlot(a >> GpuSplit::WriteGranuleBits) == + GpuSplit::WriteSlot(b >> GpuSplit::WriteGranuleBits); + } }; } // namespace VideoCommon @@ -204,6 +216,27 @@ u32 RandomArgument(std::mt19937& rng, u32 method, const Maxwell3D& engine) { return rng() % 64; } +/// A Turing MME instruction that sends the next macro parameter to `method` (as in +/// native_macro_fifo.cpp); `end` makes the instruction after it the macro's last. +std::array FifoInstruction(u32 method, bool end) { + std::array bits{}; + const auto field = [&](u32 offset, u32 count, u32 value) { + for (u32 bit = 0; bit < count; ++bit) { + bits[(offset + bit) / 32] |= ((value >> bit) & 1) << ((offset + bit) % 32); + } + }; + field(0, 1, end); + for (u32 lane = 0; lane < 2; ++lane) { + field(15 + lane * 36, 5, 24); + field(20 + lane * 36, 5, 24); + field(25 + lane * 36, 5, 24); + } + field(30, 16, method); + field(82, 3, 5); + field(85, 4, 3); + return {bits[2], bits[1], bits[0]}; +} + } // Anonymous namespace TEST_CASE("Maxwell3D fast method dispatch matches CallMethod on random method streams", @@ -1062,3 +1095,186 @@ TEST_CASE("Maxwell3D's split front uploads macro code sent as one multi-method r REQUIRE(macros.native_write_offset == pointer * 3 + code.size()); CHECK(std::equal(code.begin(), code.end(), macros.native_code.begin() + pointer * 3)); } + +TEST_CASE("Split macro DMA reads wait only for the queued work that can write the words", + "[video_core][gpu_split_dependencies]") { + using Access = VideoCommon::GpuSplitTestAccess; + using Writes = VideoCommon::GpuSplit::Writes; + Fixture f; + f.system.Initialize(); + Kernel::KProcess process{f.system.Kernel()}; + SCOPE_EXIT { + f.system.Kernel().UnregisterKernelObject(&process); + }; + auto& table = process.GetPageTable().GetImpl(); + table.Resize(20, 12); + constexpr VAddr cpu_address = 0x4000; + constexpr DAddr device_address = 0x10000; + constexpr GPUVAddr gpu_address = 0x100000; + constexpr GPUVAddr alias = 0x200000; // The first page again + auto* backing = f.physical.GetPointerFromRaw(0x10000); + for (u64 page = 0; page < 2; ++page) { + const VAddr address = cpu_address + page * 0x1000; + table.entries.GetAndFault(address >> 12) + .Store(false, Common::PageType::Memory, 1, + reinterpret_cast(backing + page * 0x1000) - address); + } + process.GetMemory().SetCurrentPageTable(process); + const auto asid = f.device_memory.RegisterProcess(&process.GetMemory()); + f.device_memory.BindInterface(&f.rasterizer); + f.device_memory.Map(device_address, cpu_address, 0x2000, asid); + SCOPE_EXIT { + f.device_memory.Unmap(device_address, 0x2000); + f.device_memory.UnregisterProcess(asid); + }; + f.memory.BindRasterizer(&f.rasterizer); + f.memory.Map(gpu_address, device_address, 0x2000, Tegra::PTEKind::PITCH, false); + f.memory.Map(alias, device_address, 0x1000, Tegra::PTEKind::PITCH, false); + std::memset(backing, 0, 0x2000); + // The words read below, in two 256-byte granules that do not share a write slot. + constexpr GPUVAddr word = gpu_address + 0x40; + constexpr GPUVAddr other = gpu_address + 0x140; + REQUIRE_FALSE(Access::ShareWriteSlot(device_address + 0x40, device_address + 0x140)); + + { + // Where reads wait: writes are placed by device memory, any GPU address of it. + VideoCommon::GpuSplit split; + Access::Initialize(split, f.system); + split.Defer([] {}, Writes::Noted); + split.NoteWrite(f.memory, alias + 0x48, 4); + const u64 noted = Access::End(split); + split.Defer([] {}, Writes::Noted); + split.NoteWrite(f.memory, 0x300000, 16); // Unmapped: nothing is written + CHECK(Access::WritesTarget(split, f.memory, word, 0x18) == noted); + CHECK(Access::WritesTarget(split, f.memory, other, 0x18) == 0); + // Writes too long to place, and work nobody noted the writes of, count as anywhere. + split.Defer([] {}, Writes::Noted); + split.NoteWrite(f.memory, gpu_address + 0x1000, 0x8000); + CHECK(Access::WritesTarget(split, f.memory, other, 0x18) == Access::End(split)); + split.Defer([] {}); + CHECK(Access::WritesTarget(split, f.memory, other, 0x18) == Access::End(split)); + } + + // The 3D engine's front, against a running back: macro 0 reads one word with + // MME_DMA_READ_FIFOED at the address in its first two parameters and sends it to a scratch + // register (the load after the read takes the word). + auto& engine = f.fast; + engine.SetNativeTuringMacroMode(true); + constexpr u32 state = MAXWELL3D_REG_INDEX(shadow_scratch) + 1; + VideoCommon::GpuSplit split; + Access::Start(split, f.system); + SCOPE_EXIT { + split.Stop(); + }; + std::array, 3> release{}; + std::atomic done{}; + std::thread reader; + SCOPE_EXIT { + for (auto& flag : release) { + flag = true; + } + if (reader.joinable()) { + reader.join(); + } + }; + const auto front = [&](u32 method, u32 argument) { + engine.FrontCallMethod(f.system, split, method, argument, true); + }; + std::vector code; + for (const auto instruction : {FifoInstruction(0x154, false), FifoInstruction(0x155, false), + FifoInstruction(0x158, true), FifoInstruction(state, false)}) { + code.insert(code.end(), instruction.begin(), instruction.end()); + } + front(MAXWELL3D_REG_INDEX(load_mme.instruction_ptr), 0); + engine.FrontCallMultiMethod(f.system, split, MAXWELL3D_REG_INDEX(load_mme.instruction), + code.data(), static_cast(code.size()), + static_cast(code.size())); + front(MAXWELL3D_REG_INDEX(load_mme.start_address_ptr), 0); + front(MAXWELL3D_REG_INDEX(load_mme.start_address), 0); + const auto dma_read = [&](GPUVAddr address) { + engine.FrontCallMethod(f.system, split, 0xe00, static_cast(address >> 32), false); + engine.FrontCallMethod(f.system, split, 0xe01, static_cast(address), false); + engine.FrontCallMethod(f.system, split, 0xe01, 1, true); + }; + const auto read_on_reader = [&](GPUVAddr address) { + done = false; + reader = std::thread([&, address] { + dma_read(address); + done = true; + }); + }; + const auto finishes = [&done] { + const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds{5}; + while (!done.load() && std::chrono::steady_clock::now() < deadline) { + std::this_thread::sleep_for(std::chrono::milliseconds{1}); + } + return done.load(); + }; + const auto waits = [&] { + const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds{5}; + while (!Access::FrontWaits(split) && !done.load() && + std::chrono::steady_clock::now() < deadline) { + std::this_thread::yield(); + } + std::this_thread::sleep_for(std::chrono::milliseconds{20}); + return !done.load(); + }; + // A constant buffer word, sent through the front. + const auto upload = [&](GPUVAddr address, u32 value) { + front(CbAddressHigh, static_cast(address >> 32)); + front(CbAddressLow, static_cast(address)); + front(CbSize, 0x100); + front(CbOffset, 0); + front(CbData, value); + }; + // An entry the back stays in until its flag is set. + const auto blocking = [&split](std::atomic& flag, Writes writes) { + split.Defer( + [&flag] { + while (!flag.load()) { + std::this_thread::yield(); + } + }, + writes); + split.Publish(); + }; + + // The first macro also ends the front's assumption that a draw may be pending. + upload(word, 0x11111111); + upload(other, 0x22222222); + dma_read(word); + split.Drain(VideoCommon::DrainSite::Other); + REQUIRE(engine.regs.reg_array[state] == 0x11111111); + + // Work that writes nothing and a write of other words, still queued: the read goes ahead. + blocking(release[0], Writes::Noted); + upload(other, 0x33333333); + read_on_reader(word); + REQUIRE(finishes()); + reader.join(); + release[0] = true; + split.Drain(VideoCommon::DrainSite::Other); + CHECK(engine.regs.reg_array[state] == 0x11111111); + CHECK(f.memory.Read(other) == 0x33333333); + + // A write of the word through its other GPU address: the read waits for it and sees it. + blocking(release[1], Writes::Noted); + upload(alias + 0x40, 0x44444444); + read_on_reader(word); + REQUIRE(waits()); + release[1] = true; + REQUIRE(finishes()); + reader.join(); + split.Drain(VideoCommon::DrainSite::Other); + CHECK(engine.regs.reg_array[state] == 0x44444444); + + // Work whose writes are not noted: the read waits for it. + blocking(release[2], Writes::Anywhere); + read_on_reader(other); + REQUIRE(waits()); + release[2] = true; + REQUIRE(finishes()); + reader.join(); + split.Drain(VideoCommon::DrainSite::Other); + CHECK(engine.regs.reg_array[state] == 0x33333333); +} diff --git a/src/video_core/engines/maxwell_3d.cpp b/src/video_core/engines/maxwell_3d.cpp index 142c8573cd..01d2439267 100644 --- a/src/video_core/engines/maxwell_3d.cpp +++ b/src/video_core/engines/maxwell_3d.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -1205,10 +1206,15 @@ void Maxwell3D::FrontWrite(VideoCommon::GpuSplit& split, FrontState& fs, u32 met break; } } + using Writes = VideoCommon::GpuSplit::Writes; if (from_macro) { - split.MacroWrite(this, method, argument); + split.MacroWrite(this, method, argument, Writes::Noted); } else { - split.Method(this, method, argument, is_last_call, nullptr); + split.Method(this, method, argument, is_last_call, nullptr, Writes::Noted); + } + // A write the back only keeps for later (its method sink) runs nothing there yet. + if (from_macro || execution_mask[method]) { + FrontNoteWrites(split, fs, method, 1); } if (method == MAXWELL3D_REG_INDEX(report_semaphore.query)) { FrontNoteReport(split, fs); @@ -1341,7 +1347,9 @@ void Maxwell3D::FrontCallMultiMethod(Core::System& system, VideoCommon::GpuSplit } } } - split.MultiMethod(this, method, base_start, amount, methods_pending, nullptr); + split.MultiMethod(this, method, base_start, amount, methods_pending, nullptr, + VideoCommon::GpuSplit::Writes::Noted); + FrontNoteWrites(split, fs, method, amount); if (method == MAXWELL3D_REG_INDEX(report_semaphore.query) && amount != 0) { FrontNoteReport(split, fs); } @@ -1439,10 +1447,76 @@ bool Maxwell3D::FrontSkipsLoopWrite(FrontState& fs, u32 method, u32 value, bool return method != MAXWELL3D_REG_INDEX(draw.end) || fs.instance_loop; } +// What the back can write to guest memory running the entry just sent, `amount` words of `method` +// (GpuSplit::Writes::Noted), for the macro DMA reads after it (FrontDmaRead). The executable +// methods (IsMethodExecutable) not named here change registers and host state only: bindings, +// barriers, counter resets, the WFI's host query copies, shadow RAM and macro code. +void Maxwell3D::FrontNoteWrites(VideoCommon::GpuSplit& split, const FrontState& fs, u32 method, + u32 amount) { + constexpr u32 cb_data = MAXWELL3D_REG_INDEX(const_buffer.buffer); + const auto address = [&fs](u32 high) { + return (GPUVAddr{fs.regs[high]} << 32) | fs.regs[high + 1]; + }; + if (fs.draw_pending) { + // A draw, or one an instance loop deferred to this entry (ProcessDirtyRegisters): where + // its shaders store, and what the caches download for it, only the back knows. + split.NoteAnyWrite(); + return; + } + if (method >= cb_data && method < cb_data + 16) { + // FrontWrite and FrontCallMultiMethod advanced the offset past these words already. + const u32 bytes = amount * static_cast(sizeof(u32)); + const u32 offset = fs.regs[MAXWELL3D_REG_INDEX(const_buffer.offset)] - bytes; + split.NoteWrite(memory_manager, + address(MAXWELL3D_REG_INDEX(const_buffer.address_high)) + offset, bytes); + return; + } + switch (method) { + case MAXWELL3D_REG_INDEX(report_semaphore.query): + // CounterReport: the value and its timestamp, now or once the host result is in. + split.NoteWrite(memory_manager, address(MAXWELL3D_REG_INDEX(report_semaphore.address_high)), + 16); + break; + case MAXWELL3D_REG_INDEX(inline_data): { + // Upload::State::ProcessData: lines of line_length_in bytes, pitch apart. + const auto launch = + std::bit_cast(fs.regs[MAXWELL3D_REG_INDEX(launch_dma)]); + const u32 lines = fs.regs[MAXWELL3D_REG_INDEX(upload.line_count)]; + const u32 length = fs.regs[MAXWELL3D_REG_INDEX(upload.line_length_in)]; + if (launch.memory_layout != Regs::LaunchDMA::Layout::Pitch) { + split.NoteAnyWrite(); // Swizzled into the destination surface + } else if (lines != 0 && length != 0) { + split.NoteWrite(memory_manager, address(MAXWELL3D_REG_INDEX(upload.dest.address_high)), + u64{fs.regs[MAXWELL3D_REG_INDEX(upload.dest.pitch)]} * (lines - 1) + + length); + } + break; + } + case MAXWELL3D_REG_INDEX(render_enable.mode): { + // A condition read from memory (ProcessQueryCondition) flushes every cache over it. + const auto mode = + static_cast(fs.regs[MAXWELL3D_REG_INDEX(render_enable.mode)]); + if (static_cast( + fs.regs[MAXWELL3D_REG_INDEX(render_enable_override)]) == + Regs::RenderEnable::Override::UseRenderEnable && + (mode == Regs::RenderEnable::Mode::Conditional || + mode == Regs::RenderEnable::Mode::IfEqual || + mode == Regs::RenderEnable::Mode::IfNotEqual)) { + split.NoteAnyWrite(); + } + break; + } + case MAXWELL3D_REG_INDEX(sync_info): + // A syncpoint increment: macros read syncpoints through memory too (MemoryManager::Read). + split.NoteAnyWrite(); + break; + default: + break; + } +} + void Maxwell3D::FrontDmaRead(VideoCommon::GpuSplit& split, FrontState& fs, u32 macro, u32 method, u32 count) { - // The words may be written by methods still queued for the back (indirect draw arguments - // from a culling dispatch), and the caches that may hold newer ones belong to it. const GPUVAddr source = (GPUVAddr{fs.regs[0x154] & 0x1ffffff} << 32) | fs.regs[0x155]; const u32 ram = fs.regs[0x156]; const u32 words = std::min(count, 0x1000); @@ -1453,34 +1527,47 @@ void Maxwell3D::FrontDmaRead(VideoCommon::GpuSplit& split, FrontState& fs, u32 m memory_manager.ReadBlockUnsafe(source, diag_before.data(), read_bytes); } VideoCommon::Nx2Profile::CountDmaRead(macro); - split.Drain(VideoCommon::DrainSite::MacroDmaRead); - if (VideoCommon::QueryDma::TraceEnabled()) { - split.Run( - [this, macro, method, source, read_bytes] { - if (VideoCommon::QueryDma::HasPendingQueries(memory_manager, *rasterizer, source, - read_bytes)) { - VideoCommon::QueryDma::Trace(memory_manager, *rasterizer, macro, method, - source, read_bytes); - } - }, - VideoCommon::DrainSite::MacroDmaFlush); - } if (VideoCommon::Nx2Profile::Enabled()) { VideoCommon::Nx2Profile::Add(VideoCommon::Nx2Profile::Counter::DmaReadBytes, read_bytes); } - if (memory_manager.IsMemoryDirty(source, read_bytes, VideoCommon::CacheType::BufferCache)) { + // Drains the split, and has the back write out what its caches hold of the words: data the + // GPU wrote (indirect draw arguments from a culling dispatch), reports the query cache holds + // until their fence or host result (else the read returns stale occlusion counts). + const auto write_out = [&] { + split.Drain(VideoCommon::DrainSite::MacroDmaRead); + if (VideoCommon::QueryDma::TraceEnabled()) { + split.Run( + [this, macro, method, source, read_bytes] { + if (VideoCommon::QueryDma::HasPendingQueries(memory_manager, *rasterizer, + source, read_bytes)) { + VideoCommon::QueryDma::Trace(memory_manager, *rasterizer, macro, method, + source, read_bytes); + } + }, + VideoCommon::DrainSite::MacroDmaFlush); + } + if (memory_manager.IsMemoryDirty(source, read_bytes, VideoCommon::CacheType::BufferCache)) { + split.Run( + [memory = &memory_manager, source, read_bytes] { + memory->FlushRegion(source, read_bytes, VideoCommon::CacheType::BufferCache); + }, + VideoCommon::DrainSite::MacroDmaFlush); + } split.Run( - [memory = &memory_manager, source, read_bytes] { - memory->FlushRegion(source, read_bytes, VideoCommon::CacheType::BufferCache); + [this, source, read_bytes] { + VideoCommon::QueryDma::FlushPending(memory_manager, *rasterizer, source, + read_bytes); }, VideoCommon::DrainSite::MacroDmaFlush); + }; + // Only the work still queued that can write the words has to run first (FrontNoteWrites, + // GpuSplit::WaitForWrites). Past it no entry can make them GPU-written or add reports there, + // so the caches have nothing to write out unless they already hold it now. + split.WaitForWrites(memory_manager, source, read_bytes); + if (memory_manager.IsMemoryDirty(source, read_bytes, VideoCommon::CacheType::BufferCache) || + VideoCommon::QueryDma::HoldsReports(memory_manager, *rasterizer, source, read_bytes)) { + write_out(); } - // Write out the reports held in the query cache, or the read returns stale occlusion counts. - split.Run( - [this, source, read_bytes] { - VideoCommon::QueryDma::FlushPending(memory_manager, *rasterizer, source, read_bytes); - }, - VideoCommon::DrainSite::MacroDmaFlush); u32 changed{}; for (u32 i = 0; i < words; ++i) { const u32 word = memory_manager.Read(source + u64{i} * sizeof(u32)); diff --git a/src/video_core/engines/maxwell_3d.h b/src/video_core/engines/maxwell_3d.h index 1fda4a9f2b..57d24164bc 100644 --- a/src/video_core/engines/maxwell_3d.h +++ b/src/video_core/engines/maxwell_3d.h @@ -3407,6 +3407,8 @@ class Maxwell3D final : public EngineInterface { void FrontRunMacro(Core::System& system, VideoCommon::GpuSplit& split, FrontState& state); void FrontTrackInstanceLoop(FrontState& state, u32 method, u32 value); bool FrontSkipsLoopWrite(FrontState& state, u32 method, u32 value, bool shadow_kept); + void FrontNoteWrites(VideoCommon::GpuSplit& split, const FrontState& state, u32 method, + u32 amount); void FrontDmaRead(VideoCommon::GpuSplit& split, FrontState& state, u32 macro, u32 method, u32 count); void FrontNoteReport(VideoCommon::GpuSplit& split, const FrontState& state); diff --git a/src/video_core/gpu_split.cpp b/src/video_core/gpu_split.cpp index f7ee05a978..ff88a7251f 100644 --- a/src/video_core/gpu_split.cpp +++ b/src/video_core/gpu_split.cpp @@ -144,6 +144,9 @@ void GpuSplit::Stop() { last_sync_position = 0; semaphore_writes.clear(); semaphore_writes_limit = 1024; + last_any_write = 0; + last_noted_write = 0; + write_positions.fill(0); published_local = 0; consumed_cache = 0; publish_every = PublishEvery; @@ -262,6 +265,68 @@ bool GpuSplit::WritersPending() const { std::min(last_writer_position, write_position); } +namespace { + +/// Calls `func(device_address, size)` for each mapped piece of [address, address + size) within +/// one 4 KiB GPU page; nothing is read or written where no device memory is mapped. +template +void ForEachDeviceRange(const Tegra::MemoryManager& memory, GPUVAddr address, u64 size, + Func&& func) { + constexpr GPUVAddr page_size = 0x1000; + const GPUVAddr end = address + size; + for (GPUVAddr begin = address; begin < end;) { + const GPUVAddr page_end = std::min((begin & ~(page_size - 1)) + page_size, end); + if (const auto device = memory.GpuToCpuAddress(begin)) { + func(*device, page_end - begin); + } + begin = page_end; + } +} + +} // Anonymous namespace + +void GpuSplit::NoteWrite(const Tegra::MemoryManager& memory, GPUVAddr address, u64 size) { + if (size > MaxNotedBytes || address + size < address) { + NoteAnyWrite(); + return; + } + ForEachDeviceRange(memory, address, size, + [this](DAddr device, u64 bytes) { NoteDeviceWrite(device, bytes); }); +} + +void GpuSplit::NoteDeviceWrite(DAddr address, u64 size) { + const u64 last = (address + size - 1) >> WriteGranuleBits; + for (u64 granule = address >> WriteGranuleBits; granule <= last; ++granule) { + write_positions[WriteSlot(granule)] = write_position; + } + last_noted_write = write_position; +} + +u64 GpuSplit::WritesTarget(const Tegra::MemoryManager& memory, GPUVAddr address, u64 size) const { + if (size > MaxNotedBytes || address + size < address) { + return std::max(last_any_write, last_noted_write); + } + u64 target = last_any_write; + ForEachDeviceRange(memory, address, size, [this, &target](DAddr device, u64 bytes) { + target = std::max(target, DeviceWritesTarget(device, bytes)); + }); + return target; +} + +u64 GpuSplit::DeviceWritesTarget(DAddr address, u64 size) const { + u64 target = 0; + const u64 last = (address + size - 1) >> WriteGranuleBits; + for (u64 granule = address >> WriteGranuleBits; granule <= last; ++granule) { + target = std::max(target, write_positions[WriteSlot(granule)]); + } + return target; +} + +bool GpuSplit::WaitForWrites(const Tegra::MemoryManager& memory, GPUVAddr address, u64 size) { + return WaitFor(std::min(WritesTarget(memory, address, size), write_position), + DrainSite::MacroDmaWrites); +} + bool GpuSplit::WaitFor(u64 target, DrainSite site) { consumed_cache = consumed.load(std::memory_order_acquire); if (consumed_cache >= target) { @@ -310,7 +375,8 @@ void GpuSplit::SwitchEngine(Tegra::Engines::EngineInterface* engine) { } void GpuSplit::MultiMethod(Tegra::Engines::EngineInterface* engine, u32 method, const u32* words, - u32 amount, u32 methods_pending, const GPUVAddr* segment) { + u32 amount, u32 methods_pending, const GPUVAddr* segment, + Writes writes) { SelectEngine(engine); // Long runs (constant buffer and inline uploads) go in pieces, as when a method's data // continues in the next GPFIFO entry. @@ -332,6 +398,9 @@ void GpuSplit::MultiMethod(Tegra::Engines::EngineInterface* engine, u32 method, amount -= piece; methods_pending -= piece; } + if (writes == Writes::Anywhere) { + NoteAnyWrite(); + } } void GpuSplit::DeferFunction(std::function&& func) { diff --git a/src/video_core/gpu_split.h b/src/video_core/gpu_split.h index f65de753f3..1378f6b49d 100644 --- a/src/video_core/gpu_split.h +++ b/src/video_core/gpu_split.h @@ -19,6 +19,10 @@ namespace Core { class System; } +namespace Tegra { +class MemoryManager; +} + namespace Tegra::Engines { class EngineInterface; } @@ -35,7 +39,8 @@ enum class DrainSite : u32 { ListWriters, ///< GPU-written command lists: the dispatches and copies sent before them ListSync, ///< Command lists on draw-written pages: the work before the last host sync ListFlush, ///< Command lists on GPU-written pages: drain and flush the pages - MacroDmaRead, ///< The front's macros read memory (MME DMA) + MacroDmaRead, ///< The front's macros read memory (MME DMA) the caches may still write + MacroDmaWrites, ///< Those reads: the queued work that can write the words (WaitForWrites) MacroDmaFlush, ///< Those words are in the buffer cache GpSyncFence, ///< A GP entry's sync bit (sync_memory_operations) Map, ///< GPU memory maps before later command reads @@ -45,9 +50,9 @@ enum class DrainSite : u32 { }; inline constexpr std::array(DrainSite::Count)> drain_site_names{ - "other", "fetch drain", "acquire", "semaphore flush", "fence release", - "list writers", "list sync", "list flush", "macro dma read", "macro dma flush", - "gp sync fence", "map", "front start", "front verify", + "other", "fetch drain", "acquire", "semaphore flush", "fence release", + "list writers", "list sync", "list flush", "macro dma read", "macro dma writes", + "macro dma flush", "gp sync fence", "map", "front start", "front verify", }; // Splits the GPU thread in two (NX2_GPU_THREAD_SPLIT.md). The GPU thread (the front) takes the @@ -118,10 +123,17 @@ class GpuSplit { void MaybeStart(Core::System& system); void Stop(); + /// What the sender of an entry tells about the guest memory the back writes running it, for + /// the front's reads of guest memory (WaitForWrites). + enum class Writes : u8 { + Anywhere, ///< Unknown: a read anywhere waits for the entry + Noted, ///< Nothing, or what the sender notes right after sending it (NoteWrite) + }; + /// An engine method write, executed as DmaPusher would (method sink, ConsumeSink, call). /// `segment`: the engine's current_dma_segment for the call, when the engine reads it. void Method(Tegra::Engines::EngineInterface* engine, u32 method, u32 argument, bool is_last, - const GPUVAddr* segment) { + const GPUVAddr* segment, Writes writes = Writes::Anywhere) { SelectEngine(engine); const size_t words = segment != nullptr ? 2 : 1; u64* const entry = Reserve(words); @@ -132,21 +144,29 @@ class GpuSplit { entry[1] = *segment; } Commit(words); + if (writes == Writes::Anywhere) { + NoteAnyWrite(); + } } void MultiMethod(Tegra::Engines::EngineInterface* engine, u32 method, const u32* words, - u32 amount, u32 methods_pending, const GPUVAddr* segment); + u32 amount, u32 methods_pending, const GPUVAddr* segment, + Writes writes = Writes::Anywhere); /// A write made by a macro: CallMethod without the method sink, as the engines' macros do. - void MacroWrite(Tegra::Engines::EngineInterface* engine, u32 method, u32 argument) { + void MacroWrite(Tegra::Engines::EngineInterface* engine, u32 method, u32 argument, + Writes writes = Writes::Anywhere) { SelectEngine(engine); u64* const entry = Reserve(1); entry[0] = static_cast(Kind::Method) | IsLast | Direct | (u64{method & 0xffff} << 8) | (u64{argument} << 32); Commit(1); + if (writes == Writes::Anywhere) { + NoteAnyWrite(); + } } /// Runs `func` on the back after everything sent before it. Must be trivially copyable. template - void Defer(const Func& func) { + void Defer(const Func& func, Writes writes = Writes::Anywhere) { static_assert(std::is_trivially_copyable_v && alignof(Func) <= alignof(u64)); constexpr size_t payload = (sizeof(Func) + sizeof(u64) - 1) / sizeof(u64); u64* const entry = Reserve(2 + payload); @@ -154,6 +174,9 @@ class GpuSplit { entry[1] = reinterpret_cast(&CallThunk); std::memcpy(entry + 2, &func, sizeof(Func)); Commit(2 + payload); + if (writes == Writes::Anywhere) { + NoteAnyWrite(); + } } void DeferFunction(std::function&& func); /// Runs `func` on the back and waits for it: for work on back-owned state, such as a buffer @@ -213,6 +236,18 @@ class GpuSplit { /// Whether compute or copy work sent so far is still queued (EDEN_NX2_DRAIN_CHECK). [[nodiscard]] bool WritersPending() const; + /// The entry just sent (Writes::Noted) writes [address, address + size) of `memory`. + void NoteWrite(const Tegra::MemoryManager& memory, GPUVAddr address, u64 size); + /// The entry just sent may write guest memory anywhere. + void NoteAnyWrite() { + last_any_write = write_position; + } + /// Waits until the back executed every entry sent so far that can write guest memory in + /// [address, address + size) of `memory`, not the others. Returns whether one was queued. + /// Writes are placed by the device memory the GPU addresses map to when noted and read: GPU + /// maps drain the split first (GPU::MapInOrder), the unmaps it is sent count as anywhere. + bool WaitForWrites(const Tegra::MemoryManager& memory, GPUVAddr address, u64 size); + private: friend struct GpuSplitTestAccess; @@ -232,6 +267,22 @@ class GpuSplit { static constexpr u32 MaxMultiWords = 8192; static constexpr size_t PublishEvery = 256; + /// NoteWrite places writes by 256-byte granule of device memory, up to MaxNotedBytes long + /// (longer ones count as anywhere), in write_positions slots hashed by granule: granules that + /// share a slot share its position, so a read may wait for a write elsewhere, never miss one. + static constexpr u32 WriteGranuleBits = 8; + static constexpr u64 MaxNotedBytes = 0x4000; + static constexpr u32 WriteSlotBits = 10; + + static size_t WriteSlot(u64 granule) { + return static_cast((granule * 0x9E3779B97F4A7C15ULL) >> (64 - WriteSlotBits)); + } + void NoteDeviceWrite(DAddr address, u64 size); + /// The position WaitForWrites waits for. + [[nodiscard]] u64 WritesTarget(const Tegra::MemoryManager& memory, GPUVAddr address, + u64 size) const; + [[nodiscard]] u64 DeviceWritesTarget(DAddr address, u64 size) const; + template static void CallThunk(const u64* payload) { (*reinterpret_cast(payload))(); @@ -301,6 +352,11 @@ class GpuSplit { /// The front's NoteSemaphoreWrite, by word address. std::unordered_map semaphore_writes; size_t semaphore_writes_limit = 1024; ///< Forget what the back ran past this many + /// After the last entry that may write guest memory anywhere (NoteAnyWrite), the last one + /// NoteWrite placed, and the last placed write in each slot's granules. + u64 last_any_write{}; + u64 last_noted_write{}; + std::array write_positions{}; u64 published_local{}; u64 consumed_cache{}; /// Words between publishes: small right after a drain, while the back has nothing to do. diff --git a/src/video_core/query_cache/query_cache_base.h b/src/video_core/query_cache/query_cache_base.h index a8b6b701b9..14678c5831 100644 --- a/src/video_core/query_cache/query_cache_base.h +++ b/src/video_core/query_cache/query_cache_base.h @@ -90,6 +90,34 @@ class QueryCacheBase : public VideoCommon::ChannelSetupCaches= report_size - 1 ? addr - (report_size - 1) : 0; + const u64 end = addr + size; + std::scoped_lock lock(cache_mutex); + for (u64 page = begin >> Core::DEVICE_PAGEBITS; page <= (end - 1) >> Core::DEVICE_PAGEBITS; + ++page) { + const auto it = cached_queries.find(page); + if (it == cached_queries.end()) { + continue; + } + const u64 page_start = page << Core::DEVICE_PAGEBITS; + for (const auto& entry : it->second) { + const u64 query = page_start + entry.first; + if (query >= begin && query < end) { + return true; + } + } + } + return false; + } + /// Return true when a CPU region is modified from the GPU [[nodiscard]] bool IsRegionGpuModified(VAddr addr, size_t size) { bool result = false; diff --git a/src/video_core/query_dma.h b/src/video_core/query_dma.h index c236cca622..40ccafe668 100644 --- a/src/video_core/query_dma.h +++ b/src/video_core/query_dma.h @@ -43,6 +43,23 @@ inline bool HasPendingQueries(const Tegra::MemoryManager& memory, return false; } +/// Whether the query cache holds reports that can still land in [source, source + size): a read +/// sees them only once FlushPending wrote them out. Any thread (RasterizerInterface::HoldsQueries). +inline bool HoldsReports(const Tegra::MemoryManager& memory, + VideoCore::RasterizerInterface& rasterizer, GPUVAddr source, size_t size) { + constexpr GPUVAddr page_size = 0x1000; + const GPUVAddr end = source + size; + for (GPUVAddr begin = source; begin < end;) { + const GPUVAddr page_end = (std::min)((begin & ~(page_size - 1)) + page_size, end); + if (const auto device = memory.GpuToCpuAddress(begin); + device && rasterizer.HoldsQueries(*device, page_end - begin)) { + return true; + } + begin = page_end; + } + return false; +} + /// Writes the reports in [source, source + size) to guest memory, waiting for pending host /// results. Runs where the rasterizer's caches are owned. inline void FlushPending(const Tegra::MemoryManager& memory, diff --git a/src/video_core/rasterizer_interface.h b/src/video_core/rasterizer_interface.h index 4b5cccc5df..5028e4211e 100644 --- a/src/video_core/rasterizer_interface.h +++ b/src/video_core/rasterizer_interface.h @@ -143,6 +143,14 @@ class RasterizerInterface { return false; } + /// Any thread: whether the query cache holds reports that can still land in the range, as + /// FlushQueryResults writes them out or once their host results come in. Without a query + /// cache that can tell, assume so. + [[nodiscard]] virtual bool HoldsQueries([[maybe_unused]] DAddr addr, + [[maybe_unused]] u64 size) { + return true; + } + /// Notify rasterizer that any caches of the specified region should be invalidated virtual void InvalidateRegion(DAddr addr, u64 size, VideoCommon::CacheType which = VideoCommon::CacheType::All) = 0; diff --git a/src/video_core/renderer_null/null_rasterizer.h b/src/video_core/renderer_null/null_rasterizer.h index c7f5849c75..71a920501e 100644 --- a/src/video_core/renderer_null/null_rasterizer.h +++ b/src/video_core/renderer_null/null_rasterizer.h @@ -51,6 +51,10 @@ class RasterizerNull final : public VideoCore::RasterizerInterface, VideoCommon::CacheType which = VideoCommon::CacheType::All) override; bool MustFlushRegion(DAddr addr, u64 size, VideoCommon::CacheType which = VideoCommon::CacheType::All) override; + /// No query cache: Query writes each report as it is made. + bool HoldsQueries([[maybe_unused]] DAddr addr, [[maybe_unused]] u64 size) override { + return false; + } void InvalidateRegion(DAddr addr, u64 size, VideoCommon::CacheType which = VideoCommon::CacheType::All) override; void OnCacheInvalidation(DAddr addr, u64 size) override; diff --git a/src/video_core/renderer_vulkan/vk_rasterizer.cpp b/src/video_core/renderer_vulkan/vk_rasterizer.cpp index 0a3e6affd8..c66da49d62 100644 --- a/src/video_core/renderer_vulkan/vk_rasterizer.cpp +++ b/src/video_core/renderer_vulkan/vk_rasterizer.cpp @@ -2037,6 +2037,10 @@ bool RasterizerVulkan::HasPendingQueries(DAddr addr, u64 size) { return query_cache.IsRegionGpuModified(addr, size); } +bool RasterizerVulkan::HoldsQueries(DAddr addr, u64 size) { + return query_cache.HoldsQueries(addr, size); +} + VideoCore::RasterizerDownloadArea RasterizerVulkan::GetFlushArea(DAddr addr, u64 size) { { std::scoped_lock lock{texture_cache.mutex}; diff --git a/src/video_core/renderer_vulkan/vk_rasterizer.h b/src/video_core/renderer_vulkan/vk_rasterizer.h index bb97cc4f2c..af899140e8 100644 --- a/src/video_core/renderer_vulkan/vk_rasterizer.h +++ b/src/video_core/renderer_vulkan/vk_rasterizer.h @@ -113,6 +113,7 @@ class RasterizerVulkan final : public VideoCore::RasterizerInterface, VideoCommon::CacheType which = VideoCommon::CacheType::All) override; VideoCore::RasterizerDownloadArea GetFlushArea(DAddr addr, u64 size) override; bool HasPendingQueries(DAddr addr, u64 size) override; + bool HoldsQueries(DAddr addr, u64 size) override; void InvalidateRegion(DAddr addr, u64 size, VideoCommon::CacheType which = VideoCommon::CacheType::All) override; void PrepareRegionWrite(DAddr addr, u64 size, VideoCommon::CacheType which) override; From 936c05e1492476eb18901ce95313dcfbb670d69b Mon Sep 17 00:00:00 2001 From: jpkto <255363096+jpkto@users.noreply.github.com> Date: Sat, 7 Nov 2026 21:52:43 +0100 Subject: [PATCH 016/105] video_core: check macro DMA reads against a drained split (EDEN_NX2_DMA_READ_VERIFY) A macro DMA read on the split's front now waits only for the queued work that can write its words and drains only when the caches hold them (FrontDmaRead). EDEN_NX2_DMA_READ_VERIFY=1 checks that against what every read did before: after reading the words without a drain, the front drains anyway, has the back write out what the buffer and query caches hold of them, reads them again and gives the macro those. It counts the reads whose words differ (CheckDmaRead) and logs every 10 s how many reads it made, how many without a drain, how many of those waited for queued writers and how many differed, with the first 16 differing reads and their words. The differences must stay 0; a guest CPU write that lands between the two reads would count too. Off, it reads one constant flag per read. Rebased onto upstream: EDEN_NX2_DMA_READ_VERIFY is a Maxwell3D member read when the engine is made, as #714 made EDEN_NX2_3D_METHODS, so it works from nx2_switches.txt too. Co-Authored-By: Claude Opus 5.5 --- src/video_core/engines/maxwell_3d.cpp | 75 +++++++++++++++++++++++++-- src/video_core/engines/maxwell_3d.h | 6 +++ 2 files changed, 78 insertions(+), 3 deletions(-) diff --git a/src/video_core/engines/maxwell_3d.cpp b/src/video_core/engines/maxwell_3d.cpp index 01d2439267..90f390e03d 100644 --- a/src/video_core/engines/maxwell_3d.cpp +++ b/src/video_core/engines/maxwell_3d.cpp @@ -8,10 +8,13 @@ #include #include #include +#include #include #include #include +#include #include +#include #include "common/assert.h" #include "common/bit_util.h" #include "common/scope_exit.h" @@ -1054,6 +1057,53 @@ bool FrontVerifyEnabled() { return verify; } +/// A macro DMA read of `words` at `source`: whether the caches held them (`drained`), and else +/// whether it waited for queued writers and what it read before the drain (`undrained`). +void CheckDmaRead(u32 macro, GPUVAddr source, bool drained, bool waited, + std::span undrained, std::span words) { + static u64 reads{}; + static u64 undrained_reads{}; + static u64 waits{}; + static u64 differences{}; + static u64 total_differences{}; + static std::chrono::steady_clock::time_point last_report{}; + ++reads; + if (!drained) { + ++undrained_reads; + waits += waited ? 1 : 0; + if (!std::ranges::equal(undrained, words)) { + ++differences; + if (++total_differences <= 16) { + const size_t shown = std::min(words.size(), 8); + LOG_WARNING(HW_GPU, + "Macro DMA read check: macro {:#x} read {} words at {:#x} as {:08x} " + "without draining the split, as {:08x} after it", + macro, words.size(), source, fmt::join(undrained.first(shown), ","), + fmt::join(words.first(shown), ",")); + } + } + } + const auto now = std::chrono::steady_clock::now(); + if (last_report == std::chrono::steady_clock::time_point{}) { + last_report = now; + return; + } + const double seconds = std::chrono::duration(now - last_report).count(); + if (seconds < 10.0) { + return; + } + last_report = now; + LOG_INFO(HW_GPU, + "Macro DMA read checks in {:.1f} s: {} reads, {} without draining the split ({} " + "waited for queued writers), {} of those with other words than after a drain ({} " + "in all)", + seconds, reads, undrained_reads, waits, differences, total_differences); + reads = 0; + undrained_reads = 0; + waits = 0; + differences = 0; +} + // Executable methods ProcessMethodCall handles itself; the rest go to the draw manager. bool FrontHandledOutsideDrawManager(u32 method) { constexpr u32 cb_data = MAXWELL3D_REG_INDEX(const_buffer.buffer); @@ -1563,11 +1613,24 @@ void Maxwell3D::FrontDmaRead(VideoCommon::GpuSplit& split, FrontState& fs, u32 m // Only the work still queued that can write the words has to run first (FrontNoteWrites, // GpuSplit::WaitForWrites). Past it no entry can make them GPU-written or add reports there, // so the caches have nothing to write out unless they already hold it now. - split.WaitForWrites(memory_manager, source, read_bytes); - if (memory_manager.IsMemoryDirty(source, read_bytes, VideoCommon::CacheType::BufferCache) || - VideoCommon::QueryDma::HoldsReports(memory_manager, *rasterizer, source, read_bytes)) { + const bool waited = split.WaitForWrites(memory_manager, source, read_bytes); + const bool drained = + memory_manager.IsMemoryDirty(source, read_bytes, VideoCommon::CacheType::BufferCache) || + VideoCommon::QueryDma::HoldsReports(memory_manager, *rasterizer, source, read_bytes); + if (drained) { write_out(); } + std::vector undrained; + std::vector checked; + if (dma_read_verify) [[unlikely]] { + if (!drained) { + for (u32 i = 0; i < words; ++i) { + undrained.push_back(memory_manager.Read(source + u64{i} * sizeof(u32))); + } + write_out(); + } + checked.reserve(words); + } u32 changed{}; for (u32 i = 0; i < words; ++i) { const u32 word = memory_manager.Read(source + u64{i} * sizeof(u32)); @@ -1579,6 +1642,12 @@ void Maxwell3D::FrontDmaRead(VideoCommon::GpuSplit& split, FrontState& fs, u32 m if (!diag_before.empty() && diag_before[i] != word) { ++changed; } + if (dma_read_verify) [[unlikely]] { + checked.push_back(word); + } + } + if (dma_read_verify) [[unlikely]] { + CheckDmaRead(macro, source, drained, waited, undrained, checked); } if (Tegra::Nx2CmdPageDiag::g_enabled) { bool gpu_written = false; diff --git a/src/video_core/engines/maxwell_3d.h b/src/video_core/engines/maxwell_3d.h index 57d24164bc..3b61d7a758 100644 --- a/src/video_core/engines/maxwell_3d.h +++ b/src/video_core/engines/maxwell_3d.h @@ -3382,6 +3382,12 @@ class Maxwell3D final : public EngineInterface { // model). Read per engine, not at static initialization, which runs before the frontend // loads nx2_switches.txt. const bool trace_first_method_writes = std::getenv("EDEN_NX2_3D_METHODS") != nullptr; + // EDEN_NX2_DMA_READ_VERIFY=1: a macro DMA read the front makes without draining the split + // (FrontDmaRead) drains it anyway and has the caches write the words out, as every read + // did before, and the macro gets those words. The front counts the reads whose words + // differ from the ones it read first and logs the counts every 10 s, the first 16 reads + // with their words. The differences must stay 0. Read per engine, as the switch above. + const bool dma_read_verify = std::getenv("EDEN_NX2_DMA_READ_VERIFY") != nullptr; /// Parameters that have been submitted to the macro call so far. std::vector macro_params; From 1072d4a1eded663614a56694456755e90287d737 Mon Sep 17 00:00:00 2001 From: jpkto <255363096+jpkto@users.noreply.github.com> Date: Sat, 7 Nov 2026 22:31:52 +0100 Subject: [PATCH 017/105] video_core: let the reading thread wait for its downloads A guest CPU read of GPU-written buffer memory (reactive flushing) had the GPU thread download it with Scheduler::Finish. The GPU thread waited for the host GPU with the buffer cache locked, and the reading thread waited for the GPU thread. In Ocarina of Time that was about 1,500 flushes every 10 seconds and 80-90 ms of each second on the split's back thread. Meanwhile every guest core that looked up a flush area spun on the buffer cache lock. Now the GPU thread only records the copies and submits them (RasterizerInterface::FlushRegionForRead, BufferCache::DownloadMemoryForRead). The reading thread waits for that submission's tick itself. It then writes the bytes to guest memory before it reads (FinishRead), and holds the lock only while it copies them out. Until the bytes land: - The ranges stay GPU-modified, so nothing treats guest memory there as current. Another read of them waits for the same copies instead of flushing again (RasterizerDownloadArea::in_flight). - CPU writes keep their bytes. The copy-out leaves out the writes that are still collected, which now includes those InvalidateGPUCache has gathered but not applied yet. Applying a write takes its bytes out of the download (ClearDownload). - A newer GPU write keeps its range GPU-modified. The older download still writes what a synchronous flush would have left there. - Downloads are written oldest first, and before a later fence's downloads (PopAsyncBuffers). None lands over newer bytes. - A write to guest memory from the GPU's side first writes the downloads under it. This covers semaphore releases, inline and constant buffer uploads (MemoryManager) and DMA clears (DMAClear). While no download is pending, the check costs one relaxed load. Co-Authored-By: Claude Opus 5.5 --- src/tests/video_core/buffer_cache_ports.cpp | 278 +++++++++++++++++- src/video_core/buffer_cache/buffer_cache.h | 235 ++++++++++++++- .../buffer_cache/buffer_cache_base.h | 49 ++- src/video_core/gpu.cpp | 74 +++-- src/video_core/memory_manager.cpp | 16 + src/video_core/rasterizer_download_area.h | 10 + src/video_core/rasterizer_interface.h | 11 + .../renderer_vulkan/vk_buffer_cache.cpp | 11 + .../renderer_vulkan/vk_buffer_cache.h | 6 + .../renderer_vulkan/vk_rasterizer.cpp | 22 ++ .../renderer_vulkan/vk_rasterizer.h | 2 + 11 files changed, 682 insertions(+), 32 deletions(-) diff --git a/src/tests/video_core/buffer_cache_ports.cpp b/src/tests/video_core/buffer_cache_ports.cpp index 034167cd9f..0436750920 100644 --- a/src/tests/video_core/buffer_cache_ports.cpp +++ b/src/tests/video_core/buffer_cache_ports.cpp @@ -4,6 +4,8 @@ #include #include #include +#include +#include #include #include #include @@ -70,17 +72,22 @@ struct Runtime { u32 GetUniformBufferAlignment() const { return 1; } - bool IsFree(u64) const { - return true; + bool IsFree(u64 tick) const { + return tick <= completed_tick; + } + u64 CurrentTick() const { + return submitted_tick + 1; } void WaitSubmittedTick(u64) {} bool CanReorderUpload(const Buffer&, std::span) const { return false; } Staging DownloadStagingBuffer(u64 size, bool = false) { - staging = std::make_unique(*this, 0, size + 128, false); + // Kept until the end: read downloads hold theirs past the next request. + stagings.push_back(std::make_unique(*this, 0, size + 128, false)); ++allocations; - return {staging.get(), 128, std::span{staging->bytes}.subspan(128)}; + Buffer* const staging = stagings.back().get(); + return {staging, 128, std::span{staging->bytes}.subspan(128)}; } Staging UploadStagingBuffer(u64 size) { return DownloadStagingBuffer(size); @@ -112,12 +119,36 @@ struct Runtime { if (before_completion) { before_completion(); } + Complete(submitted_tick); for (auto& copy : pending) { copy(); } pending.clear(); ++finishes; } + u64 SubmitForHostRead() { + ++host_read_barriers; + submitted.emplace_back(++submitted_tick, std::move(pending)); + pending.clear(); + return submitted_tick; + } + void WaitForHostRead(u64 tick) { + if (before_completion) { + before_completion(); + } + Complete(tick); + ++host_read_waits; + } + /// The host GPU runs what was submitted up to `tick`. + void Complete(u64 tick) { + while (!submitted.empty() && submitted.front().first <= tick) { + for (auto& copy : submitted.front().second) { + copy(); + } + submitted.pop_front(); + } + completed_tick = (std::max)(completed_tick, tick); + } void Wait(u64 tick) { if (before_completion) { before_completion(); @@ -129,6 +160,8 @@ struct Runtime { ++host_read_barriers; Finish(); } + void PreCopyBarrier() {} + void PostCopyBarrier() {} void BindUniformBuffer(Buffer&, u32, u32) {} std::span BindMappedUniformBuffer(size_t, u32, u32 size) { stream.resize(size); @@ -137,11 +170,13 @@ struct Runtime { u32 alignment{256}; u32 allocations{}, downloads{}, uploads{}, finishes{}, waits{}; - u32 host_read_barriers{}; + u32 host_read_barriers{}, host_read_waits{}; u64 upload_bytes{}, waited_tick{}; - std::unique_ptr staging; + u64 submitted_tick{}, completed_tick{}; + std::vector> stagings; std::vector stream; std::vector> pending; + std::deque>>> submitted; std::function before_completion; }; @@ -235,6 +270,12 @@ struct BufferCacheTestAccess { static u32 CpuMarks(Cache& cache) { return cache.memory_tracker.cpu_marks; } + static void MarkWritten(Cache& cache, BufferId id, DAddr address, u32 size) { + cache.MarkWrittenBuffer(id, address, size); + } + static size_t ReadDownloads(Cache& cache) { + return cache.read_downloads.size(); + } }; } // namespace VideoCommon @@ -298,6 +339,59 @@ struct Fixture { } }; +/// Hands FinishRead to a buffer cache, as RasterizerVulkan does. +class ReadRasterizer final : public VideoCore::RasterizerInterface { +public: + explicit ReadRasterizer(Cache& cache_) : cache{cache_} {} + + void FinishRead(DAddr addr, u64 size) override { + cache.FinishReadDownloads(addr, size); + } + void Draw(bool, u32) override {} + void DrawTexture() override {} + void Clear(u32) override {} + void DispatchCompute() override {} + void ResetCounter(VideoCommon::QueryType) override {} + void Query(GPUVAddr, VideoCommon::QueryType, VideoCommon::QueryPropertiesFlags, u32, + u32) override {} + void BindGraphicsUniformBuffer(size_t, u32, GPUVAddr, u32) override {} + void DisableGraphicsUniformBuffer(size_t, u32) override {} + void SignalFence(std::function&&) override {} + void SyncOperation(std::function&&) override {} + void SignalSyncPoint(u32) override {} + void SignalReference() override {} + void ReleaseFences(bool) override {} + void FlushAll() override {} + void FlushRegion(DAddr, u64, VideoCommon::CacheType) override {} + bool MustFlushRegion(DAddr, u64, VideoCommon::CacheType) override { + return false; + } + VideoCore::RasterizerDownloadArea GetFlushArea(DAddr addr, u64 size) override { + return {addr, addr + size, true}; + } + void InvalidateRegion(DAddr, u64, VideoCommon::CacheType) override {} + void OnCacheInvalidation(DAddr, u64) override {} + bool OnCPUWrite(DAddr, u64) override { + return false; + } + void InvalidateGPUCache() override {} + void UnmapMemory(DAddr, u64) override {} + void ModifyGPUMemory(size_t, GPUVAddr, u64) override {} + void FlushAndInvalidateRegion(DAddr, u64, VideoCommon::CacheType) override {} + void WaitForIdle() override {} + void FragmentBarrier() override {} + void TiledCacheBarrier() override {} + void FlushCommands() override {} + void TickFrame() override {} + Tegra::Engines::AccelerateDMAInterface& AccessAccelerateDMA() override { + std::terminate(); + } + void AccelerateInlineToMemory(GPUVAddr, size_t, std::span) override {} + +private: + Cache& cache; +}; + } // namespace TEST_CASE("Storage binding cache tracks map changes and descriptor dependencies", @@ -627,3 +721,175 @@ TEST_CASE("Downloads keep CPU writes collected and not yet applied", "[buffer_po REQUIRE(std::all_of(f.bytes + 0x20ff8, f.bytes + 0x21008, [](u8 b) { return b == 0xee; })); REQUIRE(f.bytes[0x21008] == 0x34); } + +TEST_CASE("Read downloads stay GPU-modified until the reading thread writes them", + "[buffer_ports]") { + Fixture f; + const bool mapped = GENERATE(false, true); + const auto id = Access::Add(*f.cache, 0x20000, 0x10000, mapped); + std::ranges::fill(Access::Get(*f.cache, id).bytes, 0x12); + Access::Dirty(*f.cache, 0x20000, 0x2000); + f.cache->DownloadMemoryForRead(0x20000, 0x1000); + REQUIRE(f.runtime.finishes == 0); + REQUIRE(f.runtime.downloads == 1); + REQUIRE(f.runtime.host_read_barriers == 1); + REQUIRE(VideoCore::g_read_downloads == 1); + REQUIRE(f.bytes[0x10000] == 0); + REQUIRE(Access::Pending(*f.cache, 0x20000, 0x1000)); + // Another read of the range waits for the same copy, and its flush copies nothing. + const auto area = f.cache->GetFlushArea(0x20800, 4); + REQUIRE(area); + REQUIRE_FALSE(area->preemtive); + REQUIRE(area->in_flight); + f.cache->DownloadMemoryForRead(0x20000, 0x1000); + REQUIRE(f.runtime.downloads == 1); + REQUIRE(f.runtime.submitted_tick == 1); + f.cache->FinishReadDownloads(0x20800, 4); + REQUIRE(f.runtime.host_read_waits == 1); + REQUIRE(VideoCore::g_read_downloads == 0); + REQUIRE(f.bytes[0x10000] == 0x12); + REQUIRE(f.bytes[0x10fff] == 0x12); + REQUIRE(f.bytes[0x11000] == 0); + REQUIRE_FALSE(Access::Pending(*f.cache, 0x20000, 0x1000)); + REQUIRE(Access::Pending(*f.cache, 0x21000, 0x1000)); + REQUIRE(f.cache->GetFlushArea(0x20800, 4)->preemtive); + f.cache->FinishReadDownloads(0x20000, 0x2000); + REQUIRE(f.runtime.host_read_waits == 1); +} + +TEST_CASE("Read downloads keep CPU writes made while they were on their way", "[buffer_ports]") { + Fixture f; + const auto id = Access::Add(*f.cache, 0x20000, 0x10000); + std::ranges::fill(Access::Get(*f.cache, id).bytes, 0x12); + Access::Dirty(*f.cache, 0x20000, 0x1000); + f.cache->DownloadMemoryForRead(0x20000, 0x1000); + // After the submission: one write still collected, one already applied. + std::memset(f.bytes + 0x10100, 0xee, 0x10); + std::memset(f.bytes + 0x10200, 0xdd, 0x10); + f.cache->SetPendingCpuWrites( + [](const std::function& callback) { callback(0x20100, 0x10); }); + const std::array, 1> applied{{{0x20200, 0x10}}}; + f.cache->ClaimCpuWrites(applied); + f.cache->FinishReadDownloads(0x20000, 0x1000); + REQUIRE(f.bytes[0x100ff] == 0x12); + REQUIRE(std::all_of(f.bytes + 0x10100, f.bytes + 0x10110, [](u8 b) { return b == 0xee; })); + REQUIRE(f.bytes[0x10110] == 0x12); + REQUIRE(std::all_of(f.bytes + 0x10200, f.bytes + 0x10210, [](u8 b) { return b == 0xdd; })); + REQUIRE(f.bytes[0x10210] == 0x12); + REQUIRE(f.bytes[0x10fff] == 0x12); + REQUIRE_FALSE(Access::Pending(*f.cache, 0x20000, 0x1000)); +} + +TEST_CASE("Read downloads land oldest first and leave newer GPU writes to download", + "[buffer_ports]") { + Fixture f; + const auto id = Access::Add(*f.cache, 0x20000, 0x10000); + auto& buffer = Access::Get(*f.cache, id); + std::ranges::fill(buffer.bytes, 0x12); + Access::Dirty(*f.cache, 0x20000, 0x1000); + f.cache->DownloadMemoryForRead(0x20000, 0x1000); + // The GPU thread's next work writes part of the range: on the host GPU, after the copy. + f.runtime.pending.push_back( + [&buffer] { std::fill_n(buffer.bytes.begin() + 0x800, 0x100, u8{0x34}); }); + Access::MarkWritten(*f.cache, id, 0x20800, 0x100); + const auto area = f.cache->GetFlushArea(0x20800, 4); + REQUIRE_FALSE(area->preemtive); + REQUIRE_FALSE(area->in_flight); + f.cache->DownloadMemoryForRead(0x20000, 0x1000); + REQUIRE(f.runtime.downloads == 2); + REQUIRE(Access::ReadDownloads(*f.cache) == 2); + REQUIRE(f.cache->GetFlushArea(0x20800, 4)->in_flight); + // The first copy holds what a synchronous flush would have left for the whole range. + f.cache->FinishReadDownloads(0x20000, 4); + REQUIRE(Access::ReadDownloads(*f.cache) == 1); + REQUIRE(f.bytes[0x10000] == 0x12); + REQUIRE(f.bytes[0x10800] == 0x12); + REQUIRE(Access::Pending(*f.cache, 0x20800, 0x100)); + REQUIRE_FALSE(Access::Pending(*f.cache, 0x20000, 0x800)); + f.cache->FinishReadDownloads(0x20800, 4); + REQUIRE(f.bytes[0x107ff] == 0x12); + REQUIRE(f.bytes[0x10800] == 0x34); + REQUIRE(f.bytes[0x108ff] == 0x34); + REQUIRE(f.bytes[0x10900] == 0x12); + REQUIRE_FALSE(Access::Pending(*f.cache, 0x20000, 0x1000)); + // Both written in order even when the newer one's reader comes first. + Access::Dirty(*f.cache, 0x20000, 0x1000); + f.cache->DownloadMemoryForRead(0x20000, 0x1000); + f.runtime.pending.push_back( + [&buffer] { std::fill_n(buffer.bytes.begin() + 0x800, 0x100, u8{0x56}); }); + Access::MarkWritten(*f.cache, id, 0x20800, 0x100); + f.cache->DownloadMemoryForRead(0x20800, 0x100); + REQUIRE(Access::ReadDownloads(*f.cache) == 2); + f.cache->FinishReadDownloads(0x20800, 4); + REQUIRE(Access::ReadDownloads(*f.cache) == 0); + REQUIRE(f.bytes[0x10800] == 0x56); + REQUIRE(f.bytes[0x10900] == 0x12); + REQUIRE_FALSE(Access::Pending(*f.cache, 0x20000, 0x1000)); +} + +TEST_CASE("Read downloads land before a later fence's downloads", "[buffer_ports]") { + Fixture f; + const auto id = Access::Add(*f.cache, 0x20000, 0x10000); + auto& buffer = Access::Get(*f.cache, id); + std::ranges::fill(buffer.bytes, 0x12); + Access::MarkWritten(*f.cache, id, 0x20000, 0x1000); + f.cache->DownloadMemoryForRead(0x20000, 0x1000); + // A later GPU write the next fence downloads, and the host GPU runs all of it before anyone + // waits for the read's copy. + f.runtime.pending.push_back([&buffer] { std::fill_n(buffer.bytes.begin(), 0x100, u8{0x34}); }); + Access::MarkWritten(*f.cache, id, 0x20000, 0x100); + f.cache->CommitAsyncFlushes(); + f.runtime.Finish(); + f.cache->PopAsyncFlushes(); + f.cache->FinishReadDownloads(0x20000, 0x1000); + REQUIRE(f.bytes[0x10000] == 0x34); + REQUIRE(f.bytes[0x100ff] == 0x34); + REQUIRE(f.bytes[0x10100] == 0x12); + REQUIRE(f.bytes[0x10fff] == 0x12); + REQUIRE(Access::ReadDownloads(*f.cache) == 0); + REQUIRE_FALSE(Access::Pending(*f.cache, 0x20000, 0x1000)); +} + +TEST_CASE("Preflushed ranges wait for the read download still on its way", "[buffer_ports]") { + Fixture f; + const auto id = Access::Add(*f.cache, 0x20000, 0x10000); + std::ranges::fill(Access::Get(*f.cache, id).bytes, 0x12); + Access::Dirty(*f.cache, 0x20000, 0x1000); + // The first read marks the range for fence downloads, then flushes it. + const auto first = f.cache->GetFlushArea(0x20000, 4); + REQUIRE_FALSE(first->preemtive); + REQUIRE_FALSE(first->in_flight); + f.cache->DownloadMemoryForRead(first->start_address, first->end_address - first->start_address); + const auto second = f.cache->GetFlushArea(0x20010, 4); + REQUIRE_FALSE(second->preemtive); + REQUIRE(second->in_flight); + f.cache->FinishReadDownloads(second->start_address, + second->end_address - second->start_address); + REQUIRE(f.bytes[0x10010] == 0x12); + REQUIRE(f.cache->GetFlushArea(0x20010, 4)->preemtive); +} + +TEST_CASE("GPU writes to guest memory land the read downloads under them first", "[buffer_ports]") { + Fixture f; + const auto id = Access::Add(*f.cache, 0x20000, 0x10000); + std::ranges::fill(Access::Get(*f.cache, id).bytes, 0x12); + ReadRasterizer reads{*f.cache}; + Tegra::MemoryManager gpu_memory{f.system, f.device, 32, 1ULL << 31, 16}; + gpu_memory.BindRasterizer(&reads); + gpu_memory.Map(0x200000, 0x20000, 0x10000, Tegra::PTEKind::PITCH, true); + Access::Dirty(*f.cache, 0x20000, 0x1000); + f.cache->DownloadMemoryForRead(0x20000, 0x1000); + // A semaphore release, then an inline upload, after the copy was taken. + gpu_memory.Write(0x200100, 0xdeadbeef); + REQUIRE(VideoCore::g_read_downloads == 0); + const u32 words = 0x55555555; + gpu_memory.WriteBlockUnsafe(0x200200, &words, sizeof(words)); + u32 value{}; + std::memcpy(&value, f.bytes + 0x10100, sizeof(value)); + REQUIRE(value == 0xdeadbeef); + std::memcpy(&value, f.bytes + 0x10200, sizeof(value)); + REQUIRE(value == 0x55555555); + REQUIRE(f.bytes[0x10000] == 0x12); + REQUIRE(f.bytes[0x10104] == 0x12); + REQUIRE(f.bytes[0x10fff] == 0x12); +} diff --git a/src/video_core/buffer_cache/buffer_cache.h b/src/video_core/buffer_cache/buffer_cache.h index 744c365feb..4d31ee972d 100644 --- a/src/video_core/buffer_cache/buffer_cache.h +++ b/src/video_core/buffer_cache/buffer_cache.h @@ -67,7 +67,10 @@ BufferCache

::BufferCache(Tegra::MaxwellDeviceMemoryManager& device_memory_, R } template -BufferCache

::~BufferCache() = default; +BufferCache

::~BufferCache() { + VideoCore::g_read_downloads.fetch_sub(static_cast(read_downloads.size()), + std::memory_order_relaxed); +} template void BufferCache

::RunGarbageCollector(bool aggressive_gc) { @@ -165,6 +168,8 @@ void BufferCache

::TickFrame() { ++frame_tick; delayed_destruction_ring.Tick(); + // Read downloads no thread waited for (the other ranges the guest read before). + WriteCompletedReadDownloads(); for (auto& buffer : async_buffers_death_ring) { runtime.FreeDeferredStagingBuffer(buffer); } @@ -276,11 +281,26 @@ std::optional BufferCache

::GetFlushArea(DA area->start_address = device_addr_start_aligned; area->end_address = device_addr_end_aligned; if (memory_tracker.IsRegionPreflushable(device_addr, size)) { - area->preemtive = true; + // Fence downloads keep it, once the first read's download is in guest memory. + area->in_flight = std::ranges::any_of(read_downloads, [&](const ReadDownload& download) { + return download.write_ranges.Intersects( + device_addr_start_aligned, device_addr_end_aligned - device_addr_start_aligned); + }); + area->preemtive = !area->in_flight; return area; }; area->preemtive = !IsRegionGpuModified(device_addr_start_aligned, device_addr_end_aligned - device_addr_start_aligned); + if (!area->preemtive && !read_downloads.empty() && + ReadDownloadsCover(device_addr_start_aligned, + device_addr_end_aligned - device_addr_start_aligned)) { + // All of it on its way for an earlier read: wait for those copies, not new ones. The core + // took the trap, as below. + area->in_flight = true; + read_trapped_ranges.Subtract(device_addr_start_aligned, + device_addr_end_aligned - device_addr_start_aligned); + return area; + } // A preflushable mark persists and downloads later GPU writes at every fence. Native titles // instead keep read traps and flush on demand. const bool preflush = !Tegra::g_nx2_ampere.load(std::memory_order_relaxed); @@ -430,8 +450,205 @@ void BufferCache

::DownloadRangesMemory(std::span> } } +template +void BufferCache

::DownloadMemoryForRead(DAddr device_addr, u64 size) { + if constexpr (IS_OPENGL || !USE_MEMORY_MAPS) { + DownloadMemory(device_addr, size); + } else { + WriteCompletedReadDownloads(); + // The ranges DownloadMemory takes: the guest CPU soon reads those it read before too. + boost::container::small_vector, 16> ranges{{device_addr, size}}; + if (download_cpu_read_ranges) { + download_cpu_read_ranges = false; + cpu_read_ranges.ForEach([&](DAddr lower, DAddr upper) { + gpu_modified_ranges.ForEachInRange( + lower, upper - lower, + [&](DAddr start, DAddr end) { ranges.emplace_back(start, end - start); }); + }); + } + struct Copy { + BufferId buffer_id; + BufferCopy copy; + }; + boost::container::small_vector copies; + ReadDownload download; + u64 total_size_bytes = 0; + const auto add_copy = [&](BufferId buffer_id, DAddr buffer_addr, DAddr begin, DAddr end) { + copies.push_back({buffer_id, BufferCopy{ + .src_offset = begin - buffer_addr, + .dst_offset = total_size_bytes, + .size = end - begin, + }}); + total_size_bytes += Common::AlignUp(end - begin, 64); + download.write_ranges.Add(begin, end - begin); + download.clean_ranges.Add(begin, end - begin); + read_download_ranges.Add(begin, end - begin); + }; + boost::container::small_vector, 8> gaps; + for (const auto& [range_addr, range_size] : ranges) { + ForEachBufferInRange(range_addr, range_size, [&](BufferId buffer_id, Buffer& buffer) { + const DAddr buffer_addr = buffer.CpuAddr(); + const DAddr begin = (std::max)(range_addr, buffer_addr); + const DAddr end = + (std::min)(range_addr + range_size, buffer_addr + buffer.SizeBytes()); + if (begin >= end) { + return; + } + // The bytes DownloadRangesMemory takes, but they stay GPU-modified until they + // are in guest memory (WriteReadDownloads). + memory_tracker.ForEachDownloadRange( + begin, end - begin, false, [&](u64 run_addr, u64 run_size) { + gpu_modified_ranges.ForEachInRange( + run_addr, run_size, [&](DAddr start, DAddr range_end) { + // Without the bytes a read download already brings (an earlier + // one, or this one: the ranges can overlap). + gaps.clear(); + DAddr next = start; + read_download_ranges.ForEachInRange( + start, range_end - start, [&](DAddr lower, DAddr upper) { + if (lower > next) { + gaps.emplace_back(next, lower); + } + next = (std::max)(next, upper); + }); + if (next < range_end) { + gaps.emplace_back(next, range_end); + } + for (const auto& [gap_begin, gap_end] : gaps) { + add_copy(buffer_id, buffer_addr, gap_begin, gap_end); + } + }); + // The downloads taken at fences are older: they must not land over these. + ClearFenceDownloads(run_addr, run_size); + }); + }); + } + if (copies.empty()) { + return; + } + auto staging = runtime.DownloadStagingBuffer(total_size_bytes, true); + for (Copy& entry : copies) { + Buffer& buffer = slot_buffers[entry.buffer_id]; + entry.copy.dst_offset += staging.offset; + buffer.MarkUsage(entry.copy.src_offset, entry.copy.size); + // Mapped buffers too: the GPU thread's later work may write them before the copy is + // read. + runtime.CopyBuffer(staging.buffer, buffer, std::span{&entry.copy, 1}, true); + download.copies.push_back(BufferCopy{ + .src_offset = buffer.CpuAddr() + entry.copy.src_offset, + .dst_offset = entry.copy.dst_offset, + .size = entry.copy.size, + }); + } + download.tick = runtime.SubmitForHostRead(); + download.staging = staging; + read_downloads.push_back(std::move(download)); + VideoCore::g_read_downloads.fetch_add(1, std::memory_order_release); + } +} + +template +void BufferCache

::FinishReadDownloads(DAddr device_addr, u64 size) { + if constexpr (!IS_OPENGL) { + std::unique_lock lock{mutex}; + // The newest one over the range: those before it are written first. + std::optional tick; + for (const ReadDownload& download : read_downloads) { + if (download.write_ranges.Intersects(device_addr, size)) { + tick = download.tick; + } + } + if (!tick) { + return; + } + if (!runtime.IsFree(*tick)) { + lock.unlock(); + runtime.WaitForHostRead(*tick); + lock.lock(); + } + WriteReadDownloads(*tick); + } +} + +template +void BufferCache

::WriteReadDownloads(u64 tick) { + if constexpr (!IS_OPENGL) { + while (!read_downloads.empty() && read_downloads.front().tick <= tick) { + ReadDownload& download = read_downloads.front(); + // CPU writes are collected before they reach guest memory, and stay pending until + // applied, which takes them out of write_ranges (ClaimCpuWrites): either way they + // keep their bytes. + const Common::RangeSet cpu_written = PendingCpuWriteRanges(); + const u8* const base = download.staging.mapped_span.data(); + for (const BufferCopy& copy : download.copies) { + const DAddr copy_addr = static_cast(copy.src_offset); + const u8* const data = base + (copy.dst_offset - download.staging.offset); + download.write_ranges.ForEachInRange( + copy_addr, copy.size, [&](DAddr start, DAddr end) { + WriteDownloadedMemory(cpu_written, start, data + (start - copy_addr), + end - start); + }); + } + download.clean_ranges.ForEach([&](DAddr start, DAddr end) { + gpu_modified_ranges.Subtract(start, end - start); + read_download_ranges.Subtract(start, end - start); + }); + // As a synchronous download leaves them: pages without GPU-modified bytes left are + // clean in the tracker too. + download.clean_ranges.ForEach([&](DAddr start, DAddr end) { + const DAddr last = Common::AlignUp(end, DEVICE_PAGESIZE); + for (DAddr page = Common::AlignDown(start, DEVICE_PAGESIZE); page < last; + page += DEVICE_PAGESIZE) { + if (!gpu_modified_ranges.Intersects(page, DEVICE_PAGESIZE)) { + memory_tracker.UnmarkRegionAsGpuModified(page, DEVICE_PAGESIZE); + } + } + }); + // Freed by the GPU thread (TickFrame). + async_buffers_death_ring.push_back(download.staging); + read_downloads.pop_front(); + VideoCore::g_read_downloads.fetch_sub(1, std::memory_order_release); + } + } +} + +template +void BufferCache

::WriteCompletedReadDownloads() { + if constexpr (!IS_OPENGL) { + while (!read_downloads.empty() && runtime.IsFree(read_downloads.front().tick)) { + WriteReadDownloads(read_downloads.front().tick); + } + } +} + +template +bool BufferCache

::ReadDownloadsCover(DAddr device_addr, u64 size) const { + bool covered = true; + gpu_modified_ranges.ForEachInRange(device_addr, size, [&](DAddr start, DAddr end) { + DAddr next = start; + read_download_ranges.ForEachInRange(start, end - start, [&](DAddr lower, DAddr upper) { + covered = covered && lower <= next; + next = (std::max)(next, upper); + }); + covered = covered && next >= end; + }); + return covered; +} + template void BufferCache

::ClearDownload(DAddr device_addr, u64 size) { + ClearFenceDownloads(device_addr, size); + if (!read_downloads.empty()) [[unlikely]] { + for (ReadDownload& download : read_downloads) { + download.write_ranges.Subtract(device_addr, size); + download.clean_ranges.Subtract(device_addr, size); + } + read_download_ranges.Subtract(device_addr, size); + } +} + +template +void BufferCache

::ClearFenceDownloads(DAddr device_addr, u64 size) { for (auto& ranges : pending_download_ranges) { ranges.Subtract(device_addr, size); } @@ -524,6 +741,9 @@ bool BufferCache

::DMAClear(GPUVAddr dst_address, u64 amount, u32 value) { } const size_t size = amount * sizeof(u32); + // The caller writes the cleared words to guest memory after this: earlier reads' downloads + // land first, so a reader of these bytes never finds neither there. + FinishReadDownloads(*cpu_dst_address, size); ClearDownload(*cpu_dst_address, size); gpu_modified_ranges.Subtract(*cpu_dst_address, size); @@ -1054,6 +1274,9 @@ void BufferCache

::WriteDownloadedMemory(const Common::RangeSet& cpu_wr template void BufferCache

::PopAsyncBuffers() { + // The host GPU is past this fence: the read downloads taken before its copies are complete, + // and go to guest memory first. Those taken after took their bytes out of its copies. + WriteCompletedReadDownloads(); if (async_buffers.empty()) { return; } @@ -2070,6 +2293,14 @@ void BufferCache

::MarkWrittenBuffer(BufferId buffer_id, DAddr device_addr, u3 for (auto& ranges : pending_download_ranges) { ranges.Subtract(device_addr, size); } + // A read download taken before still writes these bytes to guest memory, as a synchronous + // one would have, but they stay GPU-modified: a later read downloads them again. + if (!read_downloads.empty()) [[unlikely]] { + for (ReadDownload& download : read_downloads) { + download.clean_ranges.Subtract(device_addr, size); + } + read_download_ranges.Subtract(device_addr, size); + } memory_tracker.MarkRegionAsGpuModified(device_addr, size); MarkVertexBuffersStale(device_addr, size); gpu_modified_ranges.Add(device_addr, size); diff --git a/src/video_core/buffer_cache/buffer_cache_base.h b/src/video_core/buffer_cache/buffer_cache_base.h index 7a1b7e57d5..ac832ec19c 100644 --- a/src/video_core/buffer_cache/buffer_cache_base.h +++ b/src/video_core/buffer_cache/buffer_cache_base.h @@ -46,6 +46,7 @@ #include "video_core/gpu_wait_stats.h" #include "video_core/memory_manager.h" #include "video_core/nx2_download_trace.h" +#include "video_core/rasterizer_download_area.h" #include "video_core/surface.h" #include "video_core/texture_cache/types.h" @@ -254,7 +255,8 @@ class BufferCache : public VideoCommon::ChannelSetupCaches GetFlushArea(DAddr device_addr, u64 size); /// Without downloads at every fence, a guest CPU read is the only way GPU-written buffer @@ -601,6 +614,19 @@ class BufferCache : public VideoCommon::ChannelSetupCaches> ranges); + /// ClearDownload, for the downloads taken at fences only. + void ClearFenceDownloads(DAddr device_addr, u64 size); + + /// Whether every GPU-modified byte in the range is in a read download's clean_ranges. + [[nodiscard]] bool ReadDownloadsCover(DAddr device_addr, u64 size) const; + + /// Writes the read downloads up to `tick` to guest memory, oldest first (`mutex` held, the + /// host GPU past `tick`). + void WriteReadDownloads(u64 tick); + + /// WriteReadDownloads for those the host GPU has completed: nothing waits. + void WriteCompletedReadDownloads(); + void DeleteBuffer(BufferId buffer_id, bool do_not_mark = false); [[nodiscard]] Binding StorageBufferBinding(GPUVAddr ssbo_addr, u32 cbuf_index, bool is_written, @@ -752,6 +778,27 @@ class BufferCache : public VideoCommon::ChannelSetupCaches current_buffer; + /// A DownloadMemoryForRead: its copies, and what is left of them to do once the host GPU has + /// run them. + struct ReadDownload { + u64 tick{}; ///< The submission with the copies + Async_Buffer staging{}; + /// src_offset: the device address copied; dst_offset: where it is in `staging`. + boost::container::small_vector copies; + /// Still to write to guest memory: a newer value there (ClearDownload) takes its bytes + /// out. A newer GPU write doesn't: until a download of its own lands, guest memory is to + /// hold what a synchronous flush would have left. + Common::RangeSet write_ranges; + /// No longer GPU-modified once written: a newer GPU write (MarkWrittenBuffer) or value in + /// guest memory takes its bytes out. + Common::RangeSet clean_ranges; + }; + /// Oldest first, and written to guest memory in that order: none lands over newer bytes. + std::deque read_downloads; + /// All the read downloads' clean_ranges (they don't overlap): GPU-modified bytes already on + /// their way, which later reads wait for rather than download again. + Common::RangeSet read_download_ranges; + std::deque async_buffers_death_ring; size_t immediate_buffer_capacity = 0; diff --git a/src/video_core/gpu.cpp b/src/video_core/gpu.cpp index 135160132a..0542ea2581 100644 --- a/src/video_core/gpu.cpp +++ b/src/video_core/gpu.cpp @@ -112,18 +112,34 @@ struct GPU::Impl { /// Synchronizes CPU writes with Host GPU memory. void InvalidateGPUCache() { std::vector> writes; - std::function callback_writes( - [&writes](PAddr address, size_t size) { writes.emplace_back(address, size); }); - system.GatherGPUDirtyMemory(callback_writes); - if (!writes.empty()) { - renderer->ReadRasterizer()->OnCacheInvalidations(writes); + { + // Gathered writes stay pending (ForEachPendingCpuWrite) until they are applied: a + // download written to guest memory meanwhile on another thread (FinishRead) would + // otherwise write the GPU's older bytes over them. + std::scoped_lock lock{applying_writes_mutex}; + std::function callback_writes( + [&writes](PAddr address, size_t size) { writes.emplace_back(address, size); }); + system.GatherGPUDirtyMemory(callback_writes); + if (writes.empty()) { + return; + } + applying_writes.push_back(&writes); } + renderer->ReadRasterizer()->OnCacheInvalidations(writes); + std::scoped_lock lock{applying_writes_mutex}; + std::erase(applying_writes, &writes); } void ForEachPendingCpuWrite(const std::function& callback) { + std::scoped_lock lock{applying_writes_mutex}; for (auto& manager : system.GetGPUDirtyMemoryManager()) { manager.ForEachPending(callback); } + for (const auto* writes : applying_writes) { + for (const auto& [address, size] : *writes) { + callback(address, size); + } + } } /// Signal the ending of command list. @@ -269,24 +285,32 @@ struct GPU::Impl { } // Filled on the GPU thread while this thread waits for it. Stats::FlushTiming timing{}; - const u64 fence = RequestSyncOperation([this, raster_area, &timing]() { - timing.start = Stats::Now(); - const Stats::HostWaits host_before = Stats::t_host_waits; - const u64 bytes_before = Stats::t_download_bytes; - // Apply collected guest writes before downloading GPU data over guest memory. - InvalidateGPUCache(); - timing.applied = Stats::Now(); - renderer->ReadRasterizer()->FlushRegion(raster_area.start_address, raster_area.end_address - raster_area.start_address); - timing.host_ns = Stats::t_host_waits.ns - host_before.ns; - timing.finishes = Stats::t_host_waits.count - host_before.count; - timing.bytes = Stats::t_download_bytes - bytes_before; - timing.end = Stats::Now(); - }); - // The split's back runs it after the entry it is on. The tick sent through the GPU - // thread's queue still runs it without the split. - VideoCommon::GpuSplit::Get().RequestTick(); - gpu_thread.TickGPU(is_async); - WaitForSyncOperation(fence); + // In flight: an earlier read's downloads cover the area, only FinishRead is left. + if (!raster_area.in_flight) { + const u64 fence = RequestSyncOperation([this, raster_area, &timing]() { + timing.start = Stats::Now(); + const Stats::HostWaits host_before = Stats::t_host_waits; + const u64 bytes_before = Stats::t_download_bytes; + // Apply collected guest writes before downloading GPU data over guest memory. + InvalidateGPUCache(); + timing.applied = Stats::Now(); + renderer->ReadRasterizer()->FlushRegionForRead( + raster_area.start_address, raster_area.end_address - raster_area.start_address); + timing.host_ns = Stats::t_host_waits.ns - host_before.ns; + timing.finishes = Stats::t_host_waits.count - host_before.count; + timing.bytes = Stats::t_download_bytes - bytes_before; + timing.end = Stats::Now(); + }); + // The split's back runs it after the entry it is on. The tick sent through the GPU + // thread's queue still runs it without the split. + VideoCommon::GpuSplit::Get().RequestTick(); + gpu_thread.TickGPU(is_async); + WaitForSyncOperation(fence); + } + // The GPU thread submitted the downloads without waiting for the host GPU: this thread + // waits for them and writes them to guest memory, while the GPU thread goes on. + renderer->ReadRasterizer()->FinishRead(raster_area.start_address, + raster_area.end_address - raster_area.start_address); Stats::RecordReactive(raster_area, core, request, timing, Stats::Now()); return raster_area; } @@ -416,6 +440,10 @@ struct GPU::Impl { std::mutex sync_request_mutex; std::condition_variable sync_request_cv; + /// InvalidateGPUCache: the writes it gathered and is applying, on each thread doing so. + std::vector>*> applying_writes; + std::mutex applying_writes_mutex; + const bool is_async; std::unique_ptr cpu_context; diff --git a/src/video_core/memory_manager.cpp b/src/video_core/memory_manager.cpp index ba406b22f3..8bf93f9ec1 100644 --- a/src/video_core/memory_manager.cpp +++ b/src/video_core/memory_manager.cpp @@ -328,6 +328,12 @@ T MemoryManager::Read(GPUVAddr addr) const { template void MemoryManager::Write(GPUVAddr addr, T data) { + // A download for a guest read not yet in guest memory would land over this write later. + if (VideoCore::g_read_downloads.load(std::memory_order_relaxed) != 0) [[unlikely]] { + if (const auto address = GpuToCpuAddress(addr); address && rasterizer != nullptr) { + rasterizer->FinishRead(*address, sizeof(T)); + } + } if (auto page_pointer{GetPointer(addr)}; page_pointer) { // NOTE: Avoid adding any extra logic to this fast-path block std::memcpy(page_pointer, &data, sizeof(T)); @@ -528,8 +534,17 @@ void MemoryManager::WriteBlockImpl(GPUVAddr gpu_dest_addr, const void* src_buffe auto just_advance = [&]([[maybe_unused]] std::size_t page_index, [[maybe_unused]] std::size_t offset, std::size_t copy_amount) { src_buffer = static_cast(src_buffer) + copy_amount; }; + // A download for a guest read not yet in guest memory would land over this write later. + const auto finish_reads = [this](DAddr dev_addr, std::size_t amount) { + if (VideoCore::g_read_downloads.load(std::memory_order_relaxed) != 0) [[unlikely]] { + if (rasterizer != nullptr) { + rasterizer->FinishRead(dev_addr, amount); + } + } + }; auto mapped_normal = [&](std::size_t page_index, std::size_t offset, std::size_t copy_amount) { const DAddr dev_addr_base = (DAddr(page_table[page_index]) << cpu_page_bits) + offset; + finish_reads(dev_addr_base, copy_amount); if (!unsafe) { rasterizer->PrepareRegionWrite(dev_addr_base, copy_amount, which); } @@ -544,6 +559,7 @@ void MemoryManager::WriteBlockImpl(GPUVAddr gpu_dest_addr, const void* src_buffe }; auto mapped_big = [&](std::size_t page_index, std::size_t offset, std::size_t copy_amount) { const DAddr dev_addr_base = (DAddr(big_page_table_dev[page_index]) << cpu_page_bits) + offset; + finish_reads(dev_addr_base, copy_amount); if (!unsafe) { rasterizer->PrepareRegionWrite(dev_addr_base, copy_amount, which); } diff --git a/src/video_core/rasterizer_download_area.h b/src/video_core/rasterizer_download_area.h index c5d6cf17ee..8f8041e435 100644 --- a/src/video_core/rasterizer_download_area.h +++ b/src/video_core/rasterizer_download_area.h @@ -3,6 +3,8 @@ #pragma once +#include + #include "common/common_types.h" namespace VideoCore { @@ -19,6 +21,14 @@ struct RasterizerDownloadArea { VAddr end_address; bool preemtive; DownloadAreaSource source{}; + /// What guest memory lacks there is on its way for an earlier read: nothing to flush, only to + /// wait for (RasterizerInterface::FinishRead). + bool in_flight{}; }; +/// Downloads for guest CPU reads that are submitted but not yet written to guest memory +/// (RasterizerInterface::FlushRegionForRead). While there are any, a write to guest memory from +/// the GPU's side first finishes those it overlaps (FinishRead), so none lands over it later. +inline std::atomic g_read_downloads{}; + } // namespace VideoCore diff --git a/src/video_core/rasterizer_interface.h b/src/video_core/rasterizer_interface.h index 5028e4211e..198ef8c27a 100644 --- a/src/video_core/rasterizer_interface.h +++ b/src/video_core/rasterizer_interface.h @@ -107,6 +107,17 @@ class RasterizerInterface { virtual void FlushRegion(DAddr addr, u64 size, VideoCommon::CacheType which = VideoCommon::CacheType::All) = 0; + /// FlushRegion for a guest CPU read of the region (GPU::OnCPURead), on the GPU thread: a + /// cache may submit its downloads without waiting for the host GPU, for FinishRead to write. + virtual void FlushRegionForRead(DAddr addr, u64 size) { + FlushRegion(addr, size); + } + + /// Writes to guest memory what FlushRegionForRead submitted over the region (and anything + /// submitted before it), waiting for the host GPU if it must. Any thread: the reading one + /// before it reads, and the GPU's side before it writes guest memory there. + virtual void FinishRead([[maybe_unused]] DAddr addr, [[maybe_unused]] u64 size) {} + /// Switch 2, from the GPU thread's front: where a GPU-written command list's words are. enum class GpuWrittenList { Unknown, ///< Neither: the caller drains, flushes the page and notes it synced diff --git a/src/video_core/renderer_vulkan/vk_buffer_cache.cpp b/src/video_core/renderer_vulkan/vk_buffer_cache.cpp index 8e511165b5..7bc0e9760a 100644 --- a/src/video_core/renderer_vulkan/vk_buffer_cache.cpp +++ b/src/video_core/renderer_vulkan/vk_buffer_cache.cpp @@ -514,6 +514,17 @@ void BufferCacheRuntime::FinishForHostRead() { scheduler.Finish(); } +u64 BufferCacheRuntime::SubmitForHostRead() { + scheduler.RequestOutsideRenderPassOperationContext(); + scheduler.Record(HostReadBarrier); + return scheduler.Flush(); +} + +void BufferCacheRuntime::WaitForHostRead(u64 tick) { + // Flushed at once by SubmitForHostRead: the worker submits it without the GPU thread. + scheduler.GetMasterSemaphore().Wait(tick); +} + bool BufferCacheRuntime::CanReorderUpload(const Buffer& buffer, std::span copies) { if (Settings::values.disable_buffer_reorder) { diff --git a/src/video_core/renderer_vulkan/vk_buffer_cache.h b/src/video_core/renderer_vulkan/vk_buffer_cache.h index 0a1b8bb486..4703e99c11 100644 --- a/src/video_core/renderer_vulkan/vk_buffer_cache.h +++ b/src/video_core/renderer_vulkan/vk_buffer_cache.h @@ -134,6 +134,12 @@ class BufferCacheRuntime { void FinishForHostRead(); + /// FinishForHostRead without the wait: submits, and returns the tick to wait for. + u64 SubmitForHostRead(); + + /// Waits for a SubmitForHostRead tick, on any thread. + void WaitForHostRead(u64 tick); + u64 GetDeviceLocalMemory() const; u64 GetDeviceMemoryUsage() const; diff --git a/src/video_core/renderer_vulkan/vk_rasterizer.cpp b/src/video_core/renderer_vulkan/vk_rasterizer.cpp index c66da49d62..add7c48530 100644 --- a/src/video_core/renderer_vulkan/vk_rasterizer.cpp +++ b/src/video_core/renderer_vulkan/vk_rasterizer.cpp @@ -2012,6 +2012,28 @@ void RasterizerVulkan::FlushRegion(DAddr addr, u64 size, VideoCommon::CacheType } } +void RasterizerVulkan::FlushRegionForRead(DAddr addr, u64 size) { + if (addr == 0 || size == 0) { + return; + } + // As FlushRegion, but the buffer cache doesn't wait for the host GPU here (Scheduler::Finish, + // with its lock held): the reading thread does, in FinishRead. + gpu.InvalidateGPUCache(); + { + std::scoped_lock lock{texture_cache.mutex}; + texture_cache.DownloadMemory(addr, size); + } + { + std::scoped_lock lock{buffer_cache.mutex}; + buffer_cache.DownloadMemoryForRead(addr, size); + } + query_cache.FlushRegion(addr, size); +} + +void RasterizerVulkan::FinishRead(DAddr addr, u64 size) { + buffer_cache.FinishReadDownloads(addr, size); +} + bool RasterizerVulkan::MustFlushRegion(DAddr addr, u64 size, VideoCommon::CacheType which) { // The split's front asks this before each macro DMA read, and the back holds the buffer // cache's lock through each draw: pages the GPU never wrote need no look under it. diff --git a/src/video_core/renderer_vulkan/vk_rasterizer.h b/src/video_core/renderer_vulkan/vk_rasterizer.h index af899140e8..1220dd74f1 100644 --- a/src/video_core/renderer_vulkan/vk_rasterizer.h +++ b/src/video_core/renderer_vulkan/vk_rasterizer.h @@ -105,6 +105,8 @@ class RasterizerVulkan final : public VideoCore::RasterizerInterface, void FlushAll() override; void FlushRegion(DAddr addr, u64 size, VideoCommon::CacheType which = VideoCommon::CacheType::All) override; + void FlushRegionForRead(DAddr addr, u64 size) override; + void FinishRead(DAddr addr, u64 size) override; GpuWrittenList ReadGpuWrittenCommandList(DAddr addr, std::span out, bool past_sync) override; void NoteGpuWrittenCommandListSynced(DAddr page) override; From f6fc343a7732c048292cc65afea9adbd5d55ea87 Mon Sep 17 00:00:00 2001 From: jpkto <255363096+jpkto@users.noreply.github.com> Date: Sat, 7 Nov 2026 22:38:47 +0100 Subject: [PATCH 018/105] TEST: compare read downloads with a synchronous flush (EDEN_NX2_READ_CHECK) EDEN_NX2_READ_CHECK=n checks 1 in n read downloads (1 checks them all). The GPU thread copies a second time, in the same submission, what a synchronous flush of the read would have copied. Once the download is written, guest memory is compared with that copy, except where a newer value has replaced it since (CPU writes, ClearDownload). Each byte a synchronous flush would copy must also be in this download or an earlier one. Every 10 seconds the log has a line "Read download check (EDEN_NX2_READ_CHECK): ... bytes differ, ... GPU-modified bytes no download brought (both must stay 0)". Each difference also gets an error line, up to 16 of them. The "GPU waits" line now shows separately from the wake-up how long reading threads spent waiting for their downloads and copying them to guest memory ("then copies on the reading thread"). It also shows how many reads only waited for an earlier read's downloads. "downloaded" counts the read downloads again. Co-Authored-By: Claude Opus 5.5 --- src/video_core/buffer_cache/buffer_cache.h | 149 ++++++++++++++++++ .../buffer_cache/buffer_cache_base.h | 33 ++++ src/video_core/gpu.cpp | 6 +- src/video_core/gpu_wait_stats.h | 34 ++-- 4 files changed, 210 insertions(+), 12 deletions(-) diff --git a/src/video_core/buffer_cache/buffer_cache.h b/src/video_core/buffer_cache/buffer_cache.h index 4d31ee972d..b84f9c557d 100644 --- a/src/video_core/buffer_cache/buffer_cache.h +++ b/src/video_core/buffer_cache/buffer_cache.h @@ -10,6 +10,8 @@ #include #include +#include +#include #include #include #include @@ -484,6 +486,35 @@ void BufferCache

::DownloadMemoryForRead(DAddr device_addr, u64 size) { download.clean_ranges.Add(begin, end - begin); read_download_ranges.Add(begin, end - begin); }; + // TEST (EDEN_NX2_READ_CHECK): what a synchronous flush (DownloadRangesMemory) copies for + // the read itself. + std::unique_ptr check; + boost::container::small_vector check_copies; + u64 check_size = 0; + if (ReadCheckEvery() != 0 && read_check.next++ % ReadCheckEvery() == 0) [[unlikely]] { + check = std::make_unique(); + ForEachBufferInRange(device_addr, size, [&](BufferId buffer_id, Buffer& buffer) { + const DAddr buffer_addr = buffer.CpuAddr(); + const DAddr begin = (std::max)(device_addr, buffer_addr); + const DAddr end = (std::min)(device_addr + size, buffer_addr + buffer.SizeBytes()); + if (begin >= end) { + return; + } + memory_tracker.ForEachDownloadRange( + begin, end - begin, false, [&](u64 run_addr, u64 run_size) { + gpu_modified_ranges.ForEachInRange( + run_addr, run_size, [&](DAddr start, DAddr range_end) { + check_copies.push_back( + {buffer_id, BufferCopy{ + .src_offset = start - buffer_addr, + .dst_offset = check_size, + .size = range_end - start, + }}); + check_size += Common::AlignUp(range_end - start, 64); + }); + }); + }); + } boost::container::small_vector, 8> gaps; for (const auto& [range_addr, range_size] : ranges) { ForEachBufferInRange(range_addr, range_size, [&](BufferId buffer_id, Buffer& buffer) { @@ -523,9 +554,38 @@ void BufferCache

::DownloadMemoryForRead(DAddr device_addr, u64 size) { }); }); } + if (check) [[unlikely]] { + // Each byte a synchronous flush copies is in this download or an earlier one. + for (const Copy& entry : check_copies) { + const DAddr start = slot_buffers[entry.buffer_id].CpuAddr() + entry.copy.src_offset; + Common::RangeSet rest; + rest.Add(start, entry.copy.size); + const auto take_out = [&](const Common::RangeSet& taken) { + taken.ForEachInRange(start, entry.copy.size, [&](DAddr lower, DAddr upper) { + rest.Subtract(lower, upper - lower); + }); + }; + take_out(download.write_ranges); + for (const ReadDownload& earlier : read_downloads) { + take_out(earlier.write_ranges); + } + rest.ForEach([&](DAddr lower, DAddr upper) { + if (read_check.logged++ < 16) { + LOG_ERROR(HW_GPU, + "Read download check: {:#x}+{:#x} of the read [{:#x}, {:#x}) " + "is GPU-modified but no download brings it", + lower, upper - lower, device_addr, device_addr + size); + } + read_check.uncovered += upper - lower; + }); + } + } if (copies.empty()) { return; } + for (const Copy& entry : copies) { + GpuWaitStats::AddDownloadBytes(entry.copy.size); + } auto staging = runtime.DownloadStagingBuffer(total_size_bytes, true); for (Copy& entry : copies) { Buffer& buffer = slot_buffers[entry.buffer_id]; @@ -540,6 +600,22 @@ void BufferCache

::DownloadMemoryForRead(DAddr device_addr, u64 size) { .size = entry.copy.size, }); } + if (check && !check_copies.empty()) [[unlikely]] { + // The same bytes again, from the same point of the GPU's work: what a synchronous + // flush would have written. + check->staging = runtime.DownloadStagingBuffer(check_size, true); + for (Copy& entry : check_copies) { + Buffer& buffer = slot_buffers[entry.buffer_id]; + entry.copy.dst_offset += check->staging.offset; + runtime.CopyBuffer(check->staging.buffer, buffer, std::span{&entry.copy, 1}, true); + check->copies.push_back(BufferCopy{ + .src_offset = buffer.CpuAddr() + entry.copy.src_offset, + .dst_offset = entry.copy.dst_offset, + .size = entry.copy.size, + }); + } + download.check = std::move(check); + } download.tick = runtime.SubmitForHostRead(); download.staging = staging; read_downloads.push_back(std::move(download)); @@ -589,6 +665,10 @@ void BufferCache

::WriteReadDownloads(u64 tick) { end - start); }); } + if (download.check) [[unlikely]] { + CheckReadDownload(*download.check, cpu_written); + async_buffers_death_ring.push_back(download.check->staging); + } download.clean_ranges.ForEach([&](DAddr start, DAddr end) { gpu_modified_ranges.Subtract(start, end - start); read_download_ranges.Subtract(start, end - start); @@ -635,6 +715,72 @@ bool BufferCache

::ReadDownloadsCover(DAddr device_addr, u64 size) const { return covered; } +template +void BufferCache

::CheckReadDownload(const ReadCheck& check, + const Common::RangeSet& cpu_written) { + // Guest memory now holds this download and the earlier ones: compare it with the synchronous + // copy, except where a newer value replaced it since (CPU writes, ClearDownload). Writes + // collected since the download was written count as newer too. + const Common::RangeSet cpu_written_since = PendingCpuWriteRanges(); + const u8* const base = check.staging.mapped_span.data(); + std::vector guest; + for (const BufferCopy& copy : check.copies) { + const DAddr copy_addr = static_cast(copy.src_offset); + const u8* const expected = base + (copy.dst_offset - check.staging.offset); + Common::RangeSet compared; + compared.Add(copy_addr, copy.size); + const auto take_out = [&](const Common::RangeSet& taken) { + taken.ForEachInRange(copy_addr, copy.size, [&](DAddr lower, DAddr upper) { + compared.Subtract(lower, upper - lower); + }); + }; + take_out(check.changed); + take_out(cpu_written); + take_out(cpu_written_since); + compared.ForEach([&](DAddr start, DAddr end) { + const u64 length = end - start; + guest.resize(length); + device_memory.ReadBlockUnsafe(start, guest.data(), length); + read_check.compared += length; + const u8* const wanted = expected + (start - copy_addr); + if (std::memcmp(guest.data(), wanted, length) == 0) { + return; + } + for (u64 i = 0; i < length;) { + if (guest[i] == wanted[i]) { + ++i; + continue; + } + u64 run_end = i; + while (run_end < length && guest[run_end] != wanted[run_end]) { + ++run_end; + } + if (read_check.logged++ < 16) { + LOG_ERROR(HW_GPU, + "Read download check: guest memory {:#x}+{:#x} differs from a " + "synchronous flush ({:#04x} for {:#04x} at its start)", + start + i, run_end - i, guest[i], wanted[i]); + } + read_check.mismatched += run_end - i; + i = run_end; + } + }); + } + ++read_check.checked; + const u64 now = static_cast(std::chrono::duration_cast( + std::chrono::steady_clock::now().time_since_epoch()) + .count()); + if (now >= read_check.last_log + 10) { + read_check.last_log = now; + LOG_INFO(HW_GPU, + "Read download check (EDEN_NX2_READ_CHECK): {} downloads compared with a " + "synchronous flush, {} KiB; {} bytes differ, {} GPU-modified bytes no download " + "brought (both must stay 0)", + read_check.checked, read_check.compared / 1024, read_check.mismatched, + read_check.uncovered); + } +} + template void BufferCache

::ClearDownload(DAddr device_addr, u64 size) { ClearFenceDownloads(device_addr, size); @@ -642,6 +788,9 @@ void BufferCache

::ClearDownload(DAddr device_addr, u64 size) { for (ReadDownload& download : read_downloads) { download.write_ranges.Subtract(device_addr, size); download.clean_ranges.Subtract(device_addr, size); + if (download.check) [[unlikely]] { + download.check->changed.Add(device_addr, size); + } } read_download_ranges.Subtract(device_addr, size); } diff --git a/src/video_core/buffer_cache/buffer_cache_base.h b/src/video_core/buffer_cache/buffer_cache_base.h index ac832ec19c..9e24058375 100644 --- a/src/video_core/buffer_cache/buffer_cache_base.h +++ b/src/video_core/buffer_cache/buffer_cache_base.h @@ -778,6 +778,38 @@ class BufferCache : public VideoCommon::ChannelSetupCaches current_buffer; + /// TEST (EDEN_NX2_READ_CHECK=n, 1 in n read downloads): what a synchronous flush of the read + /// would have copied, copied again in the same submission, to compare guest memory with once + /// the download is written. + struct ReadCheck { + Async_Buffer staging{}; + /// src_offset: the device address copied; dst_offset: where it is in `staging`. + boost::container::small_vector copies; + /// Replaced in guest memory since (ClearDownload): not compared. + Common::RangeSet changed; + }; + struct ReadCheckTotals { + u64 next{}; ///< Read downloads seen, for the sampling + u64 checked{}; ///< Read downloads compared + u64 compared{}; ///< Bytes compared + u64 mismatched{}; ///< Bytes that differ from the synchronous copy + u64 uncovered{}; ///< Bytes a synchronous flush copies that no download brings + u64 logged{}; ///< Error lines so far (at most 16) + u64 last_log{}; + }; + ReadCheckTotals read_check{}; + static u64 ReadCheckEvery() { + static const u64 every = [] { + const char* value = std::getenv("EDEN_NX2_READ_CHECK"); + if (value == nullptr) { + return u64{0}; + } + return (std::max)(u64{1}, static_cast(std::strtoull(value, nullptr, 10))); + }(); + return every; + } + void CheckReadDownload(const ReadCheck& check, const Common::RangeSet& cpu_written); + /// A DownloadMemoryForRead: its copies, and what is left of them to do once the host GPU has /// run them. struct ReadDownload { @@ -792,6 +824,7 @@ class BufferCache : public VideoCommon::ChannelSetupCaches clean_ranges; + std::unique_ptr check; ///< TEST: EDEN_NX2_READ_CHECK }; /// Oldest first, and written to guest memory in that order: none lands over newer bytes. std::deque read_downloads; diff --git a/src/video_core/gpu.cpp b/src/video_core/gpu.cpp index 0542ea2581..8b634afe7a 100644 --- a/src/video_core/gpu.cpp +++ b/src/video_core/gpu.cpp @@ -309,9 +309,13 @@ struct GPU::Impl { } // The GPU thread submitted the downloads without waiting for the host GPU: this thread // waits for them and writes them to guest memory, while the GPU thread goes on. + const u64 finish_start = Stats::Now(); renderer->ReadRasterizer()->FinishRead(raster_area.start_address, raster_area.end_address - raster_area.start_address); - Stats::RecordReactive(raster_area, core, request, timing, Stats::Now()); + const u64 woke = Stats::Now(); + timing.finish_ns = woke - finish_start; + timing.in_flight = raster_area.in_flight; + Stats::RecordReactive(raster_area, core, request, timing, woke); return raster_area; } diff --git a/src/video_core/gpu_wait_stats.h b/src/video_core/gpu_wait_stats.h index 70656ffb56..49c02bb5da 100644 --- a/src/video_core/gpu_wait_stats.h +++ b/src/video_core/gpu_wait_stats.h @@ -25,13 +25,14 @@ // flush on the GPU thread and waits for it. Each wait is timed from the request to the wake-up and // split into: queue (until the GPU thread starts the flush), flush (applying the collected CPU // writes, then the downloads; "host GPU" is the time in Vulkan's Scheduler::Finish, waiting for -// the host GPU) and wake (until the waiting thread runs again). Also counted: guest reads and -// writes that take the rasterizer path, by core, with the time of their cache lookups -// (GetFlushArea, OnCPUWrite: both take the cache locks the GPU thread holds while it draws); the -// other sync operations guest threads wait for; the GPU thread's (and the split's back thread's) -// idle time; every Scheduler::Finish and every wait for pending query results (FlushQueryResults); -// the time between presents; and, for the pages that waited longest, the GPU writes that marked -// them. +// the host GPU), wake (until the waiting thread runs again) and the copies (the waiting thread's +// own wait for the downloads the flush submitted, and writing them to guest memory: FinishRead). +// Also counted: guest reads and writes that take the rasterizer path, by core, with the time of +// their cache lookups (GetFlushArea, OnCPUWrite: both take the cache locks the GPU thread holds +// while it draws); the other sync operations guest threads wait for; the GPU thread's (and the +// split's back thread's) idle time; every Scheduler::Finish and every wait for pending query +// results (FlushQueryResults); the time between presents; and, for the pages that waited longest, +// the GPU writes that marked them. namespace VideoCommon::GpuWaitStats { /// One slot per guest core, the last one for every other thread. @@ -104,6 +105,10 @@ struct FlushTiming { u64 host_ns{}; ///< Of the flush: time in Scheduler::Finish u64 finishes{}; u64 bytes{}; ///< Downloaded by the flush + /// The reading thread, after the flush: waiting for its downloads and writing them to guest + /// memory (RasterizerInterface::FinishRead) + u64 finish_ns{}; + bool in_flight{}; ///< No flush: only waited for an earlier read's downloads }; struct HostWaits { @@ -294,6 +299,8 @@ struct ReactiveTotals { u64 bytes{}; u64 max_bytes{}; u64 wake_ns{}; + u64 finish_ns{}; + u64 in_flight{}; u64 other_areas{}; ///< Flushes of areas past MaxAreas std::array count_by_source{}; std::array ns_by_source{}; @@ -314,12 +321,15 @@ inline void RecordReactive(const VideoCore::RasterizerDownloadArea& area, size_t ++totals.count; totals.wait_ns += wait; totals.max_ns = (std::max)(totals.max_ns, wait); - if (timing.start >= request && timing.end >= timing.start && woke >= timing.end) { + if (timing.start >= request && timing.end >= timing.start && + woke >= timing.end + timing.finish_ns) { totals.queue_ns += timing.start - request; totals.apply_ns += timing.applied - timing.start; totals.flush_ns += timing.end - timing.start; - totals.wake_ns += woke - timing.end; + totals.wake_ns += woke - timing.end - timing.finish_ns; } + totals.finish_ns += timing.finish_ns; + totals.in_flight += timing.in_flight ? 1 : 0; totals.host_ns += timing.host_ns; totals.finishes += timing.finishes; totals.without_finish += timing.finishes == 0 ? 1 : 0; @@ -466,10 +476,12 @@ inline void MaybeReport(bool split_active) { end_list(); line += fmt::format(" | queue {:.1f} ms, flush {:.1f} ms (CPU writes {:.1f}, host GPU " "{:.1f} in {} finishes, {} flushes without one; downloaded {} KiB, " - "at most {} KiB), wake {:.1f} ms | by core:", + "at most {} KiB), wake {:.1f} ms, then copies on the reading thread " + "{:.1f} ms ({} reads only waited for an earlier read's) | by core:", Ms(reactive.queue_ns), Ms(reactive.flush_ns), Ms(reactive.apply_ns), Ms(reactive.host_ns), reactive.finishes, reactive.without_finish, - reactive.bytes / 1024, reactive.max_bytes / 1024, Ms(reactive.wake_ns)); + reactive.bytes / 1024, reactive.max_bytes / 1024, Ms(reactive.wake_ns), + Ms(reactive.finish_ns), reactive.in_flight); for (size_t i = 0; i < CoreSlots; ++i) { if (reactive.ns_by_core[i] == 0) { continue; From 4c6864af91b2dc9501ba29c71ef2ec2636f957b1 Mon Sep 17 00:00:00 2001 From: jpkto <255363096+jpkto@users.noreply.github.com> Date: Sat, 7 Nov 2026 23:49:25 +0100 Subject: [PATCH 019/105] video_core: land read downloads under query results, outdate them for query copies Two writers went around the read downloads of 6f8c97dd86. The query cache writes report values and semaphore payloads into guest memory through device memory pointers, not through the memory manager: the completion operation, the immediate payload path and SemiFlushQueryDirty. A read's download still on its way over those bytes landed the GPU's older copy over them later. FlushRegionForRead even wrote them right after taking the download, where a synchronous flush had written them after its copy. They now land the downloads first, as MemoryManager::Write does. SyncValues copies query results into the buffer cache's buffers on the GPU without recording a write. A read download taken before the copy kept its bytes as the newest, so a later read skipped them. OutdateReadDownloads, which MarkWrittenBuffer now calls as well, takes them out of the earlier downloads' clean ranges, and the later read downloads them again. Both ask ReadDownloadsMayCover first: whether a read download is pending, and on Switch 2 whether the bytes' pages were ever GPU-written. The memory manager's writes ask it too, so semaphore words and constant buffer data on other pages no longer take the buffer cache's lock while a read is pending. SemiFlushQueryDirty runs with the query cache's cache_mutex held. Nothing that holds the buffer cache's lock takes cache_mutex, so the new lock order cannot deadlock. Tests: the next read downloads a query copy made after an earlier read's download again. On Switch 2, a write to a page no GPU write reached doesn't wait for a pending download, and a write to a GPU-written page lands it first. Co-Authored-By: Claude Opus 5.5 --- src/tests/video_core/buffer_cache_ports.cpp | 62 +++++++++++++++++++ src/video_core/buffer_cache/buffer_cache.h | 20 +++--- .../buffer_cache/buffer_cache_base.h | 6 ++ src/video_core/memory_manager.cpp | 6 +- src/video_core/query_cache/query_cache.h | 13 ++++ src/video_core/query_cache/query_cache_base.h | 3 + src/video_core/rasterizer_download_area.h | 12 ++++ .../renderer_vulkan/vk_query_cache.cpp | 5 ++ 8 files changed, 117 insertions(+), 10 deletions(-) diff --git a/src/tests/video_core/buffer_cache_ports.cpp b/src/tests/video_core/buffer_cache_ports.cpp index 0436750920..70adcf91c4 100644 --- a/src/tests/video_core/buffer_cache_ports.cpp +++ b/src/tests/video_core/buffer_cache_ports.cpp @@ -893,3 +893,65 @@ TEST_CASE("GPU writes to guest memory land the read downloads under them first", REQUIRE(f.bytes[0x10104] == 0x12); REQUIRE(f.bytes[0x10fff] == 0x12); } + +TEST_CASE("Read downloads take again what a GPU write outside the cache's records changed", + "[buffer_ports]") { + Fixture f; + const auto id = Access::Add(*f.cache, 0x20000, 0x10000); + auto& buffer = Access::Get(*f.cache, id); + std::ranges::fill(buffer.bytes, 0x12); + Access::MarkWritten(*f.cache, id, 0x20000, 0x100); + f.cache->DownloadMemoryForRead(0x20000, 0x1000); + // After the read's copy: a query result copy into the buffer (SyncValues), then a GPU write + // elsewhere on the page, which the next read downloads. + f.runtime.pending.push_back( + [&buffer] { std::fill_n(buffer.bytes.begin() + 0x40, 0x10, u8{0x56}); }); + f.cache->OutdateReadDownloads(0x20040, 0x10); + f.runtime.pending.push_back( + [&buffer] { std::fill_n(buffer.bytes.begin() + 0x800, 0x10, u8{0x34}); }); + Access::MarkWritten(*f.cache, id, 0x20800, 0x10); + f.cache->DownloadMemoryForRead(0x20000, 0x1000); + f.cache->FinishReadDownloads(0x20000, 0x1000); + REQUIRE(f.bytes[0x1003f] == 0x12); + REQUIRE(f.bytes[0x10040] == 0x56); + REQUIRE(f.bytes[0x1004f] == 0x56); + REQUIRE(f.bytes[0x10050] == 0x12); + REQUIRE(f.bytes[0x10800] == 0x34); + REQUIRE_FALSE(Access::Pending(*f.cache, 0x20000, 0x1000)); +} + +TEST_CASE("Switch 2 writes to guest memory land read downloads only on GPU-written pages", + "[buffer_ports]") { + Fixture f; + const auto id = Access::Add(*f.cache, 0x20000, 0x10000); + std::ranges::fill(Access::Get(*f.cache, id).bytes, 0x12); + ReadRasterizer reads{*f.cache}; + Tegra::MemoryManager gpu_memory{f.system, f.device, 32, 1ULL << 31, 16}; + gpu_memory.BindRasterizer(&reads); + gpu_memory.Map(0x200000, 0x20000, 0x10000, Tegra::PTEKind::PITCH, true); + struct Nx2Title { + Nx2Title() { + Tegra::g_nx2_ampere = true; + } + ~Nx2Title() { + Tegra::g_nx2_ampere = false; + Tegra::Nx2ClearGpuWrittenPages(); + } + } nx2_title; + Access::MarkWritten(*f.cache, id, 0x20000, 0x1000); + f.cache->DownloadMemoryForRead(0x20000, 0x1000); + // A page no GPU write reached: no download can be on its way there. + gpu_memory.Write(0x201000, 0xdeadbeef); + REQUIRE(VideoCore::g_read_downloads == 1); + REQUIRE(f.runtime.host_read_waits == 0); + gpu_memory.Write(0x200100, 0xcafef00d); + REQUIRE(VideoCore::g_read_downloads == 0); + REQUIRE(f.runtime.host_read_waits == 1); + u32 value{}; + std::memcpy(&value, f.bytes + 0x10100, sizeof(value)); + REQUIRE(value == 0xcafef00d); + std::memcpy(&value, f.bytes + 0x11000, sizeof(value)); + REQUIRE(value == 0xdeadbeef); + REQUIRE(f.bytes[0x10000] == 0x12); + REQUIRE(f.bytes[0x10fff] == 0x12); +} diff --git a/src/video_core/buffer_cache/buffer_cache.h b/src/video_core/buffer_cache/buffer_cache.h index b84f9c557d..d2bc3eca6a 100644 --- a/src/video_core/buffer_cache/buffer_cache.h +++ b/src/video_core/buffer_cache/buffer_cache.h @@ -646,6 +646,17 @@ void BufferCache

::FinishReadDownloads(DAddr device_addr, u64 size) { } } +template +void BufferCache

::OutdateReadDownloads(DAddr device_addr, u64 size) { + // The bytes stay GPU-modified when the downloads land, so a later read downloads them. + if (!read_downloads.empty()) [[unlikely]] { + for (ReadDownload& download : read_downloads) { + download.clean_ranges.Subtract(device_addr, size); + } + read_download_ranges.Subtract(device_addr, size); + } +} + template void BufferCache

::WriteReadDownloads(u64 tick) { if constexpr (!IS_OPENGL) { @@ -2442,14 +2453,7 @@ void BufferCache

::MarkWrittenBuffer(BufferId buffer_id, DAddr device_addr, u3 for (auto& ranges : pending_download_ranges) { ranges.Subtract(device_addr, size); } - // A read download taken before still writes these bytes to guest memory, as a synchronous - // one would have, but they stay GPU-modified: a later read downloads them again. - if (!read_downloads.empty()) [[unlikely]] { - for (ReadDownload& download : read_downloads) { - download.clean_ranges.Subtract(device_addr, size); - } - read_download_ranges.Subtract(device_addr, size); - } + OutdateReadDownloads(device_addr, size); memory_tracker.MarkRegionAsGpuModified(device_addr, size); MarkVertexBuffersStale(device_addr, size); gpu_modified_ranges.Add(device_addr, size); diff --git a/src/video_core/buffer_cache/buffer_cache_base.h b/src/video_core/buffer_cache/buffer_cache_base.h index 9e24058375..9d4afe641e 100644 --- a/src/video_core/buffer_cache/buffer_cache_base.h +++ b/src/video_core/buffer_cache/buffer_cache_base.h @@ -281,6 +281,12 @@ class BufferCache : public VideoCommon::ChannelSetupCaches GetFlushArea(DAddr device_addr, u64 size); /// Without downloads at every fence, a guest CPU read is the only way GPU-written buffer diff --git a/src/video_core/memory_manager.cpp b/src/video_core/memory_manager.cpp index 8bf93f9ec1..6ba3b3b6f3 100644 --- a/src/video_core/memory_manager.cpp +++ b/src/video_core/memory_manager.cpp @@ -330,7 +330,9 @@ template void MemoryManager::Write(GPUVAddr addr, T data) { // A download for a guest read not yet in guest memory would land over this write later. if (VideoCore::g_read_downloads.load(std::memory_order_relaxed) != 0) [[unlikely]] { - if (const auto address = GpuToCpuAddress(addr); address && rasterizer != nullptr) { + if (const auto address = GpuToCpuAddress(addr); + address && rasterizer != nullptr && + VideoCore::ReadDownloadsMayCover(*address, sizeof(T))) { rasterizer->FinishRead(*address, sizeof(T)); } } @@ -537,7 +539,7 @@ void MemoryManager::WriteBlockImpl(GPUVAddr gpu_dest_addr, const void* src_buffe // A download for a guest read not yet in guest memory would land over this write later. const auto finish_reads = [this](DAddr dev_addr, std::size_t amount) { if (VideoCore::g_read_downloads.load(std::memory_order_relaxed) != 0) [[unlikely]] { - if (rasterizer != nullptr) { + if (rasterizer != nullptr && VideoCore::ReadDownloadsMayCover(dev_addr, amount)) { rasterizer->FinishRead(dev_addr, amount); } } diff --git a/src/video_core/query_cache/query_cache.h b/src/video_core/query_cache/query_cache.h index efa148bf06..e906e68e5e 100644 --- a/src/video_core/query_cache/query_cache.h +++ b/src/video_core/query_cache/query_cache.h @@ -316,6 +316,7 @@ void QueryCacheBase::CounterReport(GPUVAddr addr, QueryType counter_type } query_base->value += streamer->GetAmendValue(); streamer->SetAccumulationValue(query_base->value); + FinishReads(cpu_addr, True(query_base->flags & QueryFlagBits::HasTimestamp) ? 16 : 4); if (True(query_base->flags & QueryFlagBits::HasTimestamp)) { u64 timestamp = impl->gpu.GetTicks(); std::memcpy(pointer_timestamp, ×tamp, sizeof(timestamp)); @@ -342,6 +343,7 @@ void QueryCacheBase::CounterReport(GPUVAddr addr, QueryType counter_type impl->rasterizer.SignalFence(std::move(operation)); } else { if (!Settings::IsGPULevelHigh() && counter_type == QueryType::Payload) { + FinishReads(cpu_addr, has_timestamp ? 16 : 4); if (has_timestamp) { u64 timestamp = impl->gpu.GetTicks(); u64 value = static_cast(payload); @@ -633,6 +635,8 @@ bool QueryCacheBase::SemiFlushQueryDirty(QueryCacheBase::QueryLo } if (True(query_base->flags & QueryFlagBits::IsFinalValueSynced) && False(query_base->flags & QueryFlagBits::IsGuestSynced)) { + FinishReads(query_base->guest_address, + True(query_base->flags & QueryFlagBits::HasTimestamp) ? 8 : 4); auto* ptr = impl->device_memory.template GetPointer(query_base->guest_address); if (True(query_base->flags & QueryFlagBits::HasTimestamp)) { std::memcpy(ptr, &query_base->value, sizeof(query_base->value)); @@ -656,6 +660,15 @@ bool QueryCacheBase::SemiFlushQueryDirty(QueryCacheBase::QueryLo False(query_base->flags & QueryFlagBits::IsGuestSynced); } +template +void QueryCacheBase::FinishReads(DAddr address, size_t size) { + // SemiFlushQueryDirty asks with cache_mutex held, which no holder of the buffer cache's lock + // takes. + if (VideoCore::ReadDownloadsMayCover(address, size)) [[unlikely]] { + impl->rasterizer.FinishRead(address, size); + } +} + template void QueryCacheBase::RequestGuestHostSync() { impl->rasterizer.ReleaseFences(); diff --git a/src/video_core/query_cache/query_cache_base.h b/src/video_core/query_cache/query_cache_base.h index 14678c5831..78cce90766 100644 --- a/src/video_core/query_cache/query_cache_base.h +++ b/src/video_core/query_cache/query_cache_base.h @@ -215,6 +215,9 @@ class QueryCacheBase : public VideoCommon::ChannelSetupCaches> cached_queries; std::mutex cache_mutex; diff --git a/src/video_core/rasterizer_download_area.h b/src/video_core/rasterizer_download_area.h index 8f8041e435..4eb0a6f604 100644 --- a/src/video_core/rasterizer_download_area.h +++ b/src/video_core/rasterizer_download_area.h @@ -6,6 +6,7 @@ #include #include "common/common_types.h" +#include "video_core/nx2.h" namespace VideoCore { @@ -31,4 +32,15 @@ struct RasterizerDownloadArea { /// the GPU's side first finishes those it overlaps (FinishRead), so none lands over it later. inline std::atomic g_read_downloads{}; +/// Whether such a download may still land over [addr, addr + size): what a write to guest memory +/// from the GPU's side asks before FinishRead, on any thread, without the buffer cache's lock. +/// The downloads take only GPU-modified bytes, and Switch 2 titles mark their pages +/// (Tegra::Nx2MayBeGpuWritten): writes to other pages skip the lock, which the GPU thread holds +/// for nearly every command. +[[nodiscard]] inline bool ReadDownloadsMayCover(DAddr addr, u64 size) { + return g_read_downloads.load(std::memory_order_acquire) != 0 && + (!Tegra::g_nx2_ampere.load(std::memory_order_relaxed) || + Tegra::Nx2MayBeGpuWritten(addr, size)); +} + } // namespace VideoCore diff --git a/src/video_core/renderer_vulkan/vk_query_cache.cpp b/src/video_core/renderer_vulkan/vk_query_cache.cpp index d7b14b0565..76d4f129e4 100644 --- a/src/video_core/renderer_vulkan/vk_query_cache.cpp +++ b/src/video_core/renderer_vulkan/vk_query_cache.cpp @@ -1747,6 +1747,11 @@ void QueryCacheRuntime::SyncValues(std::span values, VkBuffer ba impl->buffers_to_upload_to.emplace_back(buffer->Handle(), offset); } }); + // The copies below are GPU writes the buffer cache doesn't record: reads taken before them + // must not keep the bytes they downloaded as the newest. + for (const auto& value : values) { + impl->buffer_cache.OutdateReadDownloads(value.address, value.size); + } VkBuffer src_buffer; [[maybe_unused]] StagingBufferRef ref; From 7693bb232b824ca2df4e9090af41c35534d907f9 Mon Sep 17 00:00:00 2001 From: jpkto <255363096+jpkto@users.noreply.github.com> Date: Sat, 7 Nov 2026 23:51:37 +0100 Subject: [PATCH 020/105] TEST: read download check: compare kept bytes as they landed The check from a06257737b compared every GPU-modified byte of the read with the GPU's copy at the read, guest memory included where an earlier read's download brought the bytes. After that download lands, the GPU's side can write newer bytes there before the later download lands: constant buffer data or an inline-to-memory write through the memory manager, which lands the earlier download first and tells the buffer cache only at the next FlushCaching, or a query result. Synchronous flushes keep those bytes too: the earlier read's flush had cleaned them, so the later one doesn't copy them. The check reported them as differences. In the harness, constant buffer data written that way gives exactly the test47 line, "guest memory ...+0x30 differs from a synchronous flush", with the newer bytes in guest memory. Each check now notes the bytes earlier downloads bring (kept), and what guest memory holds there right after each of them lands. At its own landing: - Bytes rewritten since are left out and counted ("got newer values after an earlier read's download brought them"). The first 16 are logged with their writer: the memory manager's writes and the query cache's results note themselves while read downloads are pending. - Kept bytes are compared as they landed: a difference there means the GPU wrote them after the earlier download without taking them out of it, and is logged as such. - This download's own bytes are compared as before. The summary line adds the rewritten count. Tests: constant buffer data over an earlier read's download is left out, and a GPU write that skipped OutdateReadDownloads is found. Co-Authored-By: Claude Opus 5.5 --- src/tests/video_core/buffer_cache_ports.cpp | 68 +++++++++ src/video_core/buffer_cache/buffer_cache.h | 140 +++++++++++++++--- .../buffer_cache/buffer_cache_base.h | 13 ++ src/video_core/memory_manager.cpp | 13 +- src/video_core/query_cache/query_cache.h | 3 + src/video_core/rasterizer_download_area.h | 43 ++++++ 6 files changed, 256 insertions(+), 24 deletions(-) diff --git a/src/tests/video_core/buffer_cache_ports.cpp b/src/tests/video_core/buffer_cache_ports.cpp index 70adcf91c4..4a8b46468c 100644 --- a/src/tests/video_core/buffer_cache_ports.cpp +++ b/src/tests/video_core/buffer_cache_ports.cpp @@ -276,6 +276,13 @@ struct BufferCacheTestAccess { static size_t ReadDownloads(Cache& cache) { return cache.read_downloads.size(); } + /// TEST (EDEN_NX2_READ_CHECK): every read download compared with a synchronous flush. + static void CheckEveryRead(Cache& cache) { + cache.read_check.every = 1; + } + static const auto& ReadChecks(Cache& cache) { + return cache.read_check; + } }; } // namespace VideoCommon @@ -955,3 +962,64 @@ TEST_CASE("Switch 2 writes to guest memory land read downloads only on GPU-writt REQUIRE(f.bytes[0x10000] == 0x12); REQUIRE(f.bytes[0x10fff] == 0x12); } + +TEST_CASE("Read download check leaves out newer bytes over an earlier read's download", + "[buffer_ports]") { + Fixture f; + const auto id = Access::Add(*f.cache, 0x20000, 0x10000); + auto& buffer = Access::Get(*f.cache, id); + std::ranges::fill(buffer.bytes, 0x12); + Access::CheckEveryRead(*f.cache); + ReadRasterizer reads{*f.cache}; + Tegra::MemoryManager gpu_memory{f.system, f.device, 32, 1ULL << 31, 16}; + gpu_memory.BindRasterizer(&reads); + gpu_memory.Map(0x200000, 0x20000, 0x10000, Tegra::PTEKind::PITCH, true); + // The first read takes the page's GPU-written bytes, the second only those written since. + Access::MarkWritten(*f.cache, id, 0x20000, 0x100); + f.cache->DownloadMemoryForRead(0x20000, 0x1000); + f.runtime.pending.push_back( + [&buffer] { std::fill_n(buffer.bytes.begin() + 0x800, 0x10, u8{0x34}); }); + Access::MarkWritten(*f.cache, id, 0x20800, 0x10); + f.cache->DownloadMemoryForRead(0x20000, 0x1000); + REQUIRE(Access::ReadDownloads(*f.cache) == 2); + // Constant buffer data over bytes the first read brings: that read lands first, and the + // buffer cache hears of the write at the next FlushCaching. Synchronous flushes leave it too. + std::array words; + words.fill(0xb6); + gpu_memory.WriteBlockCached(0x200060, words.data(), words.size()); + REQUIRE(Access::ReadDownloads(*f.cache) == 1); + f.cache->FinishReadDownloads(0x20000, 0x1000); + REQUIRE(f.bytes[0x1005f] == 0x12); + REQUIRE(f.bytes[0x10060] == 0xb6); + REQUIRE(f.bytes[0x1008f] == 0xb6); + REQUIRE(f.bytes[0x10090] == 0x12); + REQUIRE(f.bytes[0x10800] == 0x34); + REQUIRE(Access::ReadChecks(*f.cache).checked == 2); + REQUIRE(Access::ReadChecks(*f.cache).mismatched == 0); + REQUIRE(Access::ReadChecks(*f.cache).rewritten == 0x30); + REQUIRE(Access::ReadChecks(*f.cache).uncovered == 0); +} + +TEST_CASE("Read download check finds an earlier read's bytes the GPU changed unrecorded", + "[buffer_ports]") { + Fixture f; + const auto id = Access::Add(*f.cache, 0x20000, 0x10000); + auto& buffer = Access::Get(*f.cache, id); + std::ranges::fill(buffer.bytes, 0x12); + Access::CheckEveryRead(*f.cache); + Access::MarkWritten(*f.cache, id, 0x20000, 0x100); + f.cache->DownloadMemoryForRead(0x20000, 0x1000); + // A GPU write into the first read's bytes that nothing takes out of its download. + f.runtime.pending.push_back( + [&buffer] { std::fill_n(buffer.bytes.begin() + 0x40, 0x10, u8{0x56}); }); + f.runtime.pending.push_back( + [&buffer] { std::fill_n(buffer.bytes.begin() + 0x800, 0x10, u8{0x34}); }); + Access::MarkWritten(*f.cache, id, 0x20800, 0x10); + f.cache->DownloadMemoryForRead(0x20000, 0x1000); + f.cache->FinishReadDownloads(0x20000, 0x1000); + REQUIRE(f.bytes[0x10040] == 0x12); + REQUIRE(f.bytes[0x10800] == 0x34); + REQUIRE(Access::ReadChecks(*f.cache).checked == 2); + REQUIRE(Access::ReadChecks(*f.cache).mismatched == 0x10); + REQUIRE(Access::ReadChecks(*f.cache).rewritten == 0); +} diff --git a/src/video_core/buffer_cache/buffer_cache.h b/src/video_core/buffer_cache/buffer_cache.h index d2bc3eca6a..cff7ea03e1 100644 --- a/src/video_core/buffer_cache/buffer_cache.h +++ b/src/video_core/buffer_cache/buffer_cache.h @@ -491,7 +491,8 @@ void BufferCache

::DownloadMemoryForRead(DAddr device_addr, u64 size) { std::unique_ptr check; boost::container::small_vector check_copies; u64 check_size = 0; - if (ReadCheckEvery() != 0 && read_check.next++ % ReadCheckEvery() == 0) [[unlikely]] { + const u64 check_every = read_check.every != 0 ? read_check.every : ReadCheckEvery(); + if (check_every != 0 && read_check.next++ % check_every == 0) [[unlikely]] { check = std::make_unique(); ForEachBufferInRange(device_addr, size, [&](BufferId buffer_id, Buffer& buffer) { const DAddr buffer_addr = buffer.CpuAddr(); @@ -555,19 +556,24 @@ void BufferCache

::DownloadMemoryForRead(DAddr device_addr, u64 size) { }); } if (check) [[unlikely]] { - // Each byte a synchronous flush copies is in this download or an earlier one. + // Each byte a synchronous flush copies is in this download or an earlier one (kept). for (const Copy& entry : check_copies) { const DAddr start = slot_buffers[entry.buffer_id].CpuAddr() + entry.copy.src_offset; Common::RangeSet rest; rest.Add(start, entry.copy.size); - const auto take_out = [&](const Common::RangeSet& taken) { + const auto take_out = [&](const Common::RangeSet& taken, bool earlier_one) { taken.ForEachInRange(start, entry.copy.size, [&](DAddr lower, DAddr upper) { + if (earlier_one) { + rest.ForEachInRange(lower, upper - lower, [&](DAddr from, DAddr to) { + check->kept.Add(from, to - from); + }); + } rest.Subtract(lower, upper - lower); }); }; - take_out(download.write_ranges); + take_out(download.write_ranges, false); for (const ReadDownload& earlier : read_downloads) { - take_out(earlier.write_ranges); + take_out(earlier.write_ranges, true); } rest.ForEach([&](DAddr lower, DAddr upper) { if (read_check.logged++ < 16) { @@ -614,6 +620,9 @@ void BufferCache

::DownloadMemoryForRead(DAddr device_addr, u64 size) { .size = entry.copy.size, }); } + if (!check->kept.Empty()) { + check->landed_bytes.resize(check_size); + } download.check = std::move(check); } download.tick = runtime.SubmitForHostRead(); @@ -676,6 +685,12 @@ void BufferCache

::WriteReadDownloads(u64 tick) { end - start); }); } + // TEST: what later reads' checks find of an earlier download where they kept its bytes. + for (size_t later = 1; later < read_downloads.size(); ++later) { + if (read_downloads[later].check) [[unlikely]] { + NoteLandedBytes(*read_downloads[later].check, download.write_ranges); + } + } if (download.check) [[unlikely]] { CheckReadDownload(*download.check, cpu_written); async_buffers_death_ring.push_back(download.check->staging); @@ -731,13 +746,58 @@ void BufferCache

::CheckReadDownload(const ReadCheck& check, const Common::RangeSet& cpu_written) { // Guest memory now holds this download and the earlier ones: compare it with the synchronous // copy, except where a newer value replaced it since (CPU writes, ClearDownload). Writes - // collected since the download was written count as newer too. + // collected since the download was written count as newer too. Where an earlier download + // brought the bytes (kept), the synchronous flush of its own read left them, and guest memory + // keeps what the GPU's side writes there after that, as without these downloads: those bytes + // are compared as they landed (NoteLandedBytes). const Common::RangeSet cpu_written_since = PendingCpuWriteRanges(); const u8* const base = check.staging.mapped_span.data(); + enum class Source : u8 { + Own, ///< This download brings it + Kept, ///< An earlier one was to bring it, and hasn't + Landed, ///< An earlier one brought it + }; + enum class Result : u8 { + Same, + Rewritten, ///< Newer bytes since an earlier download landed it + Differs, ///< This download's bytes differ + Outdated, ///< An earlier download's bytes differ: the GPU wrote them since + NotLanded, ///< No earlier download wrote it + }; + const auto report = [this](Result result, DAddr address, u64 size, u8 found, u8 wanted) { + if (result == Result::Rewritten) { + read_check.rewritten += size; + if (read_check.logged_rewrites++ < 16) { + LOG_INFO(HW_GPU, + "Read download check: guest memory {:#x}+{:#x} got newer bytes after an " + "earlier read's download brought it, as it does after synchronous " + "flushes: not compared (written by {})", + address, size, VideoCore::ReadCheckWrites::Find(address, size)); + } + return; + } + read_check.mismatched += size; + if (read_check.logged++ >= 16) { + return; + } + const char* const why = + result == Result::Outdated + ? ": an earlier read's download brought it, and the GPU wrote it since without " + "taking it out of that download" + : (result == Result::NotLanded ? ": no earlier read's download wrote it" : ""); + LOG_ERROR(HW_GPU, + "Read download check: guest memory {:#x}+{:#x} differs from a synchronous " + "flush ({:#04x} for {:#04x} at its start){}", + address, size, found, wanted, why); + }; std::vector guest; + std::vector sources; for (const BufferCopy& copy : check.copies) { const DAddr copy_addr = static_cast(copy.src_offset); - const u8* const expected = base + (copy.dst_offset - check.staging.offset); + const u64 copy_offset = copy.dst_offset - check.staging.offset; + const u8* const expected = base + copy_offset; + const u8* const landed = + check.landed_bytes.empty() ? nullptr : check.landed_bytes.data() + copy_offset; Common::RangeSet compared; compared.Add(copy_addr, copy.size); const auto take_out = [&](const Common::RangeSet& taken) { @@ -753,26 +813,42 @@ void BufferCache

::CheckReadDownload(const ReadCheck& check, guest.resize(length); device_memory.ReadBlockUnsafe(start, guest.data(), length); read_check.compared += length; - const u8* const wanted = expected + (start - copy_addr); - if (std::memcmp(guest.data(), wanted, length) == 0) { - return; - } + const u64 offset = start - copy_addr; + sources.assign(length, Source::Own); + const auto mark = [&](const Common::RangeSet& ranges, Source source) { + ranges.ForEachInRange(start, length, [&](DAddr lower, DAddr upper) { + std::fill(sources.begin() + static_cast(lower - start), + sources.begin() + static_cast(upper - start), source); + }); + }; + mark(check.kept, Source::Kept); + mark(check.landed, Source::Landed); + const auto result_at = [&](u64 i) { + const u8 wanted = expected[offset + i]; + switch (sources[i]) { + case Source::Own: + return guest[i] == wanted ? Result::Same : Result::Differs; + case Source::Kept: + return guest[i] == wanted ? Result::Same : Result::NotLanded; + case Source::Landed: + if (guest[i] != landed[offset + i]) { + return Result::Rewritten; + } + return guest[i] == wanted ? Result::Same : Result::Outdated; + } + return Result::Same; + }; for (u64 i = 0; i < length;) { - if (guest[i] == wanted[i]) { + const Result result = result_at(i); + if (result == Result::Same) { ++i; continue; } - u64 run_end = i; - while (run_end < length && guest[run_end] != wanted[run_end]) { + u64 run_end = i + 1; + while (run_end < length && result_at(run_end) == result) { ++run_end; } - if (read_check.logged++ < 16) { - LOG_ERROR(HW_GPU, - "Read download check: guest memory {:#x}+{:#x} differs from a " - "synchronous flush ({:#04x} for {:#04x} at its start)", - start + i, run_end - i, guest[i], wanted[i]); - } - read_check.mismatched += run_end - i; + report(result, start + i, run_end - i, guest[i], expected[offset + i]); i = run_end; } }); @@ -786,9 +862,27 @@ void BufferCache

::CheckReadDownload(const ReadCheck& check, LOG_INFO(HW_GPU, "Read download check (EDEN_NX2_READ_CHECK): {} downloads compared with a " "synchronous flush, {} KiB; {} bytes differ, {} GPU-modified bytes no download " - "brought (both must stay 0)", + "brought (both must stay 0); {} bytes got newer values after an earlier read's " + "download brought them (not compared)", read_check.checked, read_check.compared / 1024, read_check.mismatched, - read_check.uncovered); + read_check.uncovered, read_check.rewritten); + } +} + +template +void BufferCache

::NoteLandedBytes(ReadCheck& check, const Common::RangeSet& written) { + if (check.kept.Empty()) { + return; + } + for (const BufferCopy& copy : check.copies) { + const DAddr copy_addr = static_cast(copy.src_offset); + u8* const landed = check.landed_bytes.data() + (copy.dst_offset - check.staging.offset); + written.ForEachInRange(copy_addr, copy.size, [&](DAddr lower, DAddr upper) { + check.kept.ForEachInRange(lower, upper - lower, [&](DAddr start, DAddr end) { + device_memory.ReadBlockUnsafe(start, landed + (start - copy_addr), end - start); + check.landed.Add(start, end - start); + }); + }); } } diff --git a/src/video_core/buffer_cache/buffer_cache_base.h b/src/video_core/buffer_cache/buffer_cache_base.h index 9d4afe641e..f0146a93d3 100644 --- a/src/video_core/buffer_cache/buffer_cache_base.h +++ b/src/video_core/buffer_cache/buffer_cache_base.h @@ -793,14 +793,25 @@ class BufferCache : public VideoCommon::ChannelSetupCaches copies; /// Replaced in guest memory since (ClearDownload): not compared. Common::RangeSet changed; + /// Not in this download: an earlier one brings them, as the synchronous flush of its own + /// read did. Guest memory may get newer bytes there after it lands, which the + /// synchronous flushes would have kept too. + Common::RangeSet kept; + /// What guest memory held over `kept` right after those downloads landed (`landed`), at + /// the offsets the copies have in `staging`. + std::vector landed_bytes; + Common::RangeSet landed; }; struct ReadCheckTotals { + u64 every{}; ///< Tests: check 1 in `every` read downloads, over EDEN_NX2_READ_CHECK u64 next{}; ///< Read downloads seen, for the sampling u64 checked{}; ///< Read downloads compared u64 compared{}; ///< Bytes compared u64 mismatched{}; ///< Bytes that differ from the synchronous copy u64 uncovered{}; ///< Bytes a synchronous flush copies that no download brings + u64 rewritten{}; ///< Kept bytes guest memory got anew after their download landed u64 logged{}; ///< Error lines so far (at most 16) + u64 logged_rewrites{}; ///< Lines about rewritten bytes so far (at most 16) u64 last_log{}; }; ReadCheckTotals read_check{}; @@ -815,6 +826,8 @@ class BufferCache : public VideoCommon::ChannelSetupCaches& cpu_written); + /// A download landed `written`: what guest memory holds now over a later check's `kept`. + void NoteLandedBytes(ReadCheck& check, const Common::RangeSet& written); /// A DownloadMemoryForRead: its copies, and what is left of them to do once the host GPU has /// run them. diff --git a/src/video_core/memory_manager.cpp b/src/video_core/memory_manager.cpp index 6ba3b3b6f3..46af6fe4d6 100644 --- a/src/video_core/memory_manager.cpp +++ b/src/video_core/memory_manager.cpp @@ -333,6 +333,10 @@ void MemoryManager::Write(GPUVAddr addr, T data) { if (const auto address = GpuToCpuAddress(addr); address && rasterizer != nullptr && VideoCore::ReadDownloadsMayCover(*address, sizeof(T))) { + if (VideoCore::ReadCheckWrites::Enabled()) { + VideoCore::ReadCheckWrites::Note(*address, sizeof(T), + "MemoryManager::Write (a semaphore or report)"); + } rasterizer->FinishRead(*address, sizeof(T)); } } @@ -537,9 +541,16 @@ void MemoryManager::WriteBlockImpl(GPUVAddr gpu_dest_addr, const void* src_buffe src_buffer = static_cast(src_buffer) + copy_amount; }; // A download for a guest read not yet in guest memory would land over this write later. - const auto finish_reads = [this](DAddr dev_addr, std::size_t amount) { + const auto finish_reads = [this, unsafe](DAddr dev_addr, std::size_t amount) { if (VideoCore::g_read_downloads.load(std::memory_order_relaxed) != 0) [[unlikely]] { if (rasterizer != nullptr && VideoCore::ReadDownloadsMayCover(dev_addr, amount)) { + if (VideoCore::ReadCheckWrites::Enabled()) { + VideoCore::ReadCheckWrites::Note( + dev_addr, amount, + unsafe ? "MemoryManager::WriteBlockUnsafe/Cached (inline-to-memory or " + "constant buffer data)" + : "MemoryManager::WriteBlock"); + } rasterizer->FinishRead(dev_addr, amount); } } diff --git a/src/video_core/query_cache/query_cache.h b/src/video_core/query_cache/query_cache.h index e906e68e5e..a62f8c924c 100644 --- a/src/video_core/query_cache/query_cache.h +++ b/src/video_core/query_cache/query_cache.h @@ -665,6 +665,9 @@ void QueryCacheBase::FinishReads(DAddr address, size_t size) { // SemiFlushQueryDirty asks with cache_mutex held, which no holder of the buffer cache's lock // takes. if (VideoCore::ReadDownloadsMayCover(address, size)) [[unlikely]] { + if (VideoCore::ReadCheckWrites::Enabled()) { + VideoCore::ReadCheckWrites::Note(address, size, "a query result"); + } impl->rasterizer.FinishRead(address, size); } } diff --git a/src/video_core/rasterizer_download_area.h b/src/video_core/rasterizer_download_area.h index 4eb0a6f604..e817e32e16 100644 --- a/src/video_core/rasterizer_download_area.h +++ b/src/video_core/rasterizer_download_area.h @@ -3,7 +3,11 @@ #pragma once +#include +#include #include +#include +#include #include "common/common_types.h" #include "video_core/nx2.h" @@ -43,4 +47,43 @@ inline std::atomic g_read_downloads{}; Tegra::Nx2MayBeGpuWritten(addr, size)); } +/// TEST (EDEN_NX2_READ_CHECK): the latest writes from the GPU's side that ReadDownloadsMayCover +/// let through, to name the writer when the check finds newer bytes where an earlier read's +/// download landed. +class ReadCheckWrites { +public: + [[nodiscard]] static bool Enabled() { + static const bool enabled = std::getenv("EDEN_NX2_READ_CHECK") != nullptr; + return enabled; + } + + static void Note(DAddr addr, u64 size, const char* writer) { + std::scoped_lock lock{mutex}; + ring[next++ % Size] = {addr, size, writer}; + } + + /// The newest noted write over [addr, addr + size). + [[nodiscard]] static const char* Find(DAddr addr, u64 size) { + std::scoped_lock lock{mutex}; + for (u64 i = 0; i < (std::min)(next, Size); ++i) { + const Write& entry = ring[(next - 1 - i) % Size]; + if (entry.addr < addr + size && addr < entry.addr + entry.size) { + return entry.writer; + } + } + return "a writer the check doesn't hear of"; + } + +private: + struct Write { + DAddr addr; + u64 size; + const char* writer; + }; + static constexpr u64 Size = 1024; + inline static std::mutex mutex; + inline static std::array ring{}; + inline static u64 next{}; +}; + } // namespace VideoCore From 783a9a3409cef0a79f8d570ac08d5b261e8a050a Mon Sep 17 00:00:00 2001 From: jpkto <255363096+jpkto@users.noreply.github.com> Date: Sun, 8 Nov 2026 00:47:55 +0100 Subject: [PATCH 021/105] video_core: download read-back pages right behind their GPU writes Ocarina of Time reads back two compute results every frame, one page each. Each read waited 2.6-2.8 ms for its download. The read-trace (test47d) shows that the dispatch writing a page ended on the host GPU 6.8 ms before the read came. The download was recorded only at the read, behind about 770 draws, 95 render passes and 10 submissions recorded since the write. The host GPU had to run all of them before the copy. The buffer cache now learns which pages the guest reads after each GPU write. A page qualifies once two of its writes in a row were read. Four writes in a row without a read stop it, and 64 pages are kept, the one read longest ago making room. When such a page is written, its download is recorded right behind the writing work (RecordEarlyDownloads): - After a dispatch, at once: the scheduler is outside any render pass. Pages written by a draw wait for the next dispatch, a read's flush or the frame's end. - It doesn't submit on its own. The copies go with the GPU thread's next submission, as a fence's do (QueueForHostRead). A thread that must wait for one before it is submitted asks the split's back to submit it. The back submits it itself. An early download is an ordinary read download, recorded before any read asks for it. The read finds it the way it finds an earlier read's download. GetFlushArea now writes such downloads to guest memory itself once the host GPU has run them. The read then waits for nothing. Finished early downloads also land after each dispatch, every 32 draws and at the frame's end, so most reads find the bytes already in guest memory. Everything that keeps read downloads correct applies unchanged: - A newer GPU write before the read leaves its bytes to a download of their own (OutdateReadDownloads). For a learned page, that is the next early download. - CPU writes keep their values (ClearDownload, pending writes at landing). - Writes to guest memory from the GPU's side land the downloads under them first. - Downloads land oldest first, and before a later fence's downloads. Early downloads are bounded to 1 MiB a frame and 32 not yet in guest memory. Past either bound, reads flush on demand as before. EDEN_NX2_EARLY_READ_DOWNLOADS=0 turns them off for comparisons. Tests: a page read after each write gets its download behind the next write. A read before the host GPU ran it waits for that copy alone; after, it waits for nothing. A newer dispatch or draw write gets its own download. CPU writes keep their values. A page no longer read stops after four writes. A semaphore release over a pending early download lands it first. Finished downloads land within 32 draws or at the next dispatch. The 33rd download still on its way, and the frame's downloads past 1 MiB, are left to their reads' flushes. Co-Authored-By: Claude Opus 5.5 --- src/tests/video_core/buffer_cache_ports.cpp | 268 +++++++++++++++++- src/video_core/buffer_cache/buffer_cache.h | 218 ++++++++++++-- .../buffer_cache/buffer_cache_base.h | 66 +++++ .../renderer_vulkan/vk_buffer_cache.cpp | 34 ++- .../renderer_vulkan/vk_buffer_cache.h | 10 +- .../renderer_vulkan/vk_rasterizer.cpp | 3 + 6 files changed, 565 insertions(+), 34 deletions(-) diff --git a/src/tests/video_core/buffer_cache_ports.cpp b/src/tests/video_core/buffer_cache_ports.cpp index 4a8b46468c..e1be336bb0 100644 --- a/src/tests/video_core/buffer_cache_ports.cpp +++ b/src/tests/video_core/buffer_cache_ports.cpp @@ -132,6 +132,15 @@ struct Runtime { pending.clear(); return submitted_tick; } + u64 QueueForHostRead() { + ++queued_host_reads; + submitted.emplace_back(++submitted_tick, std::move(pending)); + pending.clear(); + return submitted_tick; + } + bool IsFreeNow(u64 tick) const { + return IsFree(tick); + } void WaitForHostRead(u64 tick) { if (before_completion) { before_completion(); @@ -170,7 +179,7 @@ struct Runtime { u32 alignment{256}; u32 allocations{}, downloads{}, uploads{}, finishes{}, waits{}; - u32 host_read_barriers{}, host_read_waits{}; + u32 host_read_barriers{}, host_read_waits{}, queued_host_reads{}; u64 upload_bytes{}, waited_tick{}; u64 submitted_tick{}, completed_tick{}; std::vector> stagings; @@ -1023,3 +1032,260 @@ TEST_CASE("Read download check finds an earlier read's bytes the GPU changed unr REQUIRE(Access::ReadChecks(*f.cache).mismatched == 0x10); REQUIRE(Access::ReadChecks(*f.cache).rewritten == 0); } + +namespace { +struct Nx2TitleScope { + Nx2TitleScope() { + Tegra::g_nx2_ampere = true; + } + ~Nx2TitleScope() { + Tegra::g_nx2_ampere = false; + Tegra::Nx2ClearGpuWrittenPages(); + } +}; + +/// A dispatch that writes `value` over [address, address + size) of the buffer at 0x20000. +void WriteByDispatch(Fixture& f, VideoCommon::BufferId id, DAddr address, u32 size, u8 value) { + auto& buffer = Access::Get(*f.cache, id); + f.runtime.pending.push_back([&buffer, address, size, value] { + std::fill_n(buffer.bytes.begin() + static_cast(address - 0x20000), size, + value); + }); + Access::MarkWritten(*f.cache, id, address, size); + f.cache->NoteWorkRecorded(true); +} + +/// A guest read of a word at `address`, as GPU::OnCPURead serves it. +VideoCore::RasterizerDownloadArea ReadWord(Fixture& f, DAddr address) { + const auto area = f.cache->GetFlushArea(address, 4); + REQUIRE(area); + const u64 size = area->end_address - area->start_address; + if (!area->preemtive && !area->in_flight) { + f.cache->DownloadMemoryForRead(area->start_address, size); + } + if (!area->preemtive) { + f.cache->FinishReadDownloads(area->start_address, size); + } + return *area; +} +} // namespace + +TEST_CASE("Pages read after each GPU write get their downloads right behind the writes", + "[buffer_ports]") { + Fixture f; + const Nx2TitleScope nx2_title; + const auto id = Access::Add(*f.cache, 0x20000, 0x10000); + // Learned from two writes the guest read; those reads still flush. + for (u8 frame = 1; frame <= 2; ++frame) { + WriteByDispatch(f, id, 0x20000, 0x100, frame); + REQUIRE(f.runtime.queued_host_reads == 0); + const auto area = ReadWord(f, 0x20010); + REQUIRE_FALSE(area.preemtive); + REQUIRE_FALSE(area.in_flight); + REQUIRE(f.bytes[0x10010] == frame); + } + REQUIRE(f.runtime.host_read_waits == 2); + // The next write's download goes right behind it, before any read. + WriteByDispatch(f, id, 0x20000, 0x100, 3); + REQUIRE(f.runtime.queued_host_reads == 1); + REQUIRE(Access::ReadDownloads(*f.cache) == 1); + REQUIRE(f.bytes[0x10010] == 2); + // A read before the host GPU ran it waits for that copy alone. + const auto waiting = f.cache->GetFlushArea(0x20010, 4); + REQUIRE(waiting->in_flight); + f.cache->FinishReadDownloads(waiting->start_address, + waiting->end_address - waiting->start_address); + REQUIRE(f.runtime.host_read_waits == 3); + REQUIRE(f.runtime.host_read_barriers == 2); + REQUIRE(f.bytes[0x10010] == 3); + REQUIRE(f.bytes[0x100ff] == 3); + // Once the host GPU ran it, the read finds the bytes in guest memory and waits for nothing. + WriteByDispatch(f, id, 0x20000, 0x100, 4); + REQUIRE(f.runtime.queued_host_reads == 2); + f.runtime.Complete(f.runtime.submitted_tick); + REQUIRE(ReadWord(f, 0x20010).preemtive); + REQUIRE(f.runtime.host_read_waits == 3); + REQUIRE(f.bytes[0x10010] == 4); + REQUIRE(Access::ReadDownloads(*f.cache) == 0); + REQUIRE(VideoCore::g_read_downloads == 0); + REQUIRE_FALSE(Access::Pending(*f.cache, 0x20000, 0x1000)); +} + +TEST_CASE("Early downloads leave what a newer GPU write changed to a download of its own", + "[buffer_ports]") { + Fixture f; + const Nx2TitleScope nx2_title; + const auto id = Access::Add(*f.cache, 0x20000, 0x10000); + for (u8 frame = 1; frame <= 2; ++frame) { + WriteByDispatch(f, id, 0x20000, 0x100, frame); + ReadWord(f, 0x20010); + } + // Another dispatch writes part of it again before the read: it gets an early download too. + WriteByDispatch(f, id, 0x20000, 0x100, 3); + WriteByDispatch(f, id, 0x20080, 0x40, 4); + REQUIRE(f.runtime.queued_host_reads == 2); + f.runtime.Complete(f.runtime.submitted_tick); + REQUIRE(ReadWord(f, 0x20010).preemtive); + REQUIRE(f.bytes[0x1007f] == 3); + REQUIRE(f.bytes[0x10080] == 4); + REQUIRE(f.bytes[0x100bf] == 4); + REQUIRE(f.bytes[0x100c0] == 3); + // A draw writes part of it again: the read downloads that part after the early download. + WriteByDispatch(f, id, 0x20000, 0x100, 5); + auto& buffer = Access::Get(*f.cache, id); + f.runtime.pending.push_back( + [&buffer] { std::fill_n(buffer.bytes.begin() + 0x40, 0x10, u8{6}); }); + Access::MarkWritten(*f.cache, id, 0x20040, 0x10); + const auto area = ReadWord(f, 0x20010); + REQUIRE_FALSE(area.preemtive); + REQUIRE_FALSE(area.in_flight); + REQUIRE(f.bytes[0x1003f] == 5); + REQUIRE(f.bytes[0x10040] == 6); + REQUIRE(f.bytes[0x1004f] == 6); + REQUIRE(f.bytes[0x10050] == 5); + REQUIRE(Access::ReadDownloads(*f.cache) == 0); + REQUIRE_FALSE(Access::Pending(*f.cache, 0x20000, 0x1000)); +} + +TEST_CASE("Early downloads keep CPU writes, and stop for pages no longer read", "[buffer_ports]") { + Fixture f; + const Nx2TitleScope nx2_title; + const auto id = Access::Add(*f.cache, 0x20000, 0x10000); + for (u8 frame = 1; frame <= 2; ++frame) { + WriteByDispatch(f, id, 0x20000, 0x100, frame); + ReadWord(f, 0x20010); + } + // The guest writes part of the page before the early download lands: its values stay. + WriteByDispatch(f, id, 0x20000, 0x100, 3); + std::memset(f.bytes + 0x10020, 0xee, 0x10); + const std::array, 1> applied{{{0x20020, 0x10}}}; + f.cache->ClaimCpuWrites(applied); + f.runtime.Complete(f.runtime.submitted_tick); + REQUIRE(ReadWord(f, 0x20010).preemtive); + REQUIRE(f.bytes[0x1001f] == 3); + REQUIRE(f.bytes[0x10020] == 0xee); + REQUIRE(f.bytes[0x1002f] == 0xee); + REQUIRE(f.bytes[0x10030] == 3); + // Not read any more: a few more writes get early downloads, then none. + const u32 queued = f.runtime.queued_host_reads; + for (u8 frame = 4; frame < 12; ++frame) { + WriteByDispatch(f, id, 0x20000, 0x100, frame); + f.runtime.Complete(f.runtime.submitted_tick); + } + REQUIRE(f.runtime.queued_host_reads - queued == 4); + const auto area = ReadWord(f, 0x20010); + REQUIRE_FALSE(area.preemtive); + REQUIRE(f.bytes[0x10010] == 11); + REQUIRE(Access::ReadDownloads(*f.cache) == 0); + REQUIRE_FALSE(Access::Pending(*f.cache, 0x20000, 0x1000)); +} + +TEST_CASE("GPU writes to guest memory land the early downloads under them first", + "[buffer_ports]") { + Fixture f; + const Nx2TitleScope nx2_title; + const auto id = Access::Add(*f.cache, 0x20000, 0x10000); + ReadRasterizer reads{*f.cache}; + Tegra::MemoryManager gpu_memory{f.system, f.device, 32, 1ULL << 31, 16}; + gpu_memory.BindRasterizer(&reads); + gpu_memory.Map(0x200000, 0x20000, 0x10000, Tegra::PTEKind::PITCH, true); + for (u8 frame = 1; frame <= 2; ++frame) { + WriteByDispatch(f, id, 0x20000, 0x100, frame); + ReadWord(f, 0x20010); + } + WriteByDispatch(f, id, 0x20000, 0x100, 3); + REQUIRE(VideoCore::g_read_downloads == 1); + // A semaphore release into the copied bytes before the host GPU ran the copy. + gpu_memory.Write(0x200040, 0xcafef00d); + REQUIRE(VideoCore::g_read_downloads == 0); + u32 value{}; + std::memcpy(&value, f.bytes + 0x10040, sizeof(value)); + REQUIRE(value == 0xcafef00d); + REQUIRE(f.bytes[0x1003f] == 3); + REQUIRE(f.bytes[0x10044] == 3); + REQUIRE(ReadWord(f, 0x20040).preemtive); +} + +TEST_CASE("Finished early downloads land a few draws later, or at the next dispatch", + "[buffer_ports]") { + Fixture f; + const Nx2TitleScope nx2_title; + const auto id = Access::Add(*f.cache, 0x20000, 0x10000); + for (u8 frame = 1; frame <= 2; ++frame) { + WriteByDispatch(f, id, 0x20000, 0x100, frame); + ReadWord(f, 0x20010); + } + WriteByDispatch(f, id, 0x20000, 0x100, 3); + f.runtime.Complete(f.runtime.submitted_tick); + for (u32 draw = 1; draw < 32; ++draw) { + f.cache->NoteWorkRecorded(false); + } + REQUIRE(Access::ReadDownloads(*f.cache) == 1); + f.cache->NoteWorkRecorded(false); + REQUIRE(Access::ReadDownloads(*f.cache) == 0); + REQUIRE(f.bytes[0x10010] == 3); + // A draw writes it: the early download goes behind the next dispatch. + auto& buffer = Access::Get(*f.cache, id); + f.runtime.pending.push_back([&buffer] { std::fill_n(buffer.bytes.begin(), 0x100, u8{4}); }); + Access::MarkWritten(*f.cache, id, 0x20000, 0x100); + f.cache->NoteWorkRecorded(false); + REQUIRE(f.runtime.queued_host_reads == 1); + f.cache->NoteWorkRecorded(true); + REQUIRE(f.runtime.queued_host_reads == 2); + f.runtime.Complete(f.runtime.submitted_tick); + f.cache->NoteWorkRecorded(true); + REQUIRE(Access::ReadDownloads(*f.cache) == 0); + REQUIRE(f.bytes[0x10010] == 4); + REQUIRE(ReadWord(f, 0x20010).preemtive); +} + +TEST_CASE("Early downloads stop at 32 on their way", "[buffer_ports]") { + Fixture f; + const Nx2TitleScope nx2_title; + const auto id = Access::Add(*f.cache, 0x20000, 0x30000); + // 33 pages, each read after two writes. + for (u8 frame = 1; frame <= 2; ++frame) { + for (DAddr page = 0x20000; page < 0x41000; page += 0x1000) { + WriteByDispatch(f, id, page, 0x100, frame); + ReadWord(f, page); + } + } + // Their next writes before the host GPU ran any copy: the 33rd page's read flushes. + for (DAddr page = 0x20000; page < 0x41000; page += 0x1000) { + WriteByDispatch(f, id, page, 0x100, 3); + } + REQUIRE(f.runtime.queued_host_reads == 32); + REQUIRE(Access::ReadDownloads(*f.cache) == 32); + const auto last = ReadWord(f, 0x40010); + REQUIRE_FALSE(last.preemtive); + REQUIRE_FALSE(last.in_flight); + REQUIRE(f.bytes[0x30010] == 3); + REQUIRE(f.bytes[0x10010] == 3); + REQUIRE(Access::ReadDownloads(*f.cache) == 0); + REQUIRE_FALSE(Access::Pending(*f.cache, 0x20000, 0x21000)); +} + +TEST_CASE("Early downloads stop at 1 MiB a frame", "[buffer_ports]") { + Fixture f; + const Nx2TitleScope nx2_title; + const auto id = Access::Add(*f.cache, 0x20000, 0x10000); + for (u8 frame = 1; frame <= 2; ++frame) { + WriteByDispatch(f, id, 0x20000, 0x1000, frame); + ReadWord(f, 0x20010); + } + // A whole page each, read once the host GPU ran it: 256 make 1 MiB. + for (u32 write = 0; write < 256; ++write) { + WriteByDispatch(f, id, 0x20000, 0x1000, static_cast(write)); + f.runtime.Complete(f.runtime.submitted_tick); + REQUIRE(ReadWord(f, 0x20010).preemtive); + } + REQUIRE(f.runtime.queued_host_reads == 256); + // The frame's later writes leave their reads to flush. + WriteByDispatch(f, id, 0x20000, 0x1000, 0xab); + REQUIRE(f.runtime.queued_host_reads == 256); + const auto area = ReadWord(f, 0x20010); + REQUIRE_FALSE(area.preemtive); + REQUIRE_FALSE(area.in_flight); + REQUIRE(f.bytes[0x10010] == 0xab); + REQUIRE(f.bytes[0x10fff] == 0xab); +} diff --git a/src/video_core/buffer_cache/buffer_cache.h b/src/video_core/buffer_cache/buffer_cache.h index cff7ea03e1..f4d80a5d09 100644 --- a/src/video_core/buffer_cache/buffer_cache.h +++ b/src/video_core/buffer_cache/buffer_cache.h @@ -170,7 +170,15 @@ void BufferCache

::TickFrame() { ++frame_tick; delayed_destruction_ring.Tick(); - // Read downloads no thread waited for (the other ranges the guest read before). + // Pages draws wrote: their early downloads, now that the frame's work is recorded. + RecordEarlyDownloads(); + early_bytes_this_frame = 0; + // Read downloads no thread waited for (the other ranges the guest read before, early ones). + if constexpr (!IS_OPENGL) { + if (!read_downloads.empty()) { + (void)runtime.IsFreeNow(read_downloads.front().tick); + } + } WriteCompletedReadDownloads(); for (auto& buffer : async_buffers_death_ring) { runtime.FreeDeferredStagingBuffer(buffer); @@ -293,15 +301,25 @@ std::optional BufferCache

::GetFlushArea(DA }; area->preemtive = !IsRegionGpuModified(device_addr_start_aligned, device_addr_end_aligned - device_addr_start_aligned); + NoteReadBack(device_addr_start_aligned, device_addr_end_aligned, !area->preemtive); if (!area->preemtive && !read_downloads.empty() && ReadDownloadsCover(device_addr_start_aligned, device_addr_end_aligned - device_addr_start_aligned)) { - // All of it on its way for an earlier read: wait for those copies, not new ones. The core - // took the trap, as below. - area->in_flight = true; + // All of it on its way, for an earlier read or right behind its GPU write: no new copies. + // The core took the trap, as below. read_trapped_ranges.Subtract(device_addr_start_aligned, device_addr_end_aligned - device_addr_start_aligned); - return area; + if (!WriteFinishedReadDownloads(device_addr_start_aligned, + device_addr_end_aligned - device_addr_start_aligned)) { + area->in_flight = true; + return area; + } + // The host GPU had run them: in guest memory now, nothing is left to wait for. + area->preemtive = !IsRegionGpuModified(device_addr_start_aligned, + device_addr_end_aligned - device_addr_start_aligned); + if (area->preemtive) { + return area; + } } // A preflushable mark persists and downloads later GPU writes at every fence. Native titles // instead keep read traps and flush on demand. @@ -458,6 +476,8 @@ void BufferCache

::DownloadMemoryForRead(DAddr device_addr, u64 size) { DownloadMemory(device_addr, size); } else { WriteCompletedReadDownloads(); + // Early downloads still to record are of older writes: they go first. + RecordEarlyDownloads(); // The ranges DownloadMemory takes: the guest CPU soon reads those it read before too. boost::container::small_vector, 16> ranges{{device_addr, size}}; if (download_cpu_read_ranges) { @@ -468,12 +488,22 @@ void BufferCache

::DownloadMemoryForRead(DAddr device_addr, u64 size) { [&](DAddr start, DAddr end) { ranges.emplace_back(start, end - start); }); }); } + const std::array own{std::pair{device_addr, size}}; + RecordReadDownload(ranges, own, false); + } +} + +template +void BufferCache

::RecordReadDownload(std::span> ranges, + std::span> own, bool early) { + if constexpr (!IS_OPENGL && USE_MEMORY_MAPS) { struct Copy { BufferId buffer_id; BufferCopy copy; }; boost::container::small_vector copies; ReadDownload download; + download.early = early; u64 total_size_bytes = 0; const auto add_copy = [&](BufferId buffer_id, DAddr buffer_addr, DAddr begin, DAddr end) { copies.push_back({buffer_id, BufferCopy{ @@ -494,27 +524,30 @@ void BufferCache

::DownloadMemoryForRead(DAddr device_addr, u64 size) { const u64 check_every = read_check.every != 0 ? read_check.every : ReadCheckEvery(); if (check_every != 0 && read_check.next++ % check_every == 0) [[unlikely]] { check = std::make_unique(); - ForEachBufferInRange(device_addr, size, [&](BufferId buffer_id, Buffer& buffer) { - const DAddr buffer_addr = buffer.CpuAddr(); - const DAddr begin = (std::max)(device_addr, buffer_addr); - const DAddr end = (std::min)(device_addr + size, buffer_addr + buffer.SizeBytes()); - if (begin >= end) { - return; - } - memory_tracker.ForEachDownloadRange( - begin, end - begin, false, [&](u64 run_addr, u64 run_size) { - gpu_modified_ranges.ForEachInRange( - run_addr, run_size, [&](DAddr start, DAddr range_end) { - check_copies.push_back( - {buffer_id, BufferCopy{ - .src_offset = start - buffer_addr, - .dst_offset = check_size, - .size = range_end - start, - }}); - check_size += Common::AlignUp(range_end - start, 64); - }); - }); - }); + for (const auto& [own_addr, own_size] : own) { + ForEachBufferInRange(own_addr, own_size, [&](BufferId buffer_id, Buffer& buffer) { + const DAddr buffer_addr = buffer.CpuAddr(); + const DAddr begin = (std::max)(own_addr, buffer_addr); + const DAddr end = + (std::min)(own_addr + own_size, buffer_addr + buffer.SizeBytes()); + if (begin >= end) { + return; + } + memory_tracker.ForEachDownloadRange( + begin, end - begin, false, [&](u64 run_addr, u64 run_size) { + gpu_modified_ranges.ForEachInRange( + run_addr, run_size, [&](DAddr start, DAddr range_end) { + check_copies.push_back( + {buffer_id, BufferCopy{ + .src_offset = start - buffer_addr, + .dst_offset = check_size, + .size = range_end - start, + }}); + check_size += Common::AlignUp(range_end - start, 64); + }); + }); + }); + } } boost::container::small_vector, 8> gaps; for (const auto& [range_addr, range_size] : ranges) { @@ -578,9 +611,10 @@ void BufferCache

::DownloadMemoryForRead(DAddr device_addr, u64 size) { rest.ForEach([&](DAddr lower, DAddr upper) { if (read_check.logged++ < 16) { LOG_ERROR(HW_GPU, - "Read download check: {:#x}+{:#x} of the read [{:#x}, {:#x}) " - "is GPU-modified but no download brings it", - lower, upper - lower, device_addr, device_addr + size); + "Read download check: {:#x}+{:#x} of the {} [{:#x}, {:#x}) is " + "GPU-modified but no download brings it", + lower, upper - lower, early ? "early download" : "read", + own.front().first, own.back().first + own.back().second); } read_check.uncovered += upper - lower; }); @@ -625,8 +659,13 @@ void BufferCache

::DownloadMemoryForRead(DAddr device_addr, u64 size) { } download.check = std::move(check); } - download.tick = runtime.SubmitForHostRead(); + // A read waits for its copies; an early download's go with the next submission. + download.tick = early ? runtime.QueueForHostRead() : runtime.SubmitForHostRead(); download.staging = staging; + if (early) { + ++pending_early_downloads; + early_bytes_this_frame += total_size_bytes; + } read_downloads.push_back(std::move(download)); VideoCore::g_read_downloads.fetch_add(1, std::memory_order_release); } @@ -666,6 +705,120 @@ void BufferCache

::OutdateReadDownloads(DAddr device_addr, u64 size) { } } +template +bool BufferCache

::WriteFinishedReadDownloads(DAddr device_addr, u64 size) { + if constexpr (!IS_OPENGL) { + std::optional tick; + for (const ReadDownload& download : read_downloads) { + if (download.write_ranges.Intersects(device_addr, size)) { + tick = download.tick; + } + } + if (!tick || !runtime.IsFreeNow(*tick)) { + return false; + } + WriteReadDownloads(*tick); + return true; + } else { + return false; + } +} + +template +void BufferCache

::NoteWorkRecorded(bool dispatch) { + if constexpr (!IS_OPENGL && USE_MEMORY_MAPS) { + if (dispatch) { + // Outside any render pass: the copies go right behind the dispatch. + RecordEarlyDownloads(); + } + // Early downloads the host GPU ran go to guest memory, where their reads then find them. + // Asking the host GPU is a call: after each dispatch, and every few draws. + if (pending_early_downloads != 0 && (dispatch || ++early_landing_checks % 32 == 0) && + runtime.IsFreeNow(read_downloads.front().tick)) { + WriteCompletedReadDownloads(); + } + } +} + +template +void BufferCache

::NoteReadBack(DAddr begin, DAddr end, bool gpu_modified) { + if constexpr (!IS_OPENGL && USE_MEMORY_MAPS) { + if (!EarlyDownloadsEnabled() || (!gpu_modified && read_back_pages.empty())) { + return; + } + for (DAddr page = begin; page < end; page += DEVICE_PAGESIZE) { + auto it = std::ranges::lower_bound(read_back_pages, page, {}, &ReadBackPage::page); + if (it == read_back_pages.end() || it->page != page) { + // Learned from reads that find GPU writes. + if (!gpu_modified) { + continue; + } + if (read_back_pages.size() >= MaxReadBackPages) { + // The page whose writes were read longest ago makes room. + read_back_pages.erase( + std::ranges::min_element(read_back_pages, {}, &ReadBackPage::last_read)); + it = std::ranges::lower_bound(read_back_pages, page, {}, &ReadBackPage::page); + } + it = read_back_pages.insert(it, ReadBackPage{.page = page, .written = true}); + } + if (!it->written) { + // Nothing new since its last read. + continue; + } + it->written = false; + it->unread_writes = 0; + it->reads = (std::min)(static_cast(it->reads + 1), EarlyAfterReads); + it->last_read = ++read_back_reads; + } + } +} + +template +void BufferCache

::NoteReadBackWrite(DAddr device_addr, u64 size) { + if constexpr (!IS_OPENGL && USE_MEMORY_MAPS) { + if (read_back_pages.empty()) { + return; + } + const DAddr end = device_addr + size; + for (auto it = std::ranges::lower_bound(read_back_pages, + Common::AlignDown(device_addr, DEVICE_PAGESIZE), {}, + &ReadBackPage::page); + it != read_back_pages.end() && it->page < end; ++it) { + it->written = true; + if (it->unread_writes < MaxUnreadWrites) { + ++it->unread_writes; + } else { + // The guest no longer reads it after its writes. + it->reads = 0; + } + if (it->reads >= EarlyAfterReads && + std::ranges::find(early_pages, it->page) == early_pages.end()) { + early_pages.push_back(it->page); + } + } + } +} + +template +void BufferCache

::RecordEarlyDownloads() { + if constexpr (!IS_OPENGL && USE_MEMORY_MAPS) { + if (early_pages.empty()) { + return; + } + boost::container::small_vector, 8> pages; + for (const DAddr page : early_pages) { + pages.emplace_back(page, DEVICE_PAGESIZE); + } + early_pages.clear(); + // Bounded: past it, the reads of these pages flush on demand. + if (pending_early_downloads >= MaxPendingEarlyDownloads || + early_bytes_this_frame >= EarlyBytesPerFrame) { + return; + } + RecordReadDownload(pages, pages, true); + } +} + template void BufferCache

::WriteReadDownloads(u64 tick) { if constexpr (!IS_OPENGL) { @@ -712,6 +865,9 @@ void BufferCache

::WriteReadDownloads(u64 tick) { }); // Freed by the GPU thread (TickFrame). async_buffers_death_ring.push_back(download.staging); + if (download.early) { + --pending_early_downloads; + } read_downloads.pop_front(); VideoCore::g_read_downloads.fetch_sub(1, std::memory_order_release); } @@ -960,6 +1116,7 @@ bool BufferCache

::DMACopy(GPUVAddr src_address, GPUVAddr dest_address, u64 am for (const auto& pair : tmp_intervals) { gpu_modified_ranges.Add(pair.first, pair.second); GpuWaitStats::NoteGpuWrite(pair.first, pair.second, GpuWaitStats::WriteSite::DmaCopy); + NoteReadBackWrite(pair.first, pair.second); if (Tegra::g_nx2_ampere.load(std::memory_order_relaxed)) { Tegra::Nx2MarkGpuWritten(pair.first, pair.second); Nx2NoteListWrite(buffer_b, pair.first, pair.second); @@ -2548,6 +2705,7 @@ void BufferCache

::MarkWrittenBuffer(BufferId buffer_id, DAddr device_addr, u3 ranges.Subtract(device_addr, size); } OutdateReadDownloads(device_addr, size); + NoteReadBackWrite(device_addr, size); memory_tracker.MarkRegionAsGpuModified(device_addr, size); MarkVertexBuffersStale(device_addr, size); gpu_modified_ranges.Add(device_addr, size); diff --git a/src/video_core/buffer_cache/buffer_cache_base.h b/src/video_core/buffer_cache/buffer_cache_base.h index f0146a93d3..7f3853a1bb 100644 --- a/src/video_core/buffer_cache/buffer_cache_base.h +++ b/src/video_core/buffer_cache/buffer_cache_base.h @@ -287,6 +287,11 @@ class BufferCache : public VideoCommon::ChannelSetupCaches GetFlushArea(DAddr device_addr, u64 size); /// Without downloads at every fence, a guest CPU read is the only way GPU-written buffer @@ -633,6 +638,28 @@ class BufferCache : public VideoCommon::ChannelSetupCaches> ranges, + std::span> own, bool early); + + /// A guest read of the pages [begin, end): learns which pages the guest reads after the GPU + /// writes them. `gpu_modified`: the read found GPU writes not yet in guest memory. + void NoteReadBack(DAddr begin, DAddr end, bool gpu_modified); + + /// A GPU write: the pages under it that the guest reads after each write get an early + /// download once the writing work is recorded. + void NoteReadBackWrite(DAddr device_addr, u64 size); + + /// Records the early downloads of the pages written since their last one (`mutex` held). + void RecordEarlyDownloads(); + void DeleteBuffer(BufferId buffer_id, bool do_not_mark = false); [[nodiscard]] Binding StorageBufferBinding(GPUVAddr ssbo_addr, u32 cbuf_index, bool is_written, @@ -844,6 +871,8 @@ class BufferCache : public VideoCommon::ChannelSetupCaches clean_ranges; std::unique_ptr check; ///< TEST: EDEN_NX2_READ_CHECK + /// Recorded right after a GPU write, before any read (RecordEarlyDownloads) + bool early{}; }; /// Oldest first, and written to guest memory in that order: none lands over newer bytes. std::deque read_downloads; @@ -851,6 +880,43 @@ class BufferCache : public VideoCommon::ChannelSetupCaches read_download_ranges; + /// A page the guest CPU read GPU writes from. Switch 2 titles read compute results back every + /// frame, milliseconds after the dispatch that wrote them: a download recorded at the read + /// waits for all the GPU work recorded in between. Pages read after each of their writes get + /// their download recorded right behind the write instead (RecordEarlyDownloads), which the + /// read finds landed or on its way (GetFlushArea: in_flight). + struct ReadBackPage { + DAddr page{}; + u64 last_read{}; ///< read_back_reads at its last read of a write (eviction) + u16 reads{}; ///< Writes in a row the guest read, up to EarlyAfterReads + u16 unread_writes{}; ///< GPU writes since its last read + bool written{}; ///< A GPU write since its last read + }; + /// Reads of a page's writes in a row before its writes get early downloads. + static constexpr u16 EarlyAfterReads = 2; + /// Writes in a row without a read that stop them. + static constexpr u16 MaxUnreadWrites = 4; + static constexpr size_t MaxReadBackPages = 64; + /// Early download bytes a frame, and early downloads not yet in guest memory: past them, reads + /// flush on demand. + static constexpr u64 EarlyBytesPerFrame = 1_MiB; + static constexpr u32 MaxPendingEarlyDownloads = 32; + /// EDEN_NX2_EARLY_READ_DOWNLOADS=0: none (for comparisons). + static bool EarlyDownloadsEnabled() { + static const bool enabled = [] { + const char* value = std::getenv("EDEN_NX2_EARLY_READ_DOWNLOADS"); + return value == nullptr || value[0] != '0'; + }(); + return enabled; + } + std::vector read_back_pages; ///< Sorted by page + /// Pages to download once the work writing them is recorded. + boost::container::small_vector early_pages; + u64 read_back_reads{}; + u64 early_bytes_this_frame{}; + u32 pending_early_downloads{}; + u32 early_landing_checks{}; + std::deque async_buffers_death_ring; size_t immediate_buffer_capacity = 0; diff --git a/src/video_core/renderer_vulkan/vk_buffer_cache.cpp b/src/video_core/renderer_vulkan/vk_buffer_cache.cpp index 7bc0e9760a..a87fbe02df 100644 --- a/src/video_core/renderer_vulkan/vk_buffer_cache.cpp +++ b/src/video_core/renderer_vulkan/vk_buffer_cache.cpp @@ -520,9 +520,39 @@ u64 BufferCacheRuntime::SubmitForHostRead() { return scheduler.Flush(); } +u64 BufferCacheRuntime::QueueForHostRead() { + scheduler.RequestOutsideRenderPassOperationContext(); + scheduler.Record(HostReadBarrier); + const u64 tick = scheduler.CurrentTick(); + if (!scheduler.DeferFlush()) { + scheduler.Flush(); + } + return tick; +} + void BufferCacheRuntime::WaitForHostRead(u64 tick) { - // Flushed at once by SubmitForHostRead: the worker submits it without the GPU thread. - scheduler.GetMasterSemaphore().Wait(tick); + if (scheduler.IsFree(tick)) { + return; + } + // A queued tick may not be submitted yet. Only the split's back submits: other threads ask it + // to, as for fences (InnerFence::Wait). On the back, and once it has stopped, Wait submits it. + if (!scheduler.IsSubmitted(tick) && VideoCommon::GpuSplit::Get().Active() && + !VideoCommon::GpuSplit::OnBackThread() && scheduler.WaitSubmitted(tick)) { + scheduler.GetMasterSemaphore().Wait(tick); + return; + } + scheduler.Wait(tick); +} + +bool BufferCacheRuntime::IsFreeNow(u64 tick) { + if (scheduler.IsFree(tick)) { + return true; + } + if (!scheduler.IsSubmitted(tick)) { + return false; + } + scheduler.GetMasterSemaphore().Refresh(); + return scheduler.IsFree(tick); } bool BufferCacheRuntime::CanReorderUpload(const Buffer& buffer, diff --git a/src/video_core/renderer_vulkan/vk_buffer_cache.h b/src/video_core/renderer_vulkan/vk_buffer_cache.h index 4703e99c11..230e60db14 100644 --- a/src/video_core/renderer_vulkan/vk_buffer_cache.h +++ b/src/video_core/renderer_vulkan/vk_buffer_cache.h @@ -137,9 +137,17 @@ class BufferCacheRuntime { /// FinishForHostRead without the wait: submits, and returns the tick to wait for. u64 SubmitForHostRead(); - /// Waits for a SubmitForHostRead tick, on any thread. + /// SubmitForHostRead without the submission: the copies go with the GPU thread's next one, as + /// a fence's do (Scheduler::DeferFlush). Returns the tick they go in. + u64 QueueForHostRead(); + + /// Waits for a SubmitForHostRead or QueueForHostRead tick, on any thread. void WaitForHostRead(u64 tick); + /// Whether the host GPU completed `tick`, asking it when the tick is submitted but not known + /// to be done. + [[nodiscard]] bool IsFreeNow(u64 tick); + u64 GetDeviceLocalMemory() const; u64 GetDeviceMemoryUsage() const; diff --git a/src/video_core/renderer_vulkan/vk_rasterizer.cpp b/src/video_core/renderer_vulkan/vk_rasterizer.cpp index add7c48530..16681a744c 100644 --- a/src/video_core/renderer_vulkan/vk_rasterizer.cpp +++ b/src/video_core/renderer_vulkan/vk_rasterizer.cpp @@ -827,6 +827,7 @@ void RasterizerVulkan::PrepareDraw(bool is_indexed, Func&& draw_func, bool is_in draw_func(); } submitted = true; + buffer_cache.NoteWorkRecorded(false); #ifdef EGLANTINE_STREAMLINE constexpr u64 final_vertex = 0x16aff33d999f993fULL; constexpr u64 final_fragment = 0x52402d8e8ab55263ULL; @@ -1848,6 +1849,7 @@ void RasterizerVulkan::DispatchCompute() { cmdbuf.DispatchIndirect(indirect_buffer, indirect_offset); }); }); + buffer_cache.NoteWorkRecorded(true); SubmitNx2ListCopies(); return; } @@ -1901,6 +1903,7 @@ void RasterizerVulkan::DispatchCompute() { // The deferred native binding may follow DLSS's command buffer bindings. scheduler.InvalidateState(); } + buffer_cache.NoteWorkRecorded(true); SubmitNx2ListCopies(); #ifdef EGLANTINE_STREAMLINE From 00a62b9189ef84ef8c37426bdb1e75ba5e64bea8 Mon Sep 17 00:00:00 2001 From: jpkto <255363096+jpkto@users.noreply.github.com> Date: Sun, 8 Nov 2026 01:17:23 +0100 Subject: [PATCH 022/105] TEST: read trace and read check for early read downloads The read trace (EDEN_NX2_READ_TRACE), carried onto early read downloads, and the read download check (EDEN_NX2_READ_CHECK) extended to the reads they serve. The read trace is new in this series. It places a guest read of GPU-written memory on the host GPU's timeline: Vulkan writes device timestamps at the start and end of every submission, after each draw or dispatch that writes pages the guest read back before, and around the read's copies; the reading thread reads the device's clock while the GPU thread runs the flush (calibrated timestamps, an extension the variable enables). Once the bytes are in guest memory, the wait splits into: until the write's work ended on the GPU, until the copy could start, the copy, the wake-up, the buffer cache lock and the copy into guest memory. Every 10 s the log has a "Read trace" line for all reads and one for each of the four areas that waited longest. With the variable unset, nothing changes but a render pass count. Read trace: the copy marks now go around every read download's copies (RecordReadDownload), early ones included. The work marks come from NoteWorkRecorded. For a read that waits for an early download, "the write's end to the copy's start" is the GPU time between the dispatch and its copy. Each line also counts the reads that "waited for an early download". Every 10 s a new line follows the "all" line, reported even in frames where no read waits (TickFrame): Read trace early downloads: N recorded (K KiB) | reads they served: a found the bytes in guest memory, b found them done on the host GPU (landed at the read), c waited for them (in the lines above) | writes from the GPU's side that waited for one: d (t ms) The reads counted under a and b don't wait at all, so the per-read lines above leave them out. Read check: before this commit the check never saw these reads. A landed early download leaves the page clean, and one still on its way covers the read, which then records nothing. Now an early download that lands records which bytes it wrote to guest memory and what they were. In check mode, a read that an early download serves flushes. Its check copies the read's bytes from the GPU at the read, as a synchronous flush would. The bytes still on their way are compared as kept bytes are. The landed ones are compared the same way, against their record: - If guest memory changed since the landing, the bytes are counted as rewritten and not compared. - Otherwise they must equal the GPU's copy. A difference means the GPU changed them after the early download without anything downloading them again. That is an error, logged like other differences. Newer downloaded bytes or guest values drop the record (WriteDownloadedMemory, ClearDownload), and each read takes what it compared. A check whose read gets all its bytes from other downloads becomes a download of its own. The read waits for it, as does a write from the GPU's side over those bytes. With neither variable set, nothing changes. Tests: early-downloaded bytes, landed or on their way, compare equal (0 differ, 0 uncovered). A GPU change nobody recorded after the landing is found. A guest write there is left out as rewritten. Rebased onto upstream: EDEN_NX2_READ_TRACE is a plain flag read again by VideoCore::CreateGPU, as upstream reads its diagnostic switches since #714, so it works from nx2_switches.txt too, not only from the environment. The test build also had the pipeline library work (#579) here in vulkan_device.cpp; this commit adds only the calibrated timestamps extension. Co-Authored-By: Claude Opus 5.5 --- src/tests/video_core/buffer_cache_ports.cpp | 64 +++ src/video_core/CMakeLists.txt | 1 + src/video_core/buffer_cache/buffer_cache.h | 372 +++++++++++- .../buffer_cache/buffer_cache_base.h | 51 ++ src/video_core/gpu.cpp | 25 + src/video_core/nx2_read_trace.h | 535 ++++++++++++++++++ src/video_core/rasterizer_interface.h | 10 + .../renderer_vulkan/vk_buffer_cache.cpp | 12 + .../renderer_vulkan/vk_buffer_cache.h | 5 + .../renderer_vulkan/vk_rasterizer.cpp | 5 + .../renderer_vulkan/vk_rasterizer.h | 1 + .../renderer_vulkan/vk_scheduler.cpp | 260 ++++++++- src/video_core/renderer_vulkan/vk_scheduler.h | 23 + src/video_core/video_core.cpp | 2 + .../vulkan_common/vulkan_device.cpp | 10 + 15 files changed, 1356 insertions(+), 20 deletions(-) create mode 100644 src/video_core/nx2_read_trace.h diff --git a/src/tests/video_core/buffer_cache_ports.cpp b/src/tests/video_core/buffer_cache_ports.cpp index e1be336bb0..337a407f9d 100644 --- a/src/tests/video_core/buffer_cache_ports.cpp +++ b/src/tests/video_core/buffer_cache_ports.cpp @@ -1289,3 +1289,67 @@ TEST_CASE("Early downloads stop at 1 MiB a frame", "[buffer_ports]") { REQUIRE(f.bytes[0x10010] == 0xab); REQUIRE(f.bytes[0x10fff] == 0xab); } + +TEST_CASE("Read download check compares what early downloads brought with a synchronous flush", + "[buffer_ports]") { + Fixture f; + const Nx2TitleScope nx2_title; + const auto id = Access::Add(*f.cache, 0x20000, 0x10000); + Access::CheckEveryRead(*f.cache); + for (u8 frame = 1; frame <= 2; ++frame) { + WriteByDispatch(f, id, 0x20000, 0x100, frame); + ReadWord(f, 0x20010); + } + const auto& checks = Access::ReadChecks(*f.cache); + const u64 compared = checks.compared; + // In guest memory before the read: the read flushes, and compares them as they landed. + WriteByDispatch(f, id, 0x20000, 0x100, 3); + f.runtime.Complete(f.runtime.submitted_tick); + f.cache->NoteWorkRecorded(true); + REQUIRE(Access::ReadDownloads(*f.cache) == 0); + REQUIRE(f.bytes[0x10010] == 3); + REQUIRE_FALSE(ReadWord(f, 0x20010).preemtive); + // The early download's own check and the read's. + REQUIRE(checks.compared - compared == 0x200); + // On its way at the read: compared as it lands. + WriteByDispatch(f, id, 0x20000, 0x100, 4); + REQUIRE_FALSE(ReadWord(f, 0x20010).preemtive); + REQUIRE(f.bytes[0x10010] == 4); + REQUIRE(checks.compared - compared == 0x400); + REQUIRE(checks.mismatched == 0); + REQUIRE(checks.uncovered == 0); + REQUIRE(checks.rewritten == 0); + REQUIRE(Access::ReadDownloads(*f.cache) == 0); + REQUIRE_FALSE(Access::Pending(*f.cache, 0x20000, 0x1000)); +} + +TEST_CASE("Read download check finds early downloaded bytes the GPU changed unrecorded", + "[buffer_ports]") { + Fixture f; + const Nx2TitleScope nx2_title; + const auto id = Access::Add(*f.cache, 0x20000, 0x10000); + auto& buffer = Access::Get(*f.cache, id); + Access::CheckEveryRead(*f.cache); + for (u8 frame = 1; frame <= 2; ++frame) { + WriteByDispatch(f, id, 0x20000, 0x100, frame); + ReadWord(f, 0x20010); + } + WriteByDispatch(f, id, 0x20000, 0x100, 3); + f.runtime.Complete(f.runtime.submitted_tick); + f.cache->NoteWorkRecorded(true); + // A GPU write into the landed bytes that nothing records: a synchronous flush at the read + // would bring it. + f.runtime.pending.push_back( + [&buffer] { std::fill_n(buffer.bytes.begin() + 0x40, 0x10, u8{0x56}); }); + // The guest writes others: newer values, not compared. + std::memset(f.bytes + 0x10080, 0xee, 0x8); + ReadWord(f, 0x20010); + REQUIRE(f.bytes[0x10040] == 3); + REQUIRE(Access::ReadChecks(*f.cache).mismatched == 0x10); + REQUIRE(Access::ReadChecks(*f.cache).rewritten == 0x8); + REQUIRE(Access::ReadChecks(*f.cache).uncovered == 0); + // Compared once: a later read finds nothing an early download brought since. + const u64 compared = Access::ReadChecks(*f.cache).compared; + REQUIRE(ReadWord(f, 0x20010).preemtive); + REQUIRE(Access::ReadChecks(*f.cache).compared == compared); +} diff --git a/src/video_core/CMakeLists.txt b/src/video_core/CMakeLists.txt index f85ad42cbe..3a1a413cd8 100644 --- a/src/video_core/CMakeLists.txt +++ b/src/video_core/CMakeLists.txt @@ -122,6 +122,7 @@ add_library(video_core STATIC nx2_profile.h nx2_perf.h nx2_download_trace.h + nx2_read_trace.h pending_maps.h present.h pte_kind.h diff --git a/src/video_core/buffer_cache/buffer_cache.h b/src/video_core/buffer_cache/buffer_cache.h index f4d80a5d09..21e6092db7 100644 --- a/src/video_core/buffer_cache/buffer_cache.h +++ b/src/video_core/buffer_cache/buffer_cache.h @@ -184,6 +184,9 @@ void BufferCache

::TickFrame() { runtime.FreeDeferredStagingBuffer(buffer); } async_buffers_death_ring.clear(); + if constexpr (HAS_READ_TRACE) { + ReadTrace::Tick(); + } } template @@ -302,18 +305,38 @@ std::optional BufferCache

::GetFlushArea(DA area->preemtive = !IsRegionGpuModified(device_addr_start_aligned, device_addr_end_aligned - device_addr_start_aligned); NoteReadBack(device_addr_start_aligned, device_addr_end_aligned, !area->preemtive); - if (!area->preemtive && !read_downloads.empty() && + // TEST (EDEN_NX2_READ_CHECK): a read an early download serves takes a flush, which compares + // what it finds with the GPU's bytes now. + const bool check_early = + ReadChecks() && EarlyDownloadServes(device_addr_start_aligned, + device_addr_end_aligned - device_addr_start_aligned); + if (!area->preemtive && !check_early && !read_downloads.empty() && ReadDownloadsCover(device_addr_start_aligned, device_addr_end_aligned - device_addr_start_aligned)) { // All of it on its way, for an earlier read or right behind its GPU write: no new copies. // The core took the trap, as below. read_trapped_ranges.Subtract(device_addr_start_aligned, device_addr_end_aligned - device_addr_start_aligned); + // TEST (EDEN_NX2_READ_TRACE): served by an early download. + bool early_cover = false; + if (ReadTrace::Enabled()) [[unlikely]] { + early_cover = std::ranges::any_of(read_downloads, [&](const ReadDownload& download) { + return download.early && download.write_ranges.Intersects( + device_addr_start_aligned, + device_addr_end_aligned - device_addr_start_aligned); + }); + } if (!WriteFinishedReadDownloads(device_addr_start_aligned, device_addr_end_aligned - device_addr_start_aligned)) { + if (early_cover) [[unlikely]] { + ReadTrace::Count(ReadTrace::g_early.served_in_flight); + } area->in_flight = true; return area; } + if (early_cover) [[unlikely]] { + ReadTrace::Count(ReadTrace::g_early.served_at_read); + } // The host GPU had run them: in guest memory now, nothing is left to wait for. area->preemtive = !IsRegionGpuModified(device_addr_start_aligned, device_addr_end_aligned - device_addr_start_aligned); @@ -321,6 +344,9 @@ std::optional BufferCache

::GetFlushArea(DA return area; } } + if (check_early) [[unlikely]] { + area->preemtive = false; + } // A preflushable mark persists and downloads later GPU writes at every fence. Native titles // instead keep read traps and flush on demand. const bool preflush = !Tegra::g_nx2_ampere.load(std::memory_order_relaxed); @@ -620,25 +646,77 @@ void BufferCache

::RecordReadDownload(std::span> r }); } } - if (copies.empty()) { + // TEST (EDEN_NX2_READ_CHECK): what early downloads brought to the read's pages before it + // came. No longer GPU-modified, they are compared as they landed. + struct LandedCopy { + u64 offset; ///< In the check's staging, past its own offset + std::vector as_landed; + }; + boost::container::small_vector landed_copies; + if (check && !early && !early_landed.empty()) [[unlikely]] { + const auto add_landed = [&](DAddr lower, DAddr upper, const u8* as_landed) { + ForEachBufferInRange(lower, upper - lower, [&](BufferId buffer_id, Buffer& buffer) { + const DAddr buffer_addr = buffer.CpuAddr(); + const DAddr begin = (std::max)(lower, buffer_addr); + const DAddr end = (std::min)(upper, buffer_addr + buffer.SizeBytes()); + if (begin >= end) { + return; + } + const u8* const bytes = as_landed + (begin - lower); + landed_copies.push_back( + {check_size, std::vector(bytes, bytes + (end - begin))}); + check_copies.push_back({buffer_id, BufferCopy{ + .src_offset = begin - buffer_addr, + .dst_offset = check_size, + .size = end - begin, + }}); + check_size += Common::AlignUp(end - begin, 64); + check->kept.Add(begin, end - begin); + check->landed.Add(begin, end - begin); + }); + }; + for (const auto& [own_addr, own_size] : own) { + TakeEarlyLanded(own_addr, own_size, add_landed); + } + } + // TEST: a check whose read's bytes other downloads bring is a download of its own. + const bool check_only = !early && copies.empty() && check && !check_copies.empty(); + if (copies.empty() && !check_only) { return; } for (const Copy& entry : copies) { GpuWaitStats::AddDownloadBytes(entry.copy.size); } - auto staging = runtime.DownloadStagingBuffer(total_size_bytes, true); - for (Copy& entry : copies) { - Buffer& buffer = slot_buffers[entry.buffer_id]; - entry.copy.dst_offset += staging.offset; - buffer.MarkUsage(entry.copy.src_offset, entry.copy.size); - // Mapped buffers too: the GPU thread's later work may write them before the copy is - // read. - runtime.CopyBuffer(staging.buffer, buffer, std::span{&entry.copy, 1}, true); - download.copies.push_back(BufferCopy{ - .src_offset = buffer.CpuAddr() + entry.copy.src_offset, - .dst_offset = entry.copy.dst_offset, - .size = entry.copy.size, - }); + if constexpr (HAS_READ_TRACE) { + if (ReadTrace::Enabled()) [[unlikely]] { + // Writes noted without a draw or dispatch after them end here at the latest. + TraceFlushWriteMarks(); + download.trace.start = runtime.TraceMark(); + } + } + if (!copies.empty()) { + auto staging = runtime.DownloadStagingBuffer(total_size_bytes, true); + for (Copy& entry : copies) { + Buffer& buffer = slot_buffers[entry.buffer_id]; + entry.copy.dst_offset += staging.offset; + buffer.MarkUsage(entry.copy.src_offset, entry.copy.size); + // Mapped buffers too: the GPU thread's later work may write them before the copy + // is read. + runtime.CopyBuffer(staging.buffer, buffer, std::span{&entry.copy, 1}, true); + download.copies.push_back(BufferCopy{ + .src_offset = buffer.CpuAddr() + entry.copy.src_offset, + .dst_offset = entry.copy.dst_offset, + .size = entry.copy.size, + }); + } + download.staging = staging; + } + if constexpr (HAS_READ_TRACE) { + if (ReadTrace::Enabled()) [[unlikely]] { + download.trace.end = runtime.TraceMark(); + download.trace.sequence = ++trace_sequence; + download.trace.counts = runtime.TraceCounts(); + } } if (check && !check_copies.empty()) [[unlikely]] { // The same bytes again, from the same point of the GPU's work: what a synchronous @@ -653,15 +731,31 @@ void BufferCache

::RecordReadDownload(std::span> r .dst_offset = entry.copy.dst_offset, .size = entry.copy.size, }); + if (check_only) { + check->own.Add(buffer.CpuAddr() + entry.copy.src_offset, entry.copy.size); + } } if (!check->kept.Empty()) { check->landed_bytes.resize(check_size); } + for (const LandedCopy& landed : landed_copies) { + std::ranges::copy(landed.as_landed, check->landed_bytes.data() + landed.offset); + } download.check = std::move(check); } // A read waits for its copies; an early download's go with the next submission. download.tick = early ? runtime.QueueForHostRead() : runtime.SubmitForHostRead(); - download.staging = staging; + if constexpr (HAS_READ_TRACE) { + if (ReadTrace::Enabled()) [[unlikely]] { + download.trace.tick = download.tick; + download.trace.flushed = ReadTrace::Now(); + download.trace.early = early; + if (early) { + ReadTrace::Count(ReadTrace::g_early.recorded); + ReadTrace::Count(ReadTrace::g_early.bytes, total_size_bytes); + } + } + } if (early) { ++pending_early_downloads; early_bytes_this_frame += total_size_bytes; @@ -677,19 +771,56 @@ void BufferCache

::FinishReadDownloads(DAddr device_addr, u64 size) { std::unique_lock lock{mutex}; // The newest one over the range: those before it are written first. std::optional tick; + const ReadDownload* newest = nullptr; for (const ReadDownload& download : read_downloads) { - if (download.write_ranges.Intersects(device_addr, size)) { + // TEST (EDEN_NX2_READ_CHECK): a check alone is waited for too, as its read's download. + if (download.write_ranges.Intersects(device_addr, size) || + (download.check && download.check->own.Intersects(device_addr, size))) { tick = download.tick; + newest = &download; + } + } + // TEST (EDEN_NX2_READ_TRACE): the reading thread's read, when this is one. + ReadTrace::Read* trace = nullptr; + const auto trace_now = [&trace](u64 ReadTrace::Read::*field) { + if (trace != nullptr) [[unlikely]] { + if constexpr (HAS_READ_TRACE) { + trace->*field = ReadTrace::Now(); + } + } + }; + if constexpr (HAS_READ_TRACE) { + trace = ReadTrace::t_read; + if (trace != nullptr && newest != nullptr) [[unlikely]] { + TraceReadCopy(*trace, *newest, device_addr, size); } } if (!tick) { return; } if (!runtime.IsFree(*tick)) { + // TEST: a write from the GPU's side waits for an early download. + bool gpu_side = false; + u64 wait_start = 0; + if constexpr (HAS_READ_TRACE) { + gpu_side = ReadTrace::Enabled() && trace == nullptr && newest->early; + wait_start = gpu_side ? ReadTrace::Now() : 0; + } lock.unlock(); runtime.WaitForHostRead(*tick); + trace_now(&ReadTrace::Read::woke); + if constexpr (HAS_READ_TRACE) { + if (gpu_side) [[unlikely]] { + ReadTrace::Count(ReadTrace::g_early.gpu_side_waits); + ReadTrace::Count(ReadTrace::g_early.gpu_side_wait_ns, + ReadTrace::Now() - wait_start); + } + } lock.lock(); + } else { + trace_now(&ReadTrace::Read::woke); } + trace_now(&ReadTrace::Read::locked); WriteReadDownloads(*tick); } } @@ -727,6 +858,11 @@ bool BufferCache

::WriteFinishedReadDownloads(DAddr device_addr, u64 size) { template void BufferCache

::NoteWorkRecorded(bool dispatch) { if constexpr (!IS_OPENGL && USE_MEMORY_MAPS) { + if constexpr (HAS_READ_TRACE) { + if (ReadTrace::Enabled()) [[unlikely]] { + TraceWorkRecorded(dispatch); + } + } if (dispatch) { // Outside any render pass: the copies go right behind the dispatch. RecordEarlyDownloads(); @@ -765,6 +901,11 @@ void BufferCache

::NoteReadBack(DAddr begin, DAddr end, bool gpu_modified) { // Nothing new since its last read. continue; } + // TEST (EDEN_NX2_READ_TRACE): the bytes in guest memory before the read came. + if (it->landed_early && !gpu_modified && ReadTrace::Enabled()) [[unlikely]] { + ReadTrace::Count(ReadTrace::g_early.served_landed); + } + it->landed_early = false; it->written = false; it->unread_writes = 0; it->reads = (std::min)(static_cast(it->reads + 1), EarlyAfterReads); @@ -785,6 +926,7 @@ void BufferCache

::NoteReadBackWrite(DAddr device_addr, u64 size) { &ReadBackPage::page); it != read_back_pages.end() && it->page < end; ++it) { it->written = true; + it->landed_early = false; if (it->unread_writes < MaxUnreadWrites) { ++it->unread_writes; } else { @@ -819,6 +961,163 @@ void BufferCache

::RecordEarlyDownloads() { } } +template +void BufferCache

::TraceWorkRecorded(bool dispatch) { + if constexpr (HAS_READ_TRACE) { + runtime.TraceCountWork(dispatch); + TraceFlushWriteMarks(); + } +} + +template +void BufferCache

::TraceWriteRange(DAddr device_addr, u64 size) { + if constexpr (HAS_READ_TRACE) { + if (!cpu_read_ranges.Intersects(device_addr, size)) { + return; + } + trace_writes[trace_write_count++ % trace_writes.size()] = TraceWrite{ + .start = device_addr, + .end = device_addr + size, + .sequence = ++trace_sequence, + }; + trace_marks_pending = true; + } +} + +template +void BufferCache

::TraceFlushWriteMarks() { + if constexpr (HAS_READ_TRACE) { + if (!trace_marks_pending) { + return; + } + trace_marks_pending = false; + const ReadTrace::Write write{ + .tick = runtime.CurrentTick(), + .recorded = ReadTrace::Now(), + .mark = runtime.TraceMark(), + .counts = runtime.TraceCounts(), + }; + // The newest entries without one: those noted since the last mark. + const size_t count = (std::min)(trace_write_count, trace_writes.size()); + for (size_t i = 0; i < count; ++i) { + TraceWrite& entry = trace_writes[(trace_write_count - 1 - i) % trace_writes.size()]; + if (entry.write.recorded != 0) { + break; + } + entry.write = write; + } + } +} + +template +void BufferCache

::TraceReadCopy(ReadTrace::Read& read, const ReadDownload& download, + DAddr device_addr, u64 size) const { + if (download.trace.tick == 0) { + return; + } + read.has_copy = true; + read.copy = download.trace; + // The area's last GPU write before the copy. + const size_t count = (std::min)(trace_write_count, trace_writes.size()); + for (size_t i = 0; i < count; ++i) { + const TraceWrite& entry = trace_writes[(trace_write_count - 1 - i) % trace_writes.size()]; + if (entry.sequence > download.trace.sequence || entry.write.recorded == 0 || + entry.end <= device_addr || entry.start >= device_addr + size) { + continue; + } + read.has_write = true; + read.write = entry.write; + return; + } +} + +template +bool BufferCache

::EarlyDownloadServes(DAddr device_addr, u64 size) const { + if (std::ranges::any_of(read_downloads, [&](const ReadDownload& download) { + return download.early && download.write_ranges.Intersects(device_addr, size); + })) { + return true; + } + for (DAddr page = Common::AlignDown(device_addr, DEVICE_PAGESIZE); page < device_addr + size; + page += DEVICE_PAGESIZE) { + const auto it = early_landed.find(page); + if (it != early_landed.end() && it->second.bytes.Intersects(device_addr, size)) { + return true; + } + } + return false; +} + +template +void BufferCache

::NoteEarlyLanded(const ReadDownload& download, + const Common::RangeSet& cpu_written) { + // Pages no read came for since keep theirs until a download lands there. Bounded: past it, + // their reads are not compared. + if (early_landed.size() >= 4 * MaxReadBackPages) { + early_landed.clear(); + } + download.clean_ranges.ForEach([&](DAddr start, DAddr end) { + Common::RangeSet landed; + landed.Add(start, end - start); + cpu_written.ForEachInRange(start, end - start, [&](DAddr lower, DAddr upper) { + landed.Subtract(lower, upper - lower); + }); + landed.ForEach([&](DAddr lower, DAddr upper) { + for (DAddr page = Common::AlignDown(lower, DEVICE_PAGESIZE); page < upper; + page += DEVICE_PAGESIZE) { + const DAddr from = (std::max)(lower, page); + const DAddr to = (std::min)(upper, page + DEVICE_PAGESIZE); + EarlyLanded& entry = early_landed[page]; + entry.as_landed.resize(DEVICE_PAGESIZE); + device_memory.ReadBlockUnsafe(from, entry.as_landed.data() + (from - page), + to - from); + entry.bytes.Add(from, to - from); + } + }); + }); +} + +template +template +void BufferCache

::TakeEarlyLanded(DAddr device_addr, u64 size, Func&& func) { + for (DAddr page = Common::AlignDown(device_addr, DEVICE_PAGESIZE); page < device_addr + size; + page += DEVICE_PAGESIZE) { + const auto it = early_landed.find(page); + if (it == early_landed.end()) { + continue; + } + Common::RangeSet taken; + it->second.bytes.ForEachInRange( + device_addr, size, [&](DAddr lower, DAddr upper) { taken.Add(lower, upper - lower); }); + // A newer GPU write's bytes are the read's own again. + gpu_modified_ranges.ForEachInRange(page, DEVICE_PAGESIZE, [&](DAddr lower, DAddr upper) { + taken.Subtract(lower, upper - lower); + }); + taken.ForEach([&](DAddr lower, DAddr upper) { + func(lower, upper, it->second.as_landed.data() + (lower - page)); + it->second.bytes.Subtract(lower, upper - lower); + }); + if (it->second.bytes.Empty()) { + early_landed.erase(it); + } + } +} + +template +void BufferCache

::ForgetEarlyLanded(DAddr device_addr, u64 size) { + for (DAddr page = Common::AlignDown(device_addr, DEVICE_PAGESIZE); page < device_addr + size; + page += DEVICE_PAGESIZE) { + const auto it = early_landed.find(page); + if (it == early_landed.end()) { + continue; + } + it->second.bytes.Subtract(device_addr, size); + if (it->second.bytes.Empty()) { + early_landed.erase(it); + } + } +} + template void BufferCache

::WriteReadDownloads(u64 tick) { if constexpr (!IS_OPENGL) { @@ -838,6 +1137,22 @@ void BufferCache

::WriteReadDownloads(u64 tick) { end - start); }); } + // TEST: what early downloads brought, for the reads they serve. + if (download.early) { + if (ReadTrace::Enabled()) [[unlikely]] { + download.clean_ranges.ForEach([&](DAddr start, DAddr end) { + for (auto it = std::ranges::lower_bound( + read_back_pages, Common::AlignDown(start, DEVICE_PAGESIZE), {}, + &ReadBackPage::page); + it != read_back_pages.end() && it->page < end; ++it) { + it->landed_early = true; + } + }); + } + if (ReadChecks()) [[unlikely]] { + NoteEarlyLanded(download, cpu_written); + } + } // TEST: what later reads' checks find of an earlier download where they kept its bytes. for (size_t later = 1; later < read_downloads.size(); ++later) { if (read_downloads[later].check) [[unlikely]] { @@ -863,8 +1178,10 @@ void BufferCache

::WriteReadDownloads(u64 tick) { } } }); - // Freed by the GPU thread (TickFrame). - async_buffers_death_ring.push_back(download.staging); + // Freed by the GPU thread (TickFrame). TEST: a check alone has none. + if (!download.copies.empty()) { + async_buffers_death_ring.push_back(download.staging); + } if (download.early) { --pending_early_downloads; } @@ -1045,6 +1362,10 @@ void BufferCache

::NoteLandedBytes(ReadCheck& check, const Common::RangeSet void BufferCache

::ClearDownload(DAddr device_addr, u64 size) { ClearFenceDownloads(device_addr, size); + // TEST (EDEN_NX2_READ_CHECK): a newer value there, as without early downloads. + if (!early_landed.empty()) [[unlikely]] { + ForgetEarlyLanded(device_addr, size); + } if (!read_downloads.empty()) [[unlikely]] { for (ReadDownload& download : read_downloads) { download.write_ranges.Subtract(device_addr, size); @@ -1117,6 +1438,9 @@ bool BufferCache

::DMACopy(GPUVAddr src_address, GPUVAddr dest_address, u64 am gpu_modified_ranges.Add(pair.first, pair.second); GpuWaitStats::NoteGpuWrite(pair.first, pair.second, GpuWaitStats::WriteSite::DmaCopy); NoteReadBackWrite(pair.first, pair.second); + if (ReadTrace::Enabled()) [[unlikely]] { + TraceWriteRange(pair.first, pair.second); + } if (Tegra::g_nx2_ampere.load(std::memory_order_relaxed)) { Tegra::Nx2MarkGpuWritten(pair.first, pair.second); Nx2NoteListWrite(buffer_b, pair.first, pair.second); @@ -1126,6 +1450,9 @@ bool BufferCache

::DMACopy(GPUVAddr src_address, GPUVAddr dest_address, u64 am src_buffer.MarkUsage(copy.src_offset, copy.size); dest_buffer.MarkUsage(copy.dst_offset, copy.size); runtime.CopyBuffer(dest_buffer, src_buffer, copies, true); + if (ReadTrace::Enabled()) [[unlikely]] { + TraceFlushWriteMarks(); + } if (has_new_downloads) { memory_tracker.MarkRegionAsGpuModified(*cpu_dest_address, amount); const bool should_sync = Settings::IsGPUFenceBehaviorBalanced() || Settings::IsGPUFenceBehaviorAccurate(); @@ -1669,6 +1996,10 @@ Common::RangeSet BufferCache

::PendingCpuWriteRanges() { template void BufferCache

::WriteDownloadedMemory(const Common::RangeSet& cpu_written, DAddr device_addr, const u8* data, u64 size) { + // TEST (EDEN_NX2_READ_CHECK): newer GPU bytes than an early download brought before. + if (!early_landed.empty()) [[unlikely]] { + ForgetEarlyLanded(device_addr, size); + } if (!cpu_written.Intersects(device_addr, size)) { device_memory.WriteBlockUnsafe(device_addr, data, size); return; @@ -2692,6 +3023,9 @@ template void BufferCache

::MarkWrittenBuffer(BufferId buffer_id, DAddr device_addr, u32 size, bool needs_sync, GpuWaitStats::WriteSite site) { GpuWaitStats::NoteGpuWrite(device_addr, size, site); + if (ReadTrace::Enabled()) [[unlikely]] { + TraceWriteRange(device_addr, size); + } if constexpr (!IS_OPENGL) { if (needs_sync) { Buffer& buffer = slot_buffers[buffer_id]; diff --git a/src/video_core/buffer_cache/buffer_cache_base.h b/src/video_core/buffer_cache/buffer_cache_base.h index 7f3853a1bb..e600451e52 100644 --- a/src/video_core/buffer_cache/buffer_cache_base.h +++ b/src/video_core/buffer_cache/buffer_cache_base.h @@ -46,6 +46,7 @@ #include "video_core/gpu_wait_stats.h" #include "video_core/memory_manager.h" #include "video_core/nx2_download_trace.h" +#include "video_core/nx2_read_trace.h" #include "video_core/rasterizer_download_area.h" #include "video_core/surface.h" #include "video_core/texture_cache/types.h" @@ -204,6 +205,8 @@ class BufferCache : public VideoCommon::ChannelSetupCaches landed_bytes; Common::RangeSet landed; + /// A check alone, its read's bytes all brought by other downloads: the bytes it compares, + /// which its read waits for as for a download of its own. + Common::RangeSet own; }; struct ReadCheckTotals { u64 every{}; ///< Tests: check 1 in `every` read downloads, over EDEN_NX2_READ_CHECK @@ -853,6 +859,9 @@ class BufferCache : public VideoCommon::ChannelSetupCaches& cpu_written); + [[nodiscard]] bool ReadChecks() const { + return read_check.every != 0 || ReadCheckEvery() != 0; + } /// A download landed `written`: what guest memory holds now over a later check's `kept`. void NoteLandedBytes(ReadCheck& check, const Common::RangeSet& written); @@ -873,6 +882,7 @@ class BufferCache : public VideoCommon::ChannelSetupCaches check; ///< TEST: EDEN_NX2_READ_CHECK /// Recorded right after a GPU write, before any read (RecordEarlyDownloads) bool early{}; + ReadTrace::Copy trace{}; ///< TEST: EDEN_NX2_READ_TRACE }; /// Oldest first, and written to guest memory in that order: none lands over newer bytes. std::deque read_downloads; @@ -880,6 +890,46 @@ class BufferCache : public VideoCommon::ChannelSetupCaches read_download_ranges; + /// TEST (EDEN_NX2_READ_CHECK): the bytes early downloads brought to a page's guest memory, and + /// what guest memory held there as they landed, until newer downloaded bytes or values replace + /// them. The read they serve compares them with the GPU's bytes then: a GPU write in between + /// that took nothing out of them shows. + struct EarlyLanded { + Common::RangeSet bytes; + std::vector as_landed; ///< The page, DEVICE_PAGESIZE bytes + }; + std::unordered_map early_landed; + /// Whether an early download serves this read: in check mode it flushes (and is compared). + [[nodiscard]] bool EarlyDownloadServes(DAddr device_addr, u64 size) const; + void NoteEarlyLanded(const ReadDownload& download, const Common::RangeSet& cpu_written); + /// Calls func(begin, end, the bytes as they landed) for what early downloads brought to + /// [device_addr, device_addr + size) that no GPU write changed since, which a read takes: + /// a later read compares only what lands after it. + template + void TakeEarlyLanded(DAddr device_addr, u64 size, Func&& func); + void ForgetEarlyLanded(DAddr device_addr, u64 size); + + /// TEST (EDEN_NX2_READ_TRACE): the latest GPU writes over pages the guest read back before + /// (cpu_read_ranges), newest last; those without a mark yet get the next one. + struct TraceWrite { + DAddr start{}; + DAddr end{}; + u64 sequence{}; ///< 0: unused + ReadTrace::Write write{}; + }; + std::array trace_writes{}; + size_t trace_write_count{}; + u64 trace_sequence{}; + bool trace_marks_pending{}; + /// The GPU thread recorded a draw or dispatch: a device timestamp after it dates its writes + /// to pages the guest reads back. + void TraceWorkRecorded(bool dispatch); + void TraceWriteRange(DAddr device_addr, u64 size); + void TraceFlushWriteMarks(); + /// The reading thread's read waits for `download` (`mutex` held). + void TraceReadCopy(ReadTrace::Read& read, const ReadDownload& download, DAddr device_addr, + u64 size) const; + /// A page the guest CPU read GPU writes from. Switch 2 titles read compute results back every /// frame, milliseconds after the dispatch that wrote them: a download recorded at the read /// waits for all the GPU work recorded in between. Pages read after each of their writes get @@ -891,6 +941,7 @@ class BufferCache : public VideoCommon::ChannelSetupCachesReadRasterizer()->TraceRead(trace, ReadTrace::Step::Request); + } // In flight: an earlier read's downloads cover the area, only FinishRead is left. if (!raster_area.in_flight) { const u64 fence = RequestSyncOperation([this, raster_area, &timing]() { @@ -305,14 +316,28 @@ struct GPU::Impl { // thread's queue still runs it without the split. VideoCommon::GpuSplit::Get().RequestTick(); gpu_thread.TickGPU(is_async); + if (traced) [[unlikely]] { + // While the GPU thread runs the flush. + renderer->ReadRasterizer()->TraceRead(trace, ReadTrace::Step::Calibrate); + } WaitForSyncOperation(fence); + } else if (traced) [[unlikely]] { + renderer->ReadRasterizer()->TraceRead(trace, ReadTrace::Step::Calibrate); } // The GPU thread submitted the downloads without waiting for the host GPU: this thread // waits for them and writes them to guest memory, while the GPU thread goes on. const u64 finish_start = Stats::Now(); + if (traced) [[unlikely]] { + ReadTrace::t_read = &trace; + } renderer->ReadRasterizer()->FinishRead(raster_area.start_address, raster_area.end_address - raster_area.start_address); const u64 woke = Stats::Now(); + if (traced) [[unlikely]] { + ReadTrace::t_read = nullptr; + trace.landed = woke; + renderer->ReadRasterizer()->TraceRead(trace, ReadTrace::Step::Done); + } timing.finish_ns = woke - finish_start; timing.in_flight = raster_area.in_flight; Stats::RecordReactive(raster_area, core, request, timing, woke); diff --git a/src/video_core/nx2_read_trace.h b/src/video_core/nx2_read_trace.h new file mode 100644 index 0000000000..7c798e802f --- /dev/null +++ b/src/video_core/nx2_read_trace.h @@ -0,0 +1,535 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "common/common_types.h" +#include "common/logging.h" +#include "video_core/gpu_wait_stats.h" + +// TEST (EDEN_NX2_READ_TRACE=1): where a guest read of GPU-written memory spends its wait. +// +// Each read that waits (GPU::OnCPURead) is placed on the host GPU's timeline. Vulkan writes a +// device timestamp at the start and end of every submission, after each draw or dispatch that +// writes pages the guest reads back (the area's last such write before the read's copy is "the +// write"), and before and after the read's copy. The reading thread reads the device's clock +// (VK_EXT_calibrated_timestamps) once the flush is asked for, which places the request on that +// timeline. The wait then splits into: until the write's work ended on the GPU (the GPU still +// had it, or work queued before it, to run), until the copy could start (work queued after the +// write), the copy, and the reading thread's wake-up, lock and copy into guest memory. Also kept: +// where the write was at the request (done, on the GPU, not yet queued) and how long before it +// the GPU thread recorded it, the work recorded between the write and the copy, the submissions +// in flight at the request with the GPU time they still ran, and how long the copy waited for +// its vkQueueSubmit. In the frames EDEN_DKB_FRAME_TRACE samples (each draw and dispatch timed), +// the GPU time before the copy also splits into timed draws and dispatches and the rest. +// +// Every 10 s: one "Read trace" line for all reads, and one for each of the areas that waited +// longest. Times are per read, in ms. +namespace VideoCommon::ReadTrace { + +[[nodiscard]] inline bool SwitchSet() { + const char* value = std::getenv("EDEN_NX2_READ_TRACE"); + return value != nullptr && *value != '\0' && *value != '0'; +} + +// A plain flag: checked at every read download and fence. +inline bool g_enabled = SwitchSet(); + +/// Called by VideoCore::CreateGPU: static initialization runs before the frontend loads +/// nx2_switches.txt, so a switch set only there takes effect from that point on. +inline void ReadSwitch() { + g_enabled = SwitchSet(); +} + +[[nodiscard]] inline bool Enabled() { + return g_enabled; +} + +/// Nanoseconds, the clock of the GPU wait stats (the same on every thread). +[[nodiscard]] inline u64 Now() { + return GpuWaitStats::Now(); +} + +/// Work the GPU thread recorded so far. +struct Counts { + u64 draws{}; + u64 dispatches{}; + u64 render_passes{}; +}; + +/// The area's last GPU write before the read's copy was recorded. +struct Write { + u64 tick{}; ///< The submission it went in + u64 recorded{}; ///< Now() once the writing draw or dispatch was recorded + u64 mark{}; ///< Device timestamp after it (Vulkan::Scheduler::TraceMark), 0 if none + Counts counts{}; +}; + +/// The copy a read waited for (BufferCache::DownloadMemoryForRead). +struct Copy { + u64 tick{}; ///< Its submission + u64 flushed{}; ///< Now() once the GPU thread had handed it over for submission + u64 start{}; ///< Device timestamp before the copies: all earlier work done + u64 end{}; ///< After them + u64 sequence{}; ///< Writes the buffer cache noted before it have a lower one + Counts counts{}; + bool early{}; ///< Recorded right behind its write (BufferCache::RecordEarlyDownloads) +}; + +enum class Step : u8 { + Request, ///< As the reading thread asks for the flush + Calibrate, ///< While it waits for the GPU thread to run it: the device's clock + Done, ///< The bytes are in guest memory +}; + +/// One guest read that waited for the GPU, filled in as it goes. +struct Read { + u64 area{}; + bool in_flight{}; ///< Waited for an earlier read's copy + // Step::Request + u64 request{}; + u64 completed{}; ///< The last submission the host GPU completed + u64 submitted{}; ///< The last one vkQueueSubmit returned for + u64 recording{}; ///< The one being recorded: those before it went to the Vulkan worker + // Step::Calibrate: the device's timestamp `device` at Now() `host` (+- `uncertainty`) + u64 device{}; + u64 host{}; + u64 uncertainty{}; + // FinishReadDownloads (the reading thread) + bool has_copy{}; + Copy copy{}; + bool has_write{}; + Write write{}; + u64 woke{}; ///< The copy's submission was complete + u64 locked{}; ///< Then the buffer cache lock was taken + u64 landed{}; ///< Then the bytes were in guest memory +}; + +/// The reading thread's read while it is in RasterizerInterface::FinishRead. +inline thread_local Read* t_read{}; + +/// Where a read's wait went on the device's timeline (Vulkan::Scheduler, Step::Done). +struct Timeline { + bool placed{}; ///< Calibrated, and the timestamps it needs were read + /// ns from the request: until the write ended, until the copy could start, the copy, until + /// the reading thread woke. They add up to `woke - request`. + u64 to_write{}; + u64 to_copy{}; + u64 copy{}; + u64 to_wake{}; + /// Of to_write and to_copy, the time the host GPU ran no submission + u64 idle_to_write{}; + u64 idle_to_copy{}; + /// GPU time the submissions in flight at the request still ran after it + u64 backlog_busy{}; + /// The write's work ended this long before the request (0: after it) + u64 write_lead{}; + /// From the write's end to the copy's start on the GPU, wherever the request fell + u64 write_gap{}; + /// The copy's vkQueueSubmit returned this long after the GPU thread handed it over + bool has_submit{}; + u64 submit{}; + /// Device ticks: the request, the write's end (not before the request), the copy's start + std::array window{}; +}; + +/// The device timestamps placing a read takes (Vulkan::Scheduler). +struct DeviceTimes { + double ns_per_tick{}; + u64 write_end{}; + u64 copy_start{}; + u64 copy_end{}; + /// Start and end of each submission after the last one the host GPU had completed at the + /// request (read.completed + 1 on), up to the copy's. + std::span> submissions; +}; + +/// Splits a read's wait on the device's timeline. +[[nodiscard]] inline Timeline Place(const Read& read, const DeviceTimes& times) { + // ns from the request: the device read `device` at Now() `host`. + const auto at = [&](u64 ticks) { + return static_cast(static_cast(ticks - read.device)) * times.ns_per_tick + + static_cast(read.host - read.request); + }; + // The host GPU's time running submissions in [lower, upper), those up to last_tick. + const auto busy = [&](double lower, double upper, u64 last_tick) { + double sum = 0.0; + double covered = lower; + for (size_t i = 0; i < times.submissions.size(); ++i) { + if (read.completed + 1 + i > last_tick) { + break; + } + const double from = (std::max)(at(times.submissions[i].first), covered); + const double to = (std::min)(at(times.submissions[i].second), upper); + if (to > from) { + sum += to - from; + covered = to; + } + } + return sum; + }; + const auto ns = [](double value) { return static_cast((std::max)(value, 0.0)); }; + const double woke = static_cast(read.woke - read.request); + const double write_ended = at(times.write_end); + const double written = std::clamp(write_ended, 0.0, woke); + const double copy_started = std::clamp(at(times.copy_start), written, woke); + const double copied = std::clamp(at(times.copy_end), copy_started, woke); + Timeline timeline{}; + timeline.placed = true; + timeline.to_write = ns(written); + timeline.to_copy = ns(copy_started - written); + timeline.copy = ns(copied - copy_started); + timeline.to_wake = ns(woke - copied); + timeline.idle_to_write = ns(written - busy(0.0, written, read.copy.tick)); + timeline.idle_to_copy = + ns(copy_started - written - busy(written, copy_started, read.copy.tick)); + timeline.backlog_busy = ns(busy(0.0, std::numeric_limits::infinity(), read.submitted)); + timeline.write_lead = ns(-write_ended); + timeline.write_gap = ns(at(times.copy_start) - write_ended); + const double request_ticks = static_cast(read.device) - + static_cast(read.host - read.request) / times.ns_per_tick; + const auto ticks_at = [&](double offset) { + return static_cast(request_ticks + offset / times.ns_per_tick); + }; + timeline.window = {ticks_at(0.0), ticks_at(written), ticks_at(copy_started)}; + return timeline; +} + +/// Early read downloads (BufferCache::RecordEarlyDownloads), any thread. +struct EarlyTotals { + std::atomic recorded{}; + std::atomic bytes{}; + std::atomic served_landed{}; ///< Reads that found their bytes in guest memory + std::atomic served_at_read{}; ///< Done on the host GPU: the read's lookup landed them + std::atomic served_in_flight{}; ///< Not yet done: the read waited for them + std::atomic gpu_side_waits{}; ///< Writes from the GPU's side that waited for one + std::atomic gpu_side_wait_ns{}; +}; +inline EarlyTotals g_early; + +inline void Count(std::atomic& counter, u64 amount = 1) { + counter.fetch_add(amount, std::memory_order_relaxed); +} + +struct Totals { + u64 reads{}; + u64 in_flight{}; + u64 early_copies{}; ///< The copy waited for was an early download + u64 no_copy{}; ///< Found no copy to wait for + u64 no_write{}; ///< No GPU write over the area seen since it was first read back + u64 copy_done{}; ///< The copy's submission was complete at the request + u64 wait{}; ///< Request until the bytes were in guest memory, all reads + u64 lock{}; ///< Reads with a copy: woke until locked + u64 land{}; ///< Locked until in guest memory + // Placed reads + u64 placed{}; + u64 to_write{}; + u64 to_copy{}; + u64 copy{}; + u64 to_wake{}; + u64 idle_to_write{}; + u64 idle_to_copy{}; + u64 backlog_busy{}; + u64 uncertainty{}; + u64 write_ended{}; ///< The write's work had ended at the request (write_lead is over these) + u64 write_lead{}; + u64 write_gap{}; + // Reads with a copy: the submissions in flight, and those not yet queued, at the request + u64 backlog_subs{}; + u64 unqueued_subs{}; + u64 submits{}; + u64 submit{}; + // Reads with a write: where it was at the request, when it was recorded, what came after it + u64 write_done{}; + u64 write_on_gpu{}; + u64 write_unqueued{}; ///< Flushed to the Vulkan worker, no vkQueueSubmit yet + u64 write_recording{}; ///< In the submission still being recorded + u64 written_before{}; ///< Recorded before the request (since_write is over these) + u64 since_write{}; + u64 draws{}; + u64 dispatches{}; + u64 render_passes{}; + u64 submissions{}; + // Placed reads in the frames EDEN_DKB_FRAME_TRACE samples + u64 sampled{}; + u64 timed_to_write{}; + u64 busy_to_write{}; + u64 timed_to_copy{}; + u64 busy_to_copy{}; +}; + +/// A placed read, until its frame's timed work arrives (NoteTimedWork). +struct Window { + u64 area{}; + std::array ticks{}; + u64 busy_to_write{}; + u64 busy_to_copy{}; + bool pending{}; +}; + +struct State { + std::mutex mutex; + u64 window_start{}; + u64 presents{}; + Totals all; + std::vector> areas; + Totals other_areas; ///< Past MaxAreas + std::array windows{}; + size_t next_window{}; +}; +inline State g_state; +inline constexpr size_t MaxAreas = 64; +inline constexpr size_t ShownAreas = 4; + +/// `mutex` held. +inline Totals& AreaTotals(u64 area) { + for (auto& [start, totals] : g_state.areas) { + if (start == area) { + return totals; + } + } + if (g_state.areas.size() >= MaxAreas) { + return g_state.other_areas; + } + return g_state.areas.emplace_back(area, Totals{}).second; +} + +inline void Add(Totals& totals, const Read& read, const Timeline& timeline) { + ++totals.reads; + totals.in_flight += read.in_flight ? 1 : 0; + totals.wait += read.landed - read.request; + if (!read.has_copy) { + ++totals.no_copy; + return; + } + totals.copy_done += read.copy.tick <= read.completed ? 1 : 0; + totals.early_copies += read.copy.early ? 1 : 0; + totals.lock += read.locked - read.woke; + totals.land += read.landed - read.locked; + totals.backlog_subs += read.submitted > read.completed ? read.submitted - read.completed : 0; + totals.unqueued_subs += + read.recording > read.submitted + 1 ? read.recording - read.submitted - 1 : 0; + if (timeline.has_submit) { + ++totals.submits; + totals.submit += timeline.submit; + } + if (timeline.placed) { + ++totals.placed; + totals.to_write += timeline.to_write; + totals.to_copy += timeline.to_copy; + totals.copy += timeline.copy; + totals.to_wake += timeline.to_wake; + totals.idle_to_write += timeline.idle_to_write; + totals.idle_to_copy += timeline.idle_to_copy; + totals.backlog_busy += timeline.backlog_busy; + totals.uncertainty += read.uncertainty; + totals.write_ended += timeline.write_lead != 0 ? 1 : 0; + totals.write_lead += timeline.write_lead; + totals.write_gap += timeline.write_gap; + } + if (!read.has_write) { + ++totals.no_write; + return; + } + const Write& write = read.write; + if (write.tick <= read.completed) { + ++totals.write_done; + } else if (write.tick <= read.submitted) { + ++totals.write_on_gpu; + } else if (write.tick < read.recording) { + ++totals.write_unqueued; + } else { + ++totals.write_recording; + } + if (write.recorded <= read.request) { + ++totals.written_before; + totals.since_write += read.request - write.recorded; + } + const Counts& after = read.copy.counts; + totals.draws += after.draws - write.counts.draws; + totals.dispatches += after.dispatches - write.counts.dispatches; + totals.render_passes += after.render_passes - write.counts.render_passes; + totals.submissions += read.copy.tick - write.tick; +} + +inline std::string Format(u64 area, const Totals& t) { + const auto ms = [](u64 sum, u64 count) { + return count == 0 ? 0.0 : static_cast(sum) / 1e6 / static_cast(count); + }; + const auto per = [](u64 sum, u64 count) { + return count == 0 ? 0.0 : static_cast(sum) / static_cast(count); + }; + std::string line = area == 0 ? std::string{"all"} : fmt::format("{:#x}", area); + const u64 copied = t.reads - t.no_copy; + line += fmt::format(": {} reads ({} in flight, {} waited for an early download, {} without a " + "copy, {} with it done at the request, {} placed on the GPU timeline " + "+-{:.3f}), wait {:.2f}", + t.reads, t.in_flight, t.early_copies, t.no_copy, t.copy_done, t.placed, + ms(t.uncertainty, t.placed) / 2, ms(t.wait, t.reads)); + if (t.placed != 0) { + line += fmt::format(" | from the request: to the write's end {:.2f} (GPU idle {:.2f}), to " + "the copy's start {:.2f} (idle {:.2f}), copy {:.2f}, to the wake-up " + "{:.2f}", + ms(t.to_write, t.placed), ms(t.idle_to_write, t.placed), + ms(t.to_copy, t.placed), ms(t.idle_to_copy, t.placed), + ms(t.copy, t.placed), ms(t.to_wake, t.placed)); + } + if (copied != 0) { + line += fmt::format("{}lock {:.2f}, to guest memory {:.2f}", t.placed != 0 ? ", " : " | ", + ms(t.lock, copied), ms(t.land, copied)); + } + if (t.placed != 0) { + line += + fmt::format("; the write's end to the copy's start {:.2f}; the write had ended " + "at the request {}x, {:.2f} before it", + ms(t.write_gap, t.placed), t.write_ended, ms(t.write_lead, t.write_ended)); + } + if (copied != 0) { + line += fmt::format(" | at the request: {:.1f} submissions in flight", + per(t.backlog_subs, copied)); + if (t.placed != 0) { + line += fmt::format(" (GPU busy {:.2f} more)", ms(t.backlog_busy, t.placed)); + } + line += fmt::format(", {:.1f} flushed but not queued; the copy queued {:.3f} after its " + "flush", + per(t.unqueued_subs, copied), ms(t.submit, t.submits)); + } + const u64 written = copied - t.no_write; + if (written != 0) { + line += fmt::format(" | the write at the request: done {}, on the GPU {}, flushed but not " + "queued {}, still recording {}; recorded {:.2f} before it ({} after); " + "then {:.1f} draws, {:.1f} dispatches, {:.1f} render passes, {:.1f} " + "submissions until the copy", + t.write_done, t.write_on_gpu, t.write_unqueued, t.write_recording, + ms(t.since_write, t.written_before), written - t.written_before, + per(t.draws, written), per(t.dispatches, written), + per(t.render_passes, written), per(t.submissions, written)); + } + if (t.no_write != 0) { + line += fmt::format(" ({} without a write seen)", t.no_write); + } + if (t.sampled != 0) { + line += + fmt::format(" | sampled frames, {} reads: draws and dispatches {:.2f} of the {:.2f} " + "busy to the write's end, {:.2f} of {:.2f} to the copy's start", + t.sampled, ms(t.timed_to_write, t.sampled), ms(t.busy_to_write, t.sampled), + ms(t.timed_to_copy, t.sampled), ms(t.busy_to_copy, t.sampled)); + } + return line; +} + +/// `mutex` held. +inline void MaybeReport(u64 now) { + const u64 presents = GpuWaitStats::g_presents.load(std::memory_order_relaxed); + if (g_state.window_start == 0) { + g_state.window_start = now; + g_state.presents = presents; + return; + } + if (now - g_state.window_start < GpuWaitStats::WindowNs) { + return; + } + std::ranges::sort(g_state.areas, std::greater<>{}, + [](const auto& entry) { return entry.second.wait; }); + LOG_INFO(HW_GPU, "Read trace (EDEN_NX2_READ_TRACE) in {:.1f} s, {} frames: {}", + static_cast(now - g_state.window_start) / 1e9, presents - g_state.presents, + Format(0, g_state.all)); + const auto take = [](std::atomic& counter) { + return counter.exchange(0, std::memory_order_relaxed); + }; + const u64 gpu_side_waits = take(g_early.gpu_side_waits); + LOG_INFO(HW_GPU, + "Read trace early downloads: {} recorded ({} KiB) | reads they served: {} found " + "the bytes in guest memory, {} found them done on the host GPU (landed at the read), " + "{} waited for them (in the lines above) | writes from the GPU's side that waited " + "for one: {} ({:.2f} ms)", + take(g_early.recorded), take(g_early.bytes) / 1024, take(g_early.served_landed), + take(g_early.served_at_read), take(g_early.served_in_flight), gpu_side_waits, + static_cast(take(g_early.gpu_side_wait_ns)) / 1e6); + for (size_t i = 0; i < g_state.areas.size() && i < ShownAreas; ++i) { + LOG_INFO(HW_GPU, "Read trace area {}", + Format(g_state.areas[i].first, g_state.areas[i].second)); + } + g_state.window_start = now; + g_state.presents = presents; + g_state.all = {}; + g_state.areas.clear(); + g_state.other_areas = {}; +} + +/// The reading thread, once its read is done (from Vulkan::Scheduler). +inline void Record(const Read& read, const Timeline& timeline) { + std::scoped_lock lock{g_state.mutex}; + Add(g_state.all, read, timeline); + Add(AreaTotals(read.area), read, timeline); + if (timeline.placed) { + g_state.windows[g_state.next_window++ % g_state.windows.size()] = Window{ + .area = read.area, + .ticks = timeline.window, + .busy_to_write = timeline.to_write - timeline.idle_to_write, + .busy_to_copy = timeline.to_copy - timeline.idle_to_copy, + .pending = true, + }; + } + MaybeReport(Now()); +} + +/// Once a frame (BufferCache::TickFrame): reports even while no read waits. +inline void Tick() { + if (!Enabled()) { + return; + } + std::scoped_lock lock{g_state.mutex}; + MaybeReport(Now()); +} + +/// The Vulkan worker, with the draws and dispatches of a frame EDEN_DKB_FRAME_TRACE timed +/// ([begin, end] device ticks, in order): splits the GPU time of the reads placed in it. +inline void NoteTimedWork(std::span> work, double ns_per_tick) { + if (work.empty()) { + return; + } + const u64 first = work.front().first; + const u64 last = work.back().second; + const auto timed = [&](u64 lower, u64 upper) { + u64 ticks = 0; + for (const auto& [begin, end] : work) { + const u64 from = (std::max)(begin, lower); + const u64 to = (std::min)(end, upper); + ticks += to > from ? to - from : 0; + } + return static_cast(static_cast(ticks) * ns_per_tick); + }; + std::scoped_lock lock{g_state.mutex}; + for (Window& window : g_state.windows) { + if (!window.pending || window.ticks[0] < first || window.ticks[2] > last) { + continue; + } + window.pending = false; + const u64 timed_to_write = timed(window.ticks[0], window.ticks[1]); + const u64 timed_to_copy = timed(window.ticks[1], window.ticks[2]); + for (Totals* totals : {&g_state.all, &AreaTotals(window.area)}) { + ++totals->sampled; + totals->timed_to_write += timed_to_write; + totals->busy_to_write += window.busy_to_write; + totals->timed_to_copy += timed_to_copy; + totals->busy_to_copy += window.busy_to_copy; + } + } +} + +} // namespace VideoCommon::ReadTrace diff --git a/src/video_core/rasterizer_interface.h b/src/video_core/rasterizer_interface.h index 198ef8c27a..335bccf634 100644 --- a/src/video_core/rasterizer_interface.h +++ b/src/video_core/rasterizer_interface.h @@ -38,6 +38,11 @@ namespace VideoCommon::OotNetwork { struct BypassState; } +namespace VideoCommon::ReadTrace { +struct Read; +enum class Step : u8; +} // namespace VideoCommon::ReadTrace + namespace VideoCore { enum class LoadCallbackStage { @@ -118,6 +123,11 @@ class RasterizerInterface { /// before it reads, and the GPU's side before it writes guest memory there. virtual void FinishRead([[maybe_unused]] DAddr addr, [[maybe_unused]] u64 size) {} + /// TEST (EDEN_NX2_READ_TRACE, nx2_read_trace.h): a step of a guest read that waits, on the + /// reading thread (GPU::OnCPURead). + virtual void TraceRead([[maybe_unused]] VideoCommon::ReadTrace::Read& read, + [[maybe_unused]] VideoCommon::ReadTrace::Step step) {} + /// Switch 2, from the GPU thread's front: where a GPU-written command list's words are. enum class GpuWrittenList { Unknown, ///< Neither: the caller drains, flushes the page and notes it synced diff --git a/src/video_core/renderer_vulkan/vk_buffer_cache.cpp b/src/video_core/renderer_vulkan/vk_buffer_cache.cpp index a87fbe02df..d471658892 100644 --- a/src/video_core/renderer_vulkan/vk_buffer_cache.cpp +++ b/src/video_core/renderer_vulkan/vk_buffer_cache.cpp @@ -555,6 +555,18 @@ bool BufferCacheRuntime::IsFreeNow(u64 tick) { return scheduler.IsFree(tick); } +u64 BufferCacheRuntime::TraceMark() { + return scheduler.TraceMark(); +} + +VideoCommon::ReadTrace::Counts BufferCacheRuntime::TraceCounts() const { + return scheduler.TraceCounts(); +} + +void BufferCacheRuntime::TraceCountWork(bool dispatch) { + scheduler.TraceCountWork(dispatch); +} + bool BufferCacheRuntime::CanReorderUpload(const Buffer& buffer, std::span copies) { if (Settings::values.disable_buffer_reorder) { diff --git a/src/video_core/renderer_vulkan/vk_buffer_cache.h b/src/video_core/renderer_vulkan/vk_buffer_cache.h index 230e60db14..b98cd89b5f 100644 --- a/src/video_core/renderer_vulkan/vk_buffer_cache.h +++ b/src/video_core/renderer_vulkan/vk_buffer_cache.h @@ -148,6 +148,11 @@ class BufferCacheRuntime { /// to be done. [[nodiscard]] bool IsFreeNow(u64 tick); + /// TEST (EDEN_NX2_READ_TRACE): Scheduler::TraceMark, TraceCounts and TraceCountWork. + u64 TraceMark(); + VideoCommon::ReadTrace::Counts TraceCounts() const; + void TraceCountWork(bool dispatch); + u64 GetDeviceLocalMemory() const; u64 GetDeviceMemoryUsage() const; diff --git a/src/video_core/renderer_vulkan/vk_rasterizer.cpp b/src/video_core/renderer_vulkan/vk_rasterizer.cpp index 16681a744c..a085d7600d 100644 --- a/src/video_core/renderer_vulkan/vk_rasterizer.cpp +++ b/src/video_core/renderer_vulkan/vk_rasterizer.cpp @@ -2037,6 +2037,11 @@ void RasterizerVulkan::FinishRead(DAddr addr, u64 size) { buffer_cache.FinishReadDownloads(addr, size); } +void RasterizerVulkan::TraceRead(VideoCommon::ReadTrace::Read& read, + VideoCommon::ReadTrace::Step step) { + scheduler.TraceRead(read, step); +} + bool RasterizerVulkan::MustFlushRegion(DAddr addr, u64 size, VideoCommon::CacheType which) { // The split's front asks this before each macro DMA read, and the back holds the buffer // cache's lock through each draw: pages the GPU never wrote need no look under it. diff --git a/src/video_core/renderer_vulkan/vk_rasterizer.h b/src/video_core/renderer_vulkan/vk_rasterizer.h index 1220dd74f1..94ee9a2f6a 100644 --- a/src/video_core/renderer_vulkan/vk_rasterizer.h +++ b/src/video_core/renderer_vulkan/vk_rasterizer.h @@ -107,6 +107,7 @@ class RasterizerVulkan final : public VideoCore::RasterizerInterface, VideoCommon::CacheType which = VideoCommon::CacheType::All) override; void FlushRegionForRead(DAddr addr, u64 size) override; void FinishRead(DAddr addr, u64 size) override; + void TraceRead(VideoCommon::ReadTrace::Read& read, VideoCommon::ReadTrace::Step step) override; GpuWrittenList ReadGpuWrittenCommandList(DAddr addr, std::span out, bool past_sync) override; void NoteGpuWrittenCommandListSynced(DAddr page) override; diff --git a/src/video_core/renderer_vulkan/vk_scheduler.cpp b/src/video_core/renderer_vulkan/vk_scheduler.cpp index d112c36187..4f9b4fd152 100644 --- a/src/video_core/renderer_vulkan/vk_scheduler.cpp +++ b/src/video_core/renderer_vulkan/vk_scheduler.cpp @@ -4,10 +4,13 @@ // SPDX-FileCopyrightText: Copyright 2019 yuzu Emulator Project // SPDX-License-Identifier: GPL-2.0-or-later +#include +#include #include #include #include #include +#include #include #include #include @@ -147,6 +150,9 @@ Scheduler::Scheduler(const Device& device_, StateTracker& state_tracker_) trace_timestamp_bits, trace_timestamp_period); } } + if (VideoCommon::ReadTrace::Enabled()) { + CreateReadTrace(); + } AcquireNewChunk(); AllocateWorkerCommandBuffer(); @@ -154,16 +160,237 @@ Scheduler::Scheduler(const Device& device_, StateTracker& state_tracker_) } Scheduler::~Scheduler() { - if (trace_query_pool) { + if (trace_query_pool || read_trace) { WaitWorker(); worker_thread.request_stop(); worker_thread.join(); if (trace_gpu_tick != 0) { master_semaphore->Wait(trace_gpu_tick); } + // The read trace's query pools are in every submission. + if (read_trace && CurrentTick() > 1) { + master_semaphore->Wait(CurrentTick() - 1); + } + } +} + +/// TEST (EDEN_NX2_READ_TRACE): device timestamps that place guest reads on the GPU's timeline. +struct Scheduler::ReadTraceState { + /// Submissions timed at their start and end (two queries a slot), by tick. + static constexpr u32 TickSlots = 4096; + /// TraceMark timestamps, by handle. + static constexpr u32 MarkSlots = 4096; + + vk::QueryPool ticks; + vk::QueryPool marks; + /// vkGetCalibratedTimestamps{KHR,EXT}; null without them. + PFN_vkGetCalibratedTimestampsKHR calibrate{}; + double ns_per_tick{}; + u64 next_tick{1}; ///< Worker: the submission its command buffers go in + u64 next_mark{1}; ///< Recording thread + std::atomic issued_marks{}; + std::atomic submitted{}; ///< The last tick vkQueueSubmit returned for + /// When vkQueueSubmit returned (Now()) for the tick in submit_ticks + std::array, TickSlots> submit_ticks{}; + std::array, TickSlots> submit_times{}; +}; + +void Scheduler::CreateReadTrace() { + const auto physical = device.GetPhysical(); + const u32 valid_bits = + physical.GetQueueFamilyProperties()[device.GetGraphicsFamily()].timestampValidBits; + const float period = physical.GetProperties().limits.timestampPeriod; + if (valid_bits == 0 || period <= 0.0f) { + LOG_WARNING(Render_Vulkan, "Read trace: the graphics queue has no timestamps"); + return; + } + auto created = std::make_unique(); + created->ns_per_tick = static_cast(period); + const auto create_pool = [this](u32 count) { + return device.GetLogical().CreateQueryPool({ + .sType = VK_STRUCTURE_TYPE_QUERY_POOL_CREATE_INFO, + .pNext = nullptr, + .flags = 0, + .queryType = VK_QUERY_TYPE_TIMESTAMP, + .queryCount = count, + .pipelineStatistics = 0, + }); + }; + try { + created->ticks = create_pool(ReadTraceState::TickSlots * 2); + created->marks = create_pool(ReadTraceState::MarkSlots); + } catch (const vk::Exception& error) { + LOG_WARNING(Render_Vulkan, "Read trace: no query pools: {}", error.what()); + return; + } + // The device enables either extension with EDEN_NX2_READ_TRACE (vulkan_device.cpp). + const auto& dld = device.GetDispatchLoader(); + const auto& extensions = device.GetAvailableExtensions(); + const bool khr = extensions.contains(VK_KHR_CALIBRATED_TIMESTAMPS_EXTENSION_NAME); + if (khr || extensions.contains(VK_EXT_CALIBRATED_TIMESTAMPS_EXTENSION_NAME)) { + const auto get_domains = + reinterpret_cast( + dld.vkGetInstanceProcAddr(device.GetInstance(), + khr ? "vkGetPhysicalDeviceCalibrateableTimeDomainsKHR" + : "vkGetPhysicalDeviceCalibrateableTimeDomainsEXT")); + const auto calibrate = reinterpret_cast( + dld.vkGetDeviceProcAddr(*device.GetLogical(), khr ? "vkGetCalibratedTimestampsKHR" + : "vkGetCalibratedTimestampsEXT")); + std::array domains{}; + u32 count = static_cast(domains.size()); + if (get_domains != nullptr && calibrate != nullptr && + get_domains(physical, &count, domains.data()) >= VK_SUCCESS && + std::ranges::find(domains.begin(), domains.begin() + (std::min)(count, 8u), + VK_TIME_DOMAIN_DEVICE_KHR) != + domains.begin() + (std::min)(count, 8u)) { + created->calibrate = calibrate; + } + } + LOG_INFO(Render_Vulkan, "Read trace: device timestamps every {} ns, {} bits, {}", period, + valid_bits, + created->calibrate ? "device clock readable (calibrated timestamps)" + : "no calibrated timestamps: reads stay unplaced"); + read_trace = std::move(created); +} + +u64 Scheduler::TraceMark() { + if (!read_trace) { + return 0; + } + const u64 handle = read_trace->next_mark++; + read_trace->issued_marks.store(handle, std::memory_order_release); + RecordWithUploadBuffer( + [pool = *read_trace->marks, slot = static_cast(handle % ReadTraceState::MarkSlots)]( + vk::CommandBuffer cmdbuf, vk::CommandBuffer upload_cmdbuf) { + // The upload buffer runs first in the same submission. + upload_cmdbuf.ResetQueryPool(pool, slot, 1); + cmdbuf.WriteTimestamp(VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT, pool, slot); + }); + return handle; +} + +void Scheduler::TraceRead(VideoCommon::ReadTrace::Read& read, VideoCommon::ReadTrace::Step step) { + namespace ReadTrace = VideoCommon::ReadTrace; + if (!read_trace) { + return; + } + switch (step) { + case ReadTrace::Step::Request: + master_semaphore->Refresh(); + read.completed = master_semaphore->KnownGpuTick(); + read.submitted = read_trace->submitted.load(std::memory_order_acquire); + read.recording = master_semaphore->CurrentTick(); + return; + case ReadTrace::Step::Calibrate: + if (read_trace->calibrate != nullptr) { + const VkCalibratedTimestampInfoKHR info{ + .sType = VK_STRUCTURE_TYPE_CALIBRATED_TIMESTAMP_INFO_KHR, + .pNext = nullptr, + .timeDomain = VK_TIME_DOMAIN_DEVICE_KHR, + }; + u64 value{}; + u64 deviation{}; + const u64 before = ReadTrace::Now(); + if (read_trace->calibrate(*device.GetLogical(), 1, &info, &value, &deviation) == + VK_SUCCESS) { + const u64 after = ReadTrace::Now(); + read.device = value; + read.host = before + (after - before) / 2; + read.uncertainty = after - before; + } + } + return; + case ReadTrace::Step::Done: + ReadTrace::Record(read, PlaceRead(read)); + return; } } +VideoCommon::ReadTrace::Timeline Scheduler::PlaceRead( + const VideoCommon::ReadTrace::Read& read) const { + namespace ReadTrace = VideoCommon::ReadTrace; + if (!read.has_copy) { + return {}; + } + const ReadTraceState& trace = *read_trace; + const u64 copy_tick = read.copy.tick; + struct Result { + u64 value; + u64 available; + }; + const auto& logical = device.GetLogical(); + const auto mark = [&](u64 handle) -> std::optional { + const auto reused = [&] { + return trace.issued_marks.load(std::memory_order_acquire) - handle >= + ReadTraceState::MarkSlots; + }; + Result result{}; + if (handle == 0 || reused() || + logical.GetQueryResults( + *trace.marks, static_cast(handle % ReadTraceState::MarkSlots), 1, + sizeof(result), &result, sizeof(result), + VK_QUERY_RESULT_64_BIT | VK_QUERY_RESULT_WITH_AVAILABILITY_BIT) != VK_SUCCESS || + result.available == 0 || reused()) { + return std::nullopt; + } + return result.value; + }; + // The submissions after the last one done at the request, up to the copy's: all done now. + const auto place = [&]() -> ReadTrace::Timeline { + const u64 first_tick = read.completed + 1; + if (read.device == 0 || !read.has_write || copy_tick < first_tick || + copy_tick - first_tick >= ReadTraceState::TickSlots / 2) { + return {}; + } + const auto write_end = mark(read.write.mark); + const auto copy_start = mark(read.copy.start); + const auto copy_end = mark(read.copy.end); + if (!write_end || !copy_start || !copy_end) { + return {}; + } + const u32 tick_count = static_cast(copy_tick - first_tick + 1); + std::vector results(size_t{tick_count} * 2); + for (u32 done = 0; done < tick_count;) { + const u32 slot = static_cast((first_tick + done) % ReadTraceState::TickSlots); + const u32 count = (std::min)(tick_count - done, ReadTraceState::TickSlots - slot); + if (logical.GetQueryResults( + *trace.ticks, slot * 2, count * 2, size_t{count} * 2 * sizeof(Result), + &results[size_t{done} * 2], sizeof(Result), + VK_QUERY_RESULT_64_BIT | VK_QUERY_RESULT_WITH_AVAILABILITY_BIT) != VK_SUCCESS) { + return {}; + } + done += count; + } + std::vector> submissions; + submissions.reserve(tick_count); + for (u32 i = 0; i < tick_count; ++i) { + const Result& start = results[size_t{i} * 2]; + const Result& end = results[size_t{i} * 2 + 1]; + if (start.available == 0 || end.available == 0) { + return {}; + } + submissions.emplace_back(start.value, end.value); + } + return ReadTrace::Place(read, ReadTrace::DeviceTimes{ + .ns_per_tick = trace.ns_per_tick, + .write_end = *write_end, + .copy_start = *copy_start, + .copy_end = *copy_end, + .submissions = submissions, + }); + }; + ReadTrace::Timeline timeline = place(); + const u32 copy_slot = static_cast(copy_tick % ReadTraceState::TickSlots); + if (trace.submit_ticks[copy_slot].load(std::memory_order_acquire) == copy_tick) { + const u64 submitted_at = trace.submit_times[copy_slot].load(std::memory_order_relaxed); + if (submitted_at >= read.copy.flushed) { + timeline.has_submit = true; + timeline.submit = submitted_at - read.copy.flushed; + } + } + return timeline; +} + u64 Scheduler::Flush(VkSemaphore signal_semaphore, VkSemaphore wait_semaphore, std::source_location location) { if (state.renderpass) { @@ -307,6 +534,7 @@ void Scheduler::BeginRenderPassImpl(const Framebuffer* framebuffer, VkRenderPass if (VideoCommon::Nx2Profile::Enabled()) { g_profile_renderpasses.fetch_add(1, std::memory_order_relaxed); } + ++trace_counts.render_passes; const VkFramebuffer framebuffer_handle = framebuffer->Handle(); const VkExtent2D render_area = framebuffer->RenderArea(); state.renderpass = renderpass; @@ -575,6 +803,13 @@ void Scheduler::AllocateWorkerCommandBuffer() { .flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT, .pInheritanceInfo = nullptr, }); + if (read_trace) { + // When the host GPU starts the submission these go in (its end: SubmitExecution). + const u32 query = static_cast(read_trace->next_tick % ReadTraceState::TickSlots) * 2; + current_upload_cmdbuf.ResetQueryPool(*read_trace->ticks, query, 2); + current_upload_cmdbuf.WriteTimestamp(VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, *read_trace->ticks, + query); + } if (trace_query_pool && !trace_gpu_failed) { TraceGpuSubmission(); } @@ -641,6 +876,17 @@ void Scheduler::ReportWorkTimes() { if (result != VK_SUCCESS) { return; } + if (read_trace && count < TraceMaxWork) { + // Every draw and dispatch of the frame was timed: the reads placed in it split by them. + std::vector> work; + work.reserve(count); + for (u32 i = 0; i < count; ++i) { + if (results[i * 2].available != 0 && results[i * 2 + 1].available != 0) { + work.emplace_back(results[i * 2].timestamp, results[i * 2 + 1].timestamp); + } + } + VideoCommon::ReadTrace::NoteTimedWork(work, read_trace->ns_per_tick); + } struct Totals { double total_ms{}; double max_ms{}; @@ -810,6 +1056,10 @@ u64 Scheduler::SubmitExecution(VkSemaphore signal_semaphore, VkSemaphore wait_se if (trace_query_pool) { trace_work = {}; } + if (read_trace) { + const u32 query = static_cast(signal_value % ReadTraceState::TickSlots) * 2 + 1; + cmdbuf.WriteTimestamp(VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT, *read_trace->ticks, query); + } cmdbuf.End(); if (on_submit) { @@ -834,6 +1084,14 @@ u64 Scheduler::SubmitExecution(VkSemaphore signal_semaphore, VkSemaphore wait_se vk::Check(result); break; } + if (read_trace) { + const u32 slot = static_cast(signal_value % ReadTraceState::TickSlots); + read_trace->submit_times[slot].store(VideoCommon::ReadTrace::Now(), + std::memory_order_relaxed); + read_trace->submit_ticks[slot].store(signal_value, std::memory_order_release); + read_trace->submitted.store(signal_value, std::memory_order_release); + read_trace->next_tick = signal_value + 1; + } if (trace_frame_end != 0) { if (trace_gpu_sampling) { trace_gpu_pending = true; diff --git a/src/video_core/renderer_vulkan/vk_scheduler.h b/src/video_core/renderer_vulkan/vk_scheduler.h index f97705184a..44aaa5e5f7 100644 --- a/src/video_core/renderer_vulkan/vk_scheduler.h +++ b/src/video_core/renderer_vulkan/vk_scheduler.h @@ -25,6 +25,7 @@ #include "common/polyfill_thread.h" #include "common/settings.h" #include "video_core/gpu_split.h" +#include "video_core/nx2_read_trace.h" #include "video_core/renderer_vulkan/fragment_barrier_state.h" #include "video_core/renderer_vulkan/vk_master_semaphore.h" #include "video_core/vulkan_common/vulkan_wrapper.h" @@ -98,6 +99,19 @@ class Scheduler { cmdbuf.WriteTimestamp(VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT, *trace_work_pool, query + 1); } + /// TEST (EDEN_NX2_READ_TRACE, nx2_read_trace.h), recording thread: a device timestamp written + /// once the work recorded so far is done. Returns its handle for TraceRead, 0 when off. + u64 TraceMark(); + /// Recording thread: the draws, dispatches and render passes recorded so far. + [[nodiscard]] VideoCommon::ReadTrace::Counts TraceCounts() const noexcept { + return trace_counts; + } + void TraceCountWork(bool dispatch) noexcept { + ++(dispatch ? trace_counts.dispatches : trace_counts.draws); + } + /// A reading thread's step of a guest read that waits. + void TraceRead(VideoCommon::ReadTrace::Read& read, VideoCommon::ReadTrace::Step step); + /// Sends currently recorded work to the worker thread. void DispatchWork(); @@ -432,6 +446,11 @@ class Scheduler { void ReportWorkTimes(); void TraceShader(u64 hash, u32 stage); + struct ReadTraceState; + void CreateReadTrace(); + [[nodiscard]] VideoCommon::ReadTrace::Timeline PlaceRead( + const VideoCommon::ReadTrace::Read& read) const; + u64 SubmitExecution(VkSemaphore signal_semaphore, VkSemaphore wait_semaphore); void AllocateNewContext(); @@ -491,6 +510,10 @@ class Scheduler { bool trace_gpu_pending{}; bool trace_gpu_failed{}; + /// TEST: EDEN_NX2_READ_TRACE + std::unique_ptr read_trace; + VideoCommon::ReadTrace::Counts trace_counts{}; + VideoCommon::QueryCacheBase* query_cache = nullptr; vk::CommandBuffer current_cmdbuf; diff --git a/src/video_core/video_core.cpp b/src/video_core/video_core.cpp index 734522e708..2dbfd80646 100644 --- a/src/video_core/video_core.cpp +++ b/src/video_core/video_core.cpp @@ -17,6 +17,7 @@ #include "video_core/nx2_download_trace.h" #include "video_core/nx2_drain_check.h" #include "video_core/nx2_profile.h" +#include "video_core/nx2_read_trace.h" #include "video_core/renderer_base.h" #include "video_core/renderer_null/renderer_null.h" #ifdef HAS_OPENGL @@ -56,6 +57,7 @@ void CreateGPU(std::optional& gpu, Core::Frontend::EmuWindow& emu_wi Tegra::Nx2CmdPageDiag::ReadSwitch(); VideoCommon::Nx2DownloadTrace::ReadSwitch(); VideoCommon::Nx2DrainCheck::ReadSwitch(); + VideoCommon::ReadTrace::ReadSwitch(); const auto nvdec_value = Settings::values.nvdec_emulation.GetValue(); const bool use_nvdec = nvdec_value != Settings::NvdecEmulation::Off; diff --git a/src/video_core/vulkan_common/vulkan_device.cpp b/src/video_core/vulkan_common/vulkan_device.cpp index b2c5468a8f..60f454280d 100644 --- a/src/video_core/vulkan_common/vulkan_device.cpp +++ b/src/video_core/vulkan_common/vulkan_device.cpp @@ -27,6 +27,7 @@ #include #include "common/settings.h" #include "common/settings_enums.h" +#include "video_core/nx2_read_trace.h" #include "video_core/vulkan_common/nsight_aftermath_tracker.h" #include "video_core/vulkan_common/vma.h" #include "video_core/vulkan_common/vulkan_device.h" @@ -1221,6 +1222,15 @@ bool Device::GetSuitability(bool requires_swapchain) { FOR_EACH_VK_FEATURE_EXT(FEATURE_EXTENSION); FOR_EACH_VK_EXTENSION(EXTENSION); + // TEST (EDEN_NX2_READ_TRACE): the device's clock, read from the host (Scheduler::TraceRead). + if (VideoCommon::ReadTrace::Enabled()) { + if (supported_extensions.contains(VK_KHR_CALIBRATED_TIMESTAMPS_EXTENSION_NAME)) { + loaded_extensions.insert(VK_KHR_CALIBRATED_TIMESTAMPS_EXTENSION_NAME); + } else if (supported_extensions.contains(VK_EXT_CALIBRATED_TIMESTAMPS_EXTENSION_NAME)) { + loaded_extensions.insert(VK_EXT_CALIBRATED_TIMESTAMPS_EXTENSION_NAME); + } + } + extensions.depth_stencil_resolve = extensions.depth_stencil_resolve && (instance_version >= VK_API_VERSION_1_2 || extensions.create_renderpass2); From 739a6ed95c904abed382885bd5f5886a74336c07 Mon Sep 17 00:00:00 2001 From: jpkto <255363096+jpkto@users.noreply.github.com> Date: Sun, 8 Nov 2026 00:06:17 +0100 Subject: [PATCH 023/105] vulkan: cast the LSFG mip level binding count to u32 LsfgBindings holds u32 binding counts and LSFG_MIP_LEVELS is a size_t, so the pair built from it narrows implicitly. MSVC reported that as C4267, a warning, until the read trace header that vk_scheduler.h includes in this test build turned the same instantiation into C4244, which the build treats as an error (/we4244). Make the conversion explicit. Co-Authored-By: Claude Opus 5.5 --- src/video_core/renderer_vulkan/present/lsfg_mipmaps.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/video_core/renderer_vulkan/present/lsfg_mipmaps.cpp b/src/video_core/renderer_vulkan/present/lsfg_mipmaps.cpp index 6454ca0480..3455557d1d 100644 --- a/src/video_core/renderer_vulkan/present/lsfg_mipmaps.cpp +++ b/src/video_core/renderer_vulkan/present/lsfg_mipmaps.cpp @@ -36,7 +36,7 @@ LsfgMipmaps::LsfgMipmaps(const Device& device, MemoryAllocator& memory_allocator {{1, VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER}, {1, VK_DESCRIPTOR_TYPE_SAMPLER}, {1, VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE}, - {LSFG_MIP_LEVELS, VK_DESCRIPTOR_TYPE_STORAGE_IMAGE}}); + {static_cast(LSFG_MIP_LEVELS), VK_DESCRIPTOR_TYPE_STORAGE_IMAGE}}); const VkExtent2D input_extent = (*frames)[0].Extent(); flow_extent = VkExtent2D{ From a442c38fe67b75d2deb4844c7e2d400fc44b7817 Mon Sep 17 00:00:00 2001 From: jpkto <255363096+jpkto@users.noreply.github.com> Date: Sun, 8 Nov 2026 00:12:54 +0100 Subject: [PATCH 024/105] video_core: run the macros that read memory on the split's back Ocarina of Time's render enable macro (0xe2c, code at instruction 0x142) reads the six words of a report pair with MME_DMA_READ_FIFOED and writes the render enable mode, about 25,000 times a second in Hyrule Field. On the split's front each read waits for the queued work that can write the words (FrontDmaRead): 25,060 waits a second, 252.8 ms of every second of a front that is 99.9% busy (test46b, No sharpening; the back is 77-81% busy). The back holds what the read needs at the call's point of the method stream, so such macros now run there. A static pass over the macro code (TuringMacroInterpreter::AnalyzeEffects, macro_turing.h:266) follows a call from its start along every path, each branch either way and each loop any number of times, with the register, carry and method values the code fixes: it finds the methods the call can write (as long as each is a constant of the code) and whether it uses the data RAM. A macro whose code reads memory into its FIFO, uses no data RAM and writes only registers the front can do without the value of (BackWritable, maxwell_3d.cpp:1200) goes to the back (FrontRunOnBack, line 1562): its call is one entry with its parameters (GpuSplit::DeferWords), and the back runs it with a macro engine of its own, its code copied from the front's, where ProcessMacro would. It reads the words as a drained front did. Not back-writable: macro code and bindings, the shadow RAM mode, draws, constant buffer data, reports, uploads and syncpoints (the front tracks what they do), the registers the front reads to note their memory writes, registers the back changes itself (the bind pointer, shadow_scratch[0], logic_op.enable on AMD) and MME_DMA_READ, whose words land in the front's data RAM. The registers such a macro can write, and in a tracked shadow RAM mode their entries, are then the back's until the front knows them again: a front write to one is sent even if the front's copies hold the value, a replay first takes the entry from the back (FrontTakeOver, line 1699), and a front macro or DMA read that reads one waits for the back to run the macro and takes the back's value (FrontRegister, line 1685). No entry after that macro writes the register, so the value can be read while the back runs on. The waits count as "back register", and the macro code changing under queued back calls as "back code". The back counts writes outside the predicted set; the front then runs that macro itself from then on and takes the back's registers as SPLIT_VERIFY does. Macros that don't qualify keep the narrower wait (MKW's read into data RAM word 0x27). EDEN_NX2_BACK_MACROS=0 turns this off. turing_macro_effects.cpp checks the analysis on hand-made code and against the interpreter on random code. The split DMA read test now also runs a macro on the back: the front goes on past queued work and the back reads the word written behind it, a write of the value the front's stale copy holds is sent, a front macro that reads the register waits and gets the back's value, and a replay takes the shadow RAM entry from the back. Co-Authored-By: Claude Opus 5.5 --- src/tests/CMakeLists.txt | 1 + src/tests/video_core/maxwell_fast_method.cpp | 130 ++++++-- src/tests/video_core/turing_macro_effects.cpp | 188 ++++++++++++ src/video_core/engines/maxwell_3d.cpp | 290 +++++++++++++++++- src/video_core/engines/maxwell_3d.h | 4 + src/video_core/gpu_split.h | 42 +++ src/video_core/macro.cpp | 7 +- src/video_core/macro.h | 7 +- src/video_core/macro_turing.h | 266 ++++++++++++++++ 9 files changed, 894 insertions(+), 41 deletions(-) create mode 100644 src/tests/video_core/turing_macro_effects.cpp diff --git a/src/tests/CMakeLists.txt b/src/tests/CMakeLists.txt index f10f56799d..6d97c0c1e7 100644 --- a/src/tests/CMakeLists.txt +++ b/src/tests/CMakeLists.txt @@ -158,6 +158,7 @@ add_executable(tests video_core/ampere_uniform_immediate_shift.cpp video_core/ampere_bindless_surfaces.cpp video_core/native_macro_fifo.cpp + video_core/turing_macro_effects.cpp video_core/turing_macro_ram.cpp video_core/uniform_stream_reuse.cpp video_core/usage_tracker.cpp diff --git a/src/tests/video_core/maxwell_fast_method.cpp b/src/tests/video_core/maxwell_fast_method.cpp index 1cf786d4bf..582ebb3bfd 100644 --- a/src/tests/video_core/maxwell_fast_method.cpp +++ b/src/tests/video_core/maxwell_fast_method.cpp @@ -216,27 +216,40 @@ u32 RandomArgument(std::mt19937& rng, u32 method, const Maxwell3D& engine) { return rng() % 64; } -/// A Turing MME instruction that sends the next macro parameter to `method` (as in -/// native_macro_fifo.cpp); `end` makes the instruction after it the macro's last. -std::array FifoInstruction(u32 method, bool end) { +/// A Turing MME instruction from its fields (TuringMacroInterpreter::Decode); a lane left alone +/// adds the zero register to itself. +struct Mme { + bool end{}; + std::array op{}, dst{24, 24}, src_a{24, 24}, src_b{24, 24}, imm{}; + std::array out_method{}, out_emit{}; +}; + +std::array Encode(const Mme& fields) { std::array bits{}; - const auto field = [&](u32 offset, u32 count, u32 value) { + const auto field = [&bits](u32 offset, u32 count, u32 value) { for (u32 bit = 0; bit < count; ++bit) { bits[(offset + bit) / 32] |= ((value >> bit) & 1) << ((offset + bit) % 32); } }; - field(0, 1, end); + field(0, 1, fields.end ? 1 : 0); for (u32 lane = 0; lane < 2; ++lane) { - field(15 + lane * 36, 5, 24); - field(20 + lane * 36, 5, 24); - field(25 + lane * 36, 5, 24); + field(10 + lane * 36, 5, fields.op[lane]); + field(15 + lane * 36, 5, fields.dst[lane]); + field(20 + lane * 36, 5, fields.src_a[lane]); + field(25 + lane * 36, 5, fields.src_b[lane]); + field(30 + lane * 36, 16, fields.imm[lane]); + field(82 + lane * 7, 3, fields.out_method[lane]); + field(85 + lane * 7, 4, fields.out_emit[lane]); } - field(30, 16, method); - field(82, 3, 5); - field(85, 4, 3); return {bits[2], bits[1], bits[0]}; } +/// A Turing MME instruction that sends the next macro parameter to `method` (as in +/// native_macro_fifo.cpp); `end` makes the instruction after it the macro's last. +std::array FifoInstruction(u32 method, bool end) { + return Encode(Mme{.end = end, .imm = {method, 0}, .out_method = {5, 0}, .out_emit = {3, 0}}); +} + } // Anonymous namespace TEST_CASE("Maxwell3D fast method dispatch matches CallMethod on random method streams", @@ -1096,7 +1109,8 @@ TEST_CASE("Maxwell3D's split front uploads macro code sent as one multi-method r CHECK(std::equal(code.begin(), code.end(), macros.native_code.begin() + pointer * 3)); } -TEST_CASE("Split macro DMA reads wait only for the queued work that can write the words", +TEST_CASE("Split macro DMA reads wait only for the queued work that can write the words, or run " + "on the back", "[video_core][gpu_split_dependencies]") { using Access = VideoCommon::GpuSplitTestAccess; using Writes = VideoCommon::GpuSplit::Writes; @@ -1157,16 +1171,19 @@ TEST_CASE("Split macro DMA reads wait only for the queued work that can write th // The 3D engine's front, against a running back: macro 0 reads one word with // MME_DMA_READ_FIFOED at the address in its first two parameters and sends it to a scratch - // register (the load after the read takes the word). + // register (the load after the read takes the word). It also clears a data RAM word, which + // keeps it on the front. Macro 1 is the same without that and runs on the back; macro 2 (on + // the front) copies the scratch register to the next one. auto& engine = f.fast; engine.SetNativeTuringMacroMode(true); constexpr u32 state = MAXWELL3D_REG_INDEX(shadow_scratch) + 1; + constexpr u32 copy = state + 1; VideoCommon::GpuSplit split; Access::Start(split, f.system); SCOPE_EXIT { split.Stop(); }; - std::array, 3> release{}; + std::array, 5> release{}; std::atomic done{}; std::thread reader; SCOPE_EXIT { @@ -1181,8 +1198,20 @@ TEST_CASE("Split macro DMA reads wait only for the queued work that can write th engine.FrontCallMethod(f.system, split, method, argument, true); }; std::vector code; - for (const auto instruction : {FifoInstruction(0x154, false), FifoInstruction(0x155, false), - FifoInstruction(0x158, true), FifoInstruction(state, false)}) { + const Mme clear_data_ram{ + .op = {0, 31}, .imm = {state, 0}, .out_method = {5, 0}, .out_emit = {3, 0}}; + // Macro 2: the value of `state` (MME register read) sent to `copy`, then a no-op. + const Mme read_state{.end = true, + .op = {22, 0}, + .src_a = {25, 24}, + .imm = {state, copy}, + .out_method = {6, 0}, + .out_emit = {1, 0}}; + for (const auto instruction : + {FifoInstruction(0x154, false), FifoInstruction(0x155, false), + FifoInstruction(0x158, true), Encode(clear_data_ram), FifoInstruction(0x154, false), + FifoInstruction(0x155, false), FifoInstruction(0x158, true), + FifoInstruction(state, false), Encode(read_state), Encode(Mme{})}) { code.insert(code.end(), instruction.begin(), instruction.end()); } front(MAXWELL3D_REG_INDEX(load_mme.instruction_ptr), 0); @@ -1190,16 +1219,25 @@ TEST_CASE("Split macro DMA reads wait only for the queued work that can write th code.data(), static_cast(code.size()), static_cast(code.size())); front(MAXWELL3D_REG_INDEX(load_mme.start_address_ptr), 0); - front(MAXWELL3D_REG_INDEX(load_mme.start_address), 0); - const auto dma_read = [&](GPUVAddr address) { - engine.FrontCallMethod(f.system, split, 0xe00, static_cast(address >> 32), false); - engine.FrontCallMethod(f.system, split, 0xe01, static_cast(address), false); - engine.FrontCallMethod(f.system, split, 0xe01, 1, true); + for (const u32 start : {0U, 4U, 8U}) { + front(MAXWELL3D_REG_INDEX(load_mme.start_address), start); + } + const auto dma_read = [&](GPUVAddr address, u32 macro = 0xe00) { + engine.FrontCallMethod(f.system, split, macro, static_cast(address >> 32), false); + engine.FrontCallMethod(f.system, split, macro + 1, static_cast(address), false); + engine.FrontCallMethod(f.system, split, macro + 1, 1, true); }; - const auto read_on_reader = [&](GPUVAddr address) { + const auto read_on_reader = [&](GPUVAddr address, u32 macro = 0xe00) { done = false; - reader = std::thread([&, address] { - dma_read(address); + reader = std::thread([&, address, macro] { + dma_read(address, macro); + done = true; + }); + }; + const auto copy_on_reader = [&] { + done = false; + reader = std::thread([&] { + front(0xe04, 0); done = true; }); }; @@ -1277,4 +1315,48 @@ TEST_CASE("Split macro DMA reads wait only for the queued work that can write th reader.join(); split.Drain(VideoCommon::DrainSite::Other); CHECK(engine.regs.reg_array[state] == 0x33333333); + + // Macro 1 runs on the back: the front goes on past that work, and the back reads the word + // after the upload queued behind it. + front(state, 0x77777777); + blocking(release[3], Writes::Anywhere); + upload(word, 0x55555555); + read_on_reader(word, 0xe02); + REQUIRE(finishes()); + reader.join(); + release[3] = true; + split.Drain(VideoCommon::DrainSite::Other); + CHECK(engine.regs.reg_array[state] == 0x55555555); + + // The front's copy of the register is stale: a write of the value it holds is still sent + // (the back sinks it until the next macro). + front(state, 0x77777777); + front(0xe04, 0); + split.Drain(VideoCommon::DrainSite::Other); + CHECK(engine.regs.reg_array[state] == 0x77777777); + + // A front macro that reads the register waits for the back's macro and gets its value. + blocking(release[4], Writes::Noted); + upload(word, 0x66666666); + dma_read(word, 0xe02); + copy_on_reader(); + REQUIRE(waits()); + release[4] = true; + REQUIRE(finishes()); + reader.join(); + split.Drain(VideoCommon::DrainSite::Other); + CHECK(engine.regs.reg_array[copy] == 0x66666666); + + // A replay takes the shadow RAM entry the back's macro tracked from the back. + using Control = Maxwell3D::Regs::ShadowRamControl; + front(state, 0x99999999); + upload(word, 0x88888888); + dma_read(word, 0xe02); + front(ShadowControl, static_cast(Control::Replay)); + front(state, 0); + front(ShadowControl, static_cast(Control::Track)); + front(0xe04, 0); + split.Drain(VideoCommon::DrainSite::Other); + CHECK(engine.regs.reg_array[state] == 0x88888888); + CHECK(engine.regs.reg_array[copy] == 0x88888888); } diff --git a/src/tests/video_core/turing_macro_effects.cpp b/src/tests/video_core/turing_macro_effects.cpp new file mode 100644 index 0000000000..75e1dd2f9e --- /dev/null +++ b/src/tests/video_core/turing_macro_effects.cpp @@ -0,0 +1,188 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later + +#include +#include +#include +#include +#include +#include +#include + +#include "common/common_types.h" +#include "video_core/macro_turing.h" + +namespace { + +using Effects = Tegra::TuringMacroInterpreter::Effects; + +/// A Turing MME instruction from its fields (TuringMacroInterpreter::Decode); a lane left alone +/// adds the zero register to itself. +struct Mme { + bool end{}; + u32 pred_mode{}; + u32 pred_reg{}; + std::array op{}, dst{24, 24}, src_a{24, 24}, src_b{24, 24}, imm{}; + std::array out_method{}, out_emit{}; +}; + +std::vector Code(std::span instructions) { + std::vector code; + for (const Mme& f : instructions) { + std::array bits{}; + const auto field = [&bits](u32 offset, u32 count, u32 value) { + for (u32 bit = 0; bit < count; ++bit) { + bits[(offset + bit) / 32] |= ((value >> bit) & 1) << ((offset + bit) % 32); + } + }; + field(0, 1, f.end ? 1 : 0); + field(1, 4, f.pred_mode); + field(5, 5, f.pred_reg); + for (u32 lane = 0; lane < 2; ++lane) { + field(10 + lane * 36, 5, f.op[lane]); + field(15 + lane * 36, 5, f.dst[lane]); + field(20 + lane * 36, 5, f.src_a[lane]); + field(25 + lane * 36, 5, f.src_b[lane]); + field(30 + lane * 36, 16, f.imm[lane]); + field(82 + lane * 7, 3, f.out_method[lane]); + field(85 + lane * 7, 4, f.out_emit[lane]); + } + code.insert(code.end(), {bits[2], bits[1], bits[0]}); + } + return code; +} + +/// Sends the next parameter to `method` (an immediate with its increment in bits 12-15). +Mme Fifo(u32 method, bool end = false) { + return Mme{.end = end, .imm = {method, 0}, .out_method = {5, 0}, .out_emit = {3, 0}}; +} + +Effects Analyze(std::initializer_list instructions, u32 start = 0) { + return Tegra::TuringMacroInterpreter::AnalyzeEffects( + Code(std::span{instructions.begin(), instructions.size()}), start); +} + +} // Anonymous namespace + +TEST_CASE("Turing macro effects: a FIFO read writes its registers, up to the one after END", + "[video_core][macro]") { + // Ocarina of Time's render enable macro reads six words at the address in its parameters. + const auto effects = Analyze({Fifo(0x154), Fifo(0x155), Fifo(0x158, true), Fifo(0x556), + Fifo(0x557)}); + CHECK(effects.methods_known); + CHECK_FALSE(effects.data_ram); + CHECK(effects.methods == std::vector{0x154, 0x155, 0x158, 0x556}); + // The same code entered past its END: the call runs to the end of the code. + CHECK(Analyze({Fifo(0x154), Fifo(0x155), Fifo(0x158, true), Fifo(0x556), Fifo(0x557)}, 3) + .methods == std::vector{0x556, 0x557}); +} + +TEST_CASE("Turing macro effects: increments, branches and predicates", "[video_core][macro]") { + // One method with an increment of 1, written twice: both registers. + const Mme twice{.end = true, + .imm = {0x1100, 0}, + .out_method = {5, 0}, + .out_emit = {3, 3}}; + CHECK(Analyze({twice, Fifo(0x200)}).methods == std::vector{0x100, 0x101, 0x200}); + // r0 = parameter; a branch on r0 == 0 skips the write of 0x300: both ways count. + const Mme load{.dst = {0, 24}, .src_a = {28, 24}}; + const Mme skip{.op = {29, 0}, .src_a = {0, 24}, .imm = {0x8000 | 2, 0}}; + const auto branched = Analyze({load, skip, Fifo(0x300), Fifo(0x301, true), Fifo(0x302)}); + CHECK(branched.methods == std::vector{0x300, 0x301, 0x302}); + // A predicate on the zero register: FFFF runs nothing of the instruction (its END stays). + Mme predicated = Fifo(0x400, true); + predicated.pred_mode = 2; + predicated.pred_reg = 24; + CHECK(Analyze({predicated, Fifo(0x401), Fifo(0x402)}).methods == std::vector{0x401}); +} + +TEST_CASE("Turing macro effects: what keeps a macro's writes unknown", "[video_core][macro]") { + // A method taken from a parameter. + const Mme from_fifo{.end = true, .out_method = {3, 0}, .out_emit = {4, 0}}; + CHECK_FALSE(Analyze({from_fifo, Fifo(0x100)}).methods_known); + // A loop over the parameters writing consecutive registers (increment 1). + const Mme count{.dst = {0, 24}, .src_a = {28, 24}}; + const Mme method{.imm = {0x1100, 0}, .out_method = {5, 0}}; + const Mme loop{.op = {23, 0}, .src_a = {0, 24}, .imm = {2, 0}}; + const Mme body{.out_emit = {3, 0}}; + CHECK_FALSE(Analyze({count, method, loop, body, Fifo(0x200, true), Fifo(0x201)}).methods_known); + // The same loop without the increment writes one register. + const Mme same{.imm = {0x100, 0}, .out_method = {5, 0}}; + const auto looped = Analyze({count, same, loop, body, Fifo(0x200, true), Fifo(0x201)}); + CHECK(looped.methods_known); + CHECK(looped.methods == std::vector{0x100, 0x200, 0x201}); + // A data RAM write (DWRITE) in the delay slot after END. + const Mme dwrite{.op = {31, 0}}; + CHECK(Analyze({Fifo(0x158, true), dwrite}).data_ram); + CHECK_FALSE(Analyze({Fifo(0x158, true), Fifo(0x100), dwrite}).data_ram); +} + +TEST_CASE("Turing macro effects cover every write Execute makes on random code", + "[video_core][macro]") { + std::mt19937 rng{7}; + const auto pick = [&rng](u32 count) { return static_cast(rng() % count); }; + for (u32 program = 0; program < 300; ++program) { + // Registers 0-3 set first, then ALU, register read, flow and data RAM instructions with + // methods from immediates mostly. + std::vector instructions(2); + for (u32 i = 0; i < 4; ++i) { + instructions[i / 2].dst[i % 2] = i; + instructions[i / 2].src_a[i % 2] = pick(2) != 0 ? 25 : 28 + i % 2; + instructions[i / 2].imm[i % 2] = pick(4); + } + const u32 length = 2 + pick(20); + for (u32 pc = 0; pc < length; ++pc) { + Mme f{.end = pick(10) == 0}; + if (pick(6) == 0) { + f.pred_mode = pick(16); + f.pred_reg = pick(5) == 0 ? 24 : pick(4); + } + for (u32 lane = 0; lane < 2; ++lane) { + constexpr std::array ops{0, 2, 9, 12, 14, 16, 17, 22, 23, 24, 25, 31}; + f.op[lane] = ops[pick(static_cast(ops.size()))]; + f.dst[lane] = pick(3) == 0 ? 24 : pick(4); + f.src_a[lane] = pick(2) != 0 ? pick(4) : 24 + pick(6); + f.src_b[lane] = pick(2) != 0 ? pick(4) : 24 + pick(6); + if (f.op[lane] == 23) { + f.imm[lane] = 2 + pick(4); + } else if (f.op[lane] == 24 || f.op[lane] == 25) { + // Short jumps and branches either way, some stopping the call. + const int offset = pick(8) == 0 ? -4096 : static_cast(pick(9)) - 4; + f.imm[lane] = (static_cast(offset) & 0x1fff) | (pick(2) << 15) | + (pick(2) << 14); + } else { + f.imm[lane] = pick(3) == 0 ? 0x100 + pick(8) + (pick(3) << 12) : pick(0x10000); + } + f.out_method[lane] = pick(3) == 0 ? 0 : (pick(4) == 0 ? 1 + pick(4) : 5 + lane); + f.out_emit[lane] = pick(3) == 0 ? 0 : (pick(3) == 0 ? 1 + pick(2) : 3 + lane); + } + instructions.push_back(f); + } + const auto code = Code(instructions); + const u32 start = pick(4) == 0 ? pick(length + 2) : 0; + const auto effects = Tegra::TuringMacroInterpreter::AnalyzeEffects(code, start); + for (u32 run = 0; run < 4; ++run) { + Tegra::TuringMacroInterpreter interpreter; + std::vector fifo(4 + pick(28)); + std::ranges::generate(fifo, [&] { return pick(16) == 0 ? rng() : pick(8); }); + std::vector written; + interpreter.Execute( + code, start, fifo, [&](u32) { return pick(8); }, + [&](u32 method, u32) { + written.push_back(method); + return written.size() < 512; + }); + CAPTURE(program, run); + if (effects.methods_known) { + for (const u32 method : written) { + CAPTURE(method); + CHECK(std::ranges::binary_search(effects.methods, method)); + } + } + if (!effects.data_ram) { + CHECK(interpreter.data_ram_reads.empty()); + CHECK(interpreter.data_ram_writes.empty()); + } + } + } +} diff --git a/src/video_core/engines/maxwell_3d.cpp b/src/video_core/engines/maxwell_3d.cpp index 90f390e03d..480f503bbf 100644 --- a/src/video_core/engines/maxwell_3d.cpp +++ b/src/video_core/engines/maxwell_3d.cpp @@ -1030,7 +1030,9 @@ void Maxwell3D::SetHLEReplacementAttributeType(u32 bank, u32 offset, // what macros read: the registers as the back thread will hold them when it reaches the same // point of the method stream. That is every write the front sends, with the shadow RAM applied, // plus the few changes the back makes on its own (constant buffer offset, shadow_scratch[0] on -// falcon[4], the macro bind pointer); the front loads and binds the macro code itself. +// falcon[4], the macro bind pointer); the front loads and binds the macro code itself. Macros +// that read memory run on the back instead (FrontRunOnBack), and the registers they can write +// are the back's until the front writes them or reads them back (FrontRegister). struct Maxwell3D::FrontState { std::array control_writes{}; u32 control_count{}; @@ -1046,6 +1048,29 @@ struct Maxwell3D::FrontState { bool skip_same{}; ///< SkipSameValueWrites() && FrontSkipSame(), read once /// The back's draw manager is in an instance loop (DrawMode::Instance): draw.end does nothing. bool instance_loop{}; + + /// Registers a macro the back ran may have written (FrontRunOnBack), and their shadow RAM + /// entries: the ring position after the last such macro, 0 once the front knows the value. + std::array back_regs{}; + std::array back_shadow{}; + std::bitset back_owned; ///< back_regs or back_shadow set + struct BackMacro { + u32 start{}; + bool back{}; ///< Runs on the back (else on the front) + std::vector writes; ///< The registers its code can write + std::bitset write_set; ///< The same, for the back's check + std::atomic escaped{}; ///< The back saw it write another register + }; + /// By start in the macro code of back_generation, once called. + std::vector> back_macros; + u64 back_generation{~u64{0}}; + std::vector back_demoted; ///< Starts of macros that wrote outside their write set + /// The back's macro engine, its code a copy of the front's (when back_code_copied). + std::unique_ptr back_engine; + bool back_code_copied{}; + u64 back_last{}; ///< The position after the last macro sent to the back + std::atomic back_escapes{}; ///< Back macros that wrote outside their write set + u32 back_escapes_seen{}; }; namespace { @@ -1148,6 +1173,53 @@ bool FrontSkipSame() { return skip; } +// EDEN_NX2_BACK_MACROS=0: macros that read memory run on the front too (FrontRunOnBack). +bool BackMacrosEnabled() { + static const bool enabled = [] { + const char* value = std::getenv("EDEN_NX2_BACK_MACROS"); + return value == nullptr || value[0] != '0'; + }(); + return enabled; +} + +// Whether a macro the split's back runs may write `method` (FrontRunOnBack): the front then only +// loses track of the register's value. Not the methods with effects the front tracks (macro code +// and bindings, the shadow RAM mode, draws, constant buffer data, reports, uploads, syncpoints), +// nor the registers it reads to note their memory writes, nor registers the back changes itself, +// nor MME_DMA_READ, whose words land in the front's data RAM. +bool BackWritable(const Maxwell3D& maxwell3d, u32 method) { + constexpr u32 const_buffer = MAXWELL3D_REG_INDEX(const_buffer); + constexpr u32 report = MAXWELL3D_REG_INDEX(report_semaphore); + constexpr u32 upload = MAXWELL3D_REG_INDEX(upload); + constexpr u32 upload_end = MAXWELL3D_REG_INDEX(launch_dma); + if (method - const_buffer < 4 || method - report < 4 || method - upload < upload_end - upload) { + return false; + } + switch (method) { + case MAXWELL3D_REG_INDEX(load_mme.start_address_ptr): // Advanced by load_mme.start_address + case MAXWELL3D_REG_INDEX(shadow_scratch): // Set by falcon[4] + case MAXWELL3D_REG_INDEX(logic_op.enable): // Cleared by draws on AMD + case 0x157: + return false; + case Maxwell3D::RootTables::SelectMethod: + case Maxwell3D::RootTables::LoadMethod: + case MAXWELL3D_REG_INDEX(wait_for_idle): + case MAXWELL3D_REG_INDEX(bind_groups[0].raw_config): + case MAXWELL3D_REG_INDEX(bind_groups[1].raw_config): + case MAXWELL3D_REG_INDEX(bind_groups[2].raw_config): + case MAXWELL3D_REG_INDEX(bind_groups[3].raw_config): + case MAXWELL3D_REG_INDEX(bind_groups[4].raw_config): + case MAXWELL3D_REG_INDEX(render_enable.mode): + case MAXWELL3D_REG_INDEX(clear_report_value): + case MAXWELL3D_REG_INDEX(fragment_barrier): + case MAXWELL3D_REG_INDEX(invalidate_texture_data_cache): + case MAXWELL3D_REG_INDEX(tiled_cache_barrier): + return true; // Executable, with effects on the back only + default: + return method < MacroRegistersStart && !maxwell3d.execution_mask[method]; + } +} + } // Anonymous namespace Maxwell3D::FrontState& Maxwell3D::Front(Core::System& system, VideoCommon::GpuSplit& split) { @@ -1176,7 +1248,12 @@ Maxwell3D::FrontState& Maxwell3D::Front(Core::System& system, VideoCommon::GpuSp void Maxwell3D::ReleaseFront() { executing_macro = front->executing_macro; macro_params = std::move(front->parameters); - std::memcpy(shadow_state.reg_array.data(), front->shadow.data(), sizeof(front->shadow)); + // Entries macros the back ran may have written are the engine's already. + for (u32 i = 0; i < Regs::NUM_REGS; ++i) { + if (front->back_shadow[i] == 0) { + shadow_state.reg_array[i] = front->shadow[i]; + } + } shadow_state.shadow_ram_control = front->control; front.reset(); front_macros = false; @@ -1186,12 +1263,13 @@ void Maxwell3D::ReleaseFront() { void Maxwell3D::FrontWrite(VideoCommon::GpuSplit& split, FrontState& fs, u32 method, u32 argument, bool is_last_call, bool from_macro) { namespace Profile = VideoCommon::Nx2Profile; - // As ProcessShadowRam does on the back. + // As ProcessShadowRam does on the back. A register a macro the back ran may have written is + // the front's again, and its write is sent even if the front's copies hold the value. u32 value = argument; - bool shadow_kept = true; + bool shadow_kept = !(fs.back_owned[method] && FrontTakeOver(split, fs, method)); if (fs.control == Regs::ShadowRamControl::Track || fs.control == Regs::ShadowRamControl::TrackWithFilter) { - shadow_kept = fs.shadow[method] == argument; + shadow_kept = shadow_kept && fs.shadow[method] == argument; fs.shadow[method] = argument; } else if (fs.control == Regs::ShadowRamControl::Replay) { value = fs.shadow[method]; @@ -1323,7 +1401,11 @@ void Maxwell3D::FrontCallMultiMethod(Core::System& system, VideoCommon::GpuSplit } return; } - // What CallMultiMethod does to the registers on the back, then one entry for all the words. + // What CallMultiMethod does to the registers on the back, then one entry for all the words + // (sent whatever the front's copies hold). + if (fs.back_owned[method]) { + FrontTakeOver(split, fs, method); + } constexpr u32 cb_data = MAXWELL3D_REG_INDEX(const_buffer.buffer); const auto tracked = [&fs] { return fs.control == Regs::ShadowRamControl::Track || @@ -1413,12 +1495,15 @@ void Maxwell3D::FrontRunMacro(Core::System& system, VideoCommon::GpuSplit& split const auto positions = GetNativeMacroPositions(); const u32 start = positions[((method - MacroRegistersStart) >> 1) % positions.size()]; VideoCommon::Nx2Profile::CountMethod(method, false); - { + if (!FrontRunOnBack(system, split, fs, engine, start)) { VideoCommon::Nx2Profile::Scope profile_scope{VideoCommon::Nx2Profile::Zone::Macro}; engine.native_interpreter.active_dram = &native_data_ram; const auto parameter_count = fs.parameters.size(); - const auto read = [&fs](u32 address) { - return address < Regs::NUM_REGS ? fs.regs[address] : 0U; + const auto read = [&](u32 address) { + if (address >= Regs::NUM_REGS) { + return 0U; + } + return fs.back_owned[address] ? FrontRegister(split, fs, address) : fs.regs[address]; }; const auto emit = [&](u32 address, u32 value) { if (address >= MacroRegistersStart) { @@ -1458,6 +1543,166 @@ void Maxwell3D::FrontRunMacro(Core::System& system, VideoCommon::GpuSplit& split } } +// A macro whose code reads memory into its FIFO (MME_DMA_READ_FIFOED) and writes only registers +// BackWritable allows runs on the back, at its point of the method stream: it reads the words as +// a drained front would, and the front goes on without waiting for the work queued before it. +// The registers it can write (and, in a tracked shadow RAM mode, their entries) are the back's +// until the front writes them (FrontTakeOver) or needs them (FrontRegister). Returns whether the +// call went to the back. +bool Maxwell3D::FrontRunOnBack(Core::System& system, VideoCommon::GpuSplit& split, FrontState& fs, + const MacroEngine& engine, u32 start) { + using DrainSite = VideoCommon::DrainSite; + using BackMacro = FrontState::BackMacro; + if (!BackMacrosEnabled() || fs.parameters.size() > 0x1000) { + return false; + } + // A macro the back ran wrote another register: the front's copies may be stale anywhere. It + // runs on the front from now on, and the front takes the back's registers (FrontVerify). + const auto take_escapes = [&] { + fs.back_escapes_seen = fs.back_escapes.load(std::memory_order_acquire); + for (const auto& macro : fs.back_macros) { + if (macro && macro->back && macro->escaped.load(std::memory_order_relaxed)) { + macro->back = false; + fs.back_demoted.push_back(macro->start); + LOG_ERROR(HW_GPU, + "Split back macro at {:#x} wrote outside the registers its code " + "can write; it runs on the front from now on", + macro->start); + } + } + FrontVerify(split, fs); + }; + if (fs.back_generation != engine.native_code_generation) [[unlikely]] { + // The macros sent before run the code they were sent with, and check their records. + split.WaitUntil(fs.back_last, DrainSite::BackCode); + if (fs.back_escapes.load(std::memory_order_relaxed) != fs.back_escapes_seen) { + take_escapes(); + } + fs.back_macros.clear(); + fs.back_macros.resize(engine.native_code.size() / 3); + fs.back_generation = engine.native_code_generation; + fs.back_code_copied = false; + } else if (fs.back_escapes.load(std::memory_order_relaxed) != fs.back_escapes_seen) + [[unlikely]] { + take_escapes(); + } + if (start >= fs.back_macros.size()) { + return false; + } + auto& known = fs.back_macros[start]; + if (!known) [[unlikely]] { + known = std::make_unique(); + known->start = start; + const auto effects = TuringMacroInterpreter::AnalyzeEffects(engine.native_code, start); + const auto& methods = effects.methods; + const bool fifo_read = std::ranges::binary_search(methods, 0x158U); + if (fifo_read || std::ranges::binary_search(methods, 0x157U)) { + std::string reason; + if (std::ranges::find(fs.back_demoted, start) != fs.back_demoted.end()) { + reason = "it wrote another register on the back before"; + } else if (!effects.methods_known) { + reason = "it computes the method of a write"; + } else if (effects.data_ram) { + reason = "it uses the data RAM"; + } else if (const auto it = std::ranges::find_if( + methods, [this](u32 m) { return !BackWritable(*this, m); }); + it != methods.end()) { + reason = fmt::format("it writes {:#x}", *it); + } + known->back = reason.empty() && fifo_read; + if (known->back) { + known->writes = methods; + for (const u32 written : methods) { + known->write_set.set(written); + } + } + static u32 logged{}; + if (logged < 32) { + ++logged; + LOG_INFO(HW_GPU, "Split macro at {:#x} reads memory and runs on the {}: {}", start, + known->back ? "back" : "front", + known->back ? fmt::format("writes {:#x}", fmt::join(methods, ",")) + : reason); + } + } + } + if (!known->back) { + return false; + } + if (!fs.back_engine) { + fs.back_engine = std::make_unique(true); + fs.back_engine->native_turing = true; + } + MacroEngine& back = *fs.back_engine; + if (!fs.back_code_copied) { + // Nothing sent with the old code is left to run (waited for above). + back.native_code = engine.native_code; + back.native_code_generation = engine.native_code_generation; + fs.back_code_copied = true; + } + split.DeferWords( + [this, system_ptr = &system, engine_ptr = &back, macro = known.get(), + escapes = &fs.back_escapes](std::span parameters) { + // As ProcessMacro calls a macro, with the back's engine. + ConsumeSink(*system_ptr); + engine_ptr->split_writes = ¯o->write_set; + const u64 unexpected = engine_ptr->split_unexpected_writes; + engine_ptr->Execute(*system_ptr, *this, macro->start, parameters); + if (engine_ptr->split_unexpected_writes != unexpected) [[unlikely]] { + macro->escaped.store(true, std::memory_order_relaxed); + escapes->fetch_add(1, std::memory_order_release); + } + }, + fs.parameters); + const u64 position = split.Position(); + fs.back_last = position; + const bool tracked = fs.control == Regs::ShadowRamControl::Track || + fs.control == Regs::ShadowRamControl::TrackWithFilter; + for (const u32 written : known->writes) { + fs.back_regs[written] = position; + if (tracked) { + fs.back_shadow[written] = position; + } + fs.back_owned[written] = true; + } + return true; +} + +// A register as the back holds it once it ran what the front sent: the front's copy, or the +// back's once it ran the last macro there that can write it. No entry sent after that macro +// writes the register (a front write takes it over first, a later macro there would be the one +// waited for), so the back's value stays until the front writes the register again. +u32 Maxwell3D::FrontRegister(VideoCommon::GpuSplit& split, FrontState& fs, u32 method) { + if (fs.back_regs[method] != 0) { + split.WaitUntil(fs.back_regs[method], VideoCommon::DrainSite::BackRegister); + fs.regs[method] = regs.reg_array[method]; + fs.back_regs[method] = 0; + fs.back_owned[method] = fs.back_shadow[method] != 0; + } + return fs.regs[method]; +} + +// Before the front writes `method`: the register, and its shadow RAM entry unless the mode +// leaves that alone, are the front's again; a replay takes the entry from the back first (as +// FrontRegister). Returns whether the back may hold another value or entry than the front's +// copies, so that the write must be sent even if it seems to change nothing. +bool Maxwell3D::FrontTakeOver(VideoCommon::GpuSplit& split, FrontState& fs, u32 method) { + using Control = Regs::ShadowRamControl; + const bool tracked = fs.control == Control::Track || fs.control == Control::TrackWithFilter; + if (fs.back_shadow[method] != 0 && fs.control == Control::Replay) { + split.WaitUntil(fs.back_shadow[method], VideoCommon::DrainSite::BackRegister); + fs.shadow[method] = shadow_state.reg_array[method]; + fs.back_shadow[method] = 0; + } + const bool differs = fs.back_regs[method] != 0 || (tracked && fs.back_shadow[method] != 0); + fs.back_regs[method] = 0; + if (tracked) { + fs.back_shadow[method] = 0; + } + fs.back_owned[method] = fs.back_shadow[method] != 0; + return differs; +} + void Maxwell3D::FrontTrackInstanceLoop(FrontState& fs, u32 method, u32 value) { switch (method) { case MAXWELL3D_REG_INDEX(draw.begin): @@ -1543,12 +1788,14 @@ void Maxwell3D::FrontNoteWrites(VideoCommon::GpuSplit& split, const FrontState& break; } case MAXWELL3D_REG_INDEX(render_enable.mode): { - // A condition read from memory (ProcessQueryCondition) flushes every cache over it. + // A condition read from memory (ProcessQueryCondition) flushes every cache over it (also + // under an override a macro the back ran may have set). + constexpr u32 override_method = MAXWELL3D_REG_INDEX(render_enable_override); const auto mode = static_cast(fs.regs[MAXWELL3D_REG_INDEX(render_enable.mode)]); - if (static_cast( - fs.regs[MAXWELL3D_REG_INDEX(render_enable_override)]) == - Regs::RenderEnable::Override::UseRenderEnable && + if ((fs.back_regs[override_method] != 0 || + static_cast(fs.regs[override_method]) == + Regs::RenderEnable::Override::UseRenderEnable) && (mode == Regs::RenderEnable::Mode::Conditional || mode == Regs::RenderEnable::Mode::IfEqual || mode == Regs::RenderEnable::Mode::IfNotEqual)) { @@ -1567,8 +1814,9 @@ void Maxwell3D::FrontNoteWrites(VideoCommon::GpuSplit& split, const FrontState& void Maxwell3D::FrontDmaRead(VideoCommon::GpuSplit& split, FrontState& fs, u32 macro, u32 method, u32 count) { - const GPUVAddr source = (GPUVAddr{fs.regs[0x154] & 0x1ffffff} << 32) | fs.regs[0x155]; - const u32 ram = fs.regs[0x156]; + const GPUVAddr source = (GPUVAddr{FrontRegister(split, fs, 0x154) & 0x1ffffff} << 32) | + FrontRegister(split, fs, 0x155); + const u32 ram = FrontRegister(split, fs, 0x156); const u32 words = std::min(count, 0x1000); const size_t read_bytes = size_t{words} * sizeof(u32); std::vector diag_before; @@ -1672,6 +1920,18 @@ void Maxwell3D::FrontVerify(VideoCommon::GpuSplit& split, FrontState& fs) { ? shadow_state.reg_array[method] : value; } + // What macros the back ran may have written, the front takes over (FrontRunOnBack). + for (u32 i = 0; i < Regs::NUM_REGS; ++i) { + if (fs.back_regs[i] != 0) { + fs.regs[i] = expected[i]; + } + if (fs.back_shadow[i] != 0) { + fs.shadow[i] = shadow_state.reg_array[i]; + } + } + fs.back_regs.fill(0); + fs.back_shadow.fill(0); + fs.back_owned.reset(); static u64 control_differences{}; if ((fs.control != shadow_state.shadow_ram_control || fs.shadow[MAXWELL3D_REG_INDEX(shadow_ram_control)] != shadow_state.reg_array[MAXWELL3D_REG_INDEX(shadow_ram_control)] || diff --git a/src/video_core/engines/maxwell_3d.h b/src/video_core/engines/maxwell_3d.h index 3b61d7a758..5ed7ef2e5f 100644 --- a/src/video_core/engines/maxwell_3d.h +++ b/src/video_core/engines/maxwell_3d.h @@ -3411,6 +3411,10 @@ class Maxwell3D final : public EngineInterface { void FrontWrite(VideoCommon::GpuSplit& split, FrontState& state, u32 method, u32 argument, bool is_last_call, bool from_macro); void FrontRunMacro(Core::System& system, VideoCommon::GpuSplit& split, FrontState& state); + bool FrontRunOnBack(Core::System& system, VideoCommon::GpuSplit& split, FrontState& state, + const MacroEngine& engine, u32 start); + u32 FrontRegister(VideoCommon::GpuSplit& split, FrontState& state, u32 method); + bool FrontTakeOver(VideoCommon::GpuSplit& split, FrontState& state, u32 method); void FrontTrackInstanceLoop(FrontState& state, u32 method, u32 value); bool FrontSkipsLoopWrite(FrontState& state, u32 method, u32 value, bool shadow_kept); void FrontNoteWrites(VideoCommon::GpuSplit& split, const FrontState& state, u32 method, diff --git a/src/video_core/gpu_split.h b/src/video_core/gpu_split.h index 1378f6b49d..4eb62b807c 100644 --- a/src/video_core/gpu_split.h +++ b/src/video_core/gpu_split.h @@ -3,12 +3,14 @@ #pragma once +#include #include #include #include #include #include #include +#include #include #include #include @@ -46,6 +48,8 @@ enum class DrainSite : u32 { Map, ///< GPU memory maps before later command reads FrontStart, ///< The front takes an engine's macros over FrontVerify, ///< EDEN_NX2_FRONT_VERIFY + BackRegister, ///< A register a macro the back ran may have written (Maxwell3D) + BackCode, ///< The macro code changes under the macros the back has yet to run Count, }; @@ -53,6 +57,7 @@ inline constexpr std::array(DrainSite::Count)> "other", "fetch drain", "acquire", "semaphore flush", "fence release", "list writers", "list sync", "list flush", "macro dma read", "macro dma writes", "macro dma flush", "gp sync fence", "map", "front start", "front verify", + "back register", "back code", }; // Splits the GPU thread in two (NX2_GPU_THREAD_SPLIT.md). The GPU thread (the front) takes the @@ -178,6 +183,27 @@ class GpuSplit { NoteAnyWrite(); } } + /// Defer, calling `func` with a copy of `words` (a std::span). + template + void DeferWords(const Func& func, std::span words, + Writes writes = Writes::Anywhere) { + static_assert(std::is_trivially_copyable_v && alignof(Func) <= alignof(u64)); + constexpr size_t function_words = (sizeof(Func) + sizeof(u64) - 1) / sizeof(u64); + const size_t payload = function_words + 1 + (words.size() + 1) / 2; + u64* const entry = Reserve(2 + payload); + entry[0] = static_cast(Kind::Call) | (u64{payload} << 8); + entry[1] = reinterpret_cast(&CallWordsThunk); + std::memcpy(entry + 2, &func, sizeof(Func)); + entry[2 + function_words] = words.size(); + if (!words.empty()) { + entry[1 + payload] = 0; // Padding after an odd count + std::memcpy(entry + 3 + function_words, words.data(), words.size_bytes()); + } + Commit(2 + payload); + if (writes == Writes::Anywhere) { + NoteAnyWrite(); + } + } void DeferFunction(std::function&& func); /// Runs `func` on the back and waits for it: for work on back-owned state, such as a buffer /// cache download that records into the Vulkan scheduler. A drained back still submits from @@ -201,6 +227,15 @@ class GpuSplit { void Publish(); /// Waits until the back executed everything sent so far. void Drain(DrainSite site = DrainSite::Other); + /// The ring position after the entries sent so far, for WaitUntil. + [[nodiscard]] u64 Position() const { + return write_position; + } + /// Waits until the back executed the entries sent before `position`. Returns whether any of + /// them was still queued. + bool WaitUntil(u64 position, DrainSite site) { + return WaitFor((std::min)(position, write_position), site); + } /// Compute or copy work was just sent: what can write GPU-written command lists. void NoteWriter() { @@ -287,6 +322,13 @@ class GpuSplit { static void CallThunk(const u64* payload) { (*reinterpret_cast(payload))(); } + template + static void CallWordsThunk(const u64* payload) { + constexpr size_t function_words = (sizeof(Func) + sizeof(u64) - 1) / sizeof(u64); + (*reinterpret_cast(payload))( + std::span{reinterpret_cast(payload + function_words + 1), + static_cast(payload[function_words])}); + } /// Room for an entry of `words` at the write position (never across the ring's end). u64* Reserve(size_t words) { diff --git a/src/video_core/macro.cpp b/src/video_core/macro.cpp index c1c29f4edc..0743a8907f 100644 --- a/src/video_core/macro.cpp +++ b/src/video_core/macro.cpp @@ -1365,7 +1365,9 @@ void MacroEngine::Execute(Core::System& system, Engines::Maxwell3D& maxwell3d, u // Reused between calls (a macro never calls another: methods >= 0xe00 are refused below); // filled only for calls that are interpreted. auto& fifo = native_fifo; - native_interpreter.active_dram = &maxwell3d.GetNativeDataRam(); + // The channel's data RAM is the split's front's while it runs macros (split_writes). + native_interpreter.active_dram = + split_writes != nullptr ? &native_interpreter.dram : &maxwell3d.GetNativeDataRam(); // EDEN_NX2_MME_RECORD=: write the first 200000 calls (u32 start, u32 count, the // parameters) and then the macro code to .code, to check interpreter changes // offline against the previous interpreter. EDEN_NX2_MME_RECORD_SKIP=: skip the first @@ -1396,6 +1398,9 @@ void MacroEngine::Execute(Core::System& system, Engines::Maxwell3D& maxwell3d, u bool read_memory = false; const auto emit = [&](u32 address, u32 value) { if (address >= 0xe00) return false; + if (split_writes != nullptr && !(*split_writes)[address]) [[unlikely]] { + ++split_unexpected_writes; + } if (address == 0x556) { // SET_RENDER_ENABLE_OVERRIDE: 0 use the render-enable condition, 1 always, // 2 never. Log a histogram every 20000 macro writes (pop-in diagnostics). diff --git a/src/video_core/macro.h b/src/video_core/macro.h index c116b25ad8..d213b86efd 100644 --- a/src/video_core/macro.h +++ b/src/video_core/macro.h @@ -6,6 +6,7 @@ #pragma once +#include #include #include "video_core/macro_memo.h" #include "video_core/macro_turing.h" @@ -265,7 +266,11 @@ struct MacroEngine { std::shared_ptr native_jit; MacroMemoPairs native_writes; u64 native_calls{}; - + /// Set while the split GPU thread's back runs a call in its front's place (Maxwell3D:: + /// FrontRunOnBack): the 3D registers the call can write. Such a call uses no data RAM; its + /// writes to other registers are counted in split_unexpected_writes. + const std::bitset<0xe00>* split_writes{}; + u64 split_unexpected_writes{}; }; struct NativeMacroContext { diff --git a/src/video_core/macro_turing.h b/src/video_core/macro_turing.h index 600db48119..56af4e9dbe 100644 --- a/src/video_core/macro_turing.h +++ b/src/video_core/macro_turing.h @@ -10,6 +10,7 @@ #include #include #include +#include #include namespace Tegra { @@ -253,6 +254,271 @@ class TuringMacroInterpreter { return {false, pc, consumed, error}; } + /// What a call from `start` can do on any path through the code (each branch either way, + /// each loop any number of times), as Execute runs it: the methods it can write, each a + /// constant of the code unless `methods_known` is false, and whether it uses the data RAM. + struct Effects { + std::vector methods; ///< Ascending, each once + bool methods_known{true}; + bool data_ram{}; + }; + + static Effects AnalyzeEffects(std::span code, Word start) { + // The values the code fixes, before an instruction runs: registers, carry, the method + // and its increment. Unknown values are always {false, 0}. + struct Value { + bool known; + Word value; + bool operator==(const Value&) const = default; + }; + struct State { + std::array regs; + Value carry, method, increment; + bool operator==(const State&) const = default; + }; + // Where an instruction may continue: unchanged, at `pc` (a taken branch also sets + // whether it cancels an END), or nowhere (a branch that stops the call). + struct Flow { + enum Kind { Next, Jump, Branch, Stop } kind; + Word pc; + bool cancel_end; + }; + constexpr Value unknown{false, 0}; + const auto known = [](Word value) { return Value{true, value}; }; + const Word count = static_cast(code.size() / 3); + Effects effects; + // By instruction, and whether an END before it makes it the call's last (pc * 2 + 1). + std::unordered_map states; + // By a loop's last instruction: where its loops start over. + std::unordered_map> loop_starts; + std::vector work; + const auto reach = [&](Word pc, bool last, const State& state) { + if (pc >= count) { + return; // The call fails there + } + const Word index = pc * 2 + (last ? 1U : 0U); + const auto [it, inserted] = states.try_emplace(index, state); + if (!inserted) { + State joined = it->second; + const auto join = [](Value& to, const Value& from) { + if (to != from) to = Value{false, 0}; + }; + for (std::size_t i = 0; i < joined.regs.size(); ++i) { + join(joined.regs[i], state.regs[i]); + } + join(joined.carry, state.carry); + join(joined.method, state.method); + join(joined.increment, state.increment); + if (joined == it->second) return; + it->second = joined; + } + work.push_back(index); + }; + State entry; + entry.regs.fill(unknown); // Reading one before writing it fails the call + entry.carry = entry.method = entry.increment = known(0); + reach(start, false, entry); + while (!work.empty()) { + const Word index = work.back(); + work.pop_back(); + const Word pc = index / 2; + const bool last = (index & 1) != 0; + const State before = states.at(index); + const Decoded d = Decode({code[pc * 3 + 2], code[pc * 3 + 1], code[pc * 3]}); + const auto get = [&d](const State& state, Word reg, Word lane) -> Value { + if (reg < 24) return state.regs[reg]; + switch (reg) { + case 24: return Value{true, 0}; + case 25: return Value{true, Word(std::int32_t(std::int16_t(d.imm[lane])))}; + case 26: return Value{true, Word(std::int32_t(std::int16_t(d.imm[1 - lane])))}; + case 27: return Value{true, (d.imm[0] << 16) | d.imm[1]}; + default: return Value{false, 0}; // FIFO loads + } + }; + // The lanes and outputs the predicate enables, both sets if its register is unknown. + std::array masks{15, 15}; + std::size_t mask_count = 1; + if (d.pred_mode) { + const Value predicate = get(before, d.pred_reg, 0); + masks = {d.mask_true, d.mask_false}; + if (predicate.known) masks[0] = predicate.value ? d.mask_true : d.mask_false; + else mask_count = 2; + } + for (std::size_t m = 0; m < mask_count; ++m) { + const Word mask = masks[m]; + State state = before; + std::array xs{unknown, unknown}, ys{unknown, unknown}; + for (Word lane = 0; lane < 2; ++lane) { + if (mask & (1U << lane)) { + xs[lane] = get(before, d.src_a[lane], lane); + ys[lane] = get(before, d.src_b[lane], lane); + } + } + std::array alu{known(0), known(0)}; + std::array, 2> flows{}; + std::array flow_counts{1, 1}; // Flow::Next by default + for (Word lane = 0; lane < 2; ++lane) { + if (!(mask & (1U << lane))) continue; + const Value x = xs[lane], y = ys[lane]; + const bool both = x.known && y.known; + const Word a = x.value, b = y.value, imm = d.imm[lane]; + auto& flow = flows[lane]; + auto& flow_count = flow_counts[lane]; + const auto taken = [&](Word op) { + switch (op) { + case 17: case 25: return std::int32_t(a) < std::int32_t(b); + case 18: case 26: return a < b; + case 19: case 27: return std::int32_t(a) <= std::int32_t(b); + case 20: case 28: return a <= b; + default: return a == b; + } + }; + Value value = known(0); + switch (d.op[lane]) { + case 0: case 2: { + const Word result = d.op[lane] == 0 ? a + b : a - b; + value = both ? known(result) : unknown; + const bool carry = d.op[lane] == 0 ? result < a : result > a; + state.carry = both ? known(carry) : unknown; + break; + } + case 1: case 3: + value = both && state.carry.known + ? known(d.op[lane] == 1 ? a + b + state.carry.value + : a - b - state.carry.value) + : unknown; + break; + case 4: case 6: { + const std::uint64_t product = + d.op[lane] == 4 ? std::uint64_t(std::int64_t(std::int32_t(a))) * + std::uint64_t(std::int64_t(std::int32_t(b))) + : std::uint64_t{a} * b; + value = both ? known(Word(product)) : unknown; + state.carry = both ? known(Word(product >> 32)) : unknown; + break; + } + case 5: value = state.carry; break; + case 8: value = x.known ? known(Word(std::countl_zero(a))) : unknown; break; + case 9: value = both ? known(a << (b & 31)) : unknown; break; + case 10: value = both ? known(a >> (b & 31)) : unknown; break; + case 11: + value = both ? known(Word(std::int32_t(a) >> (b & 31))) : unknown; + break; + case 12: value = both ? known(a & b) : unknown; break; + case 13: value = both ? known(~(a & b)) : unknown; break; + case 14: value = both ? known(a | b) : unknown; break; + case 15: value = both ? known(a ^ b) : unknown; break; + case 16: { + const Word dest = (imm >> 10) & 63, bits = (imm >> 5) & 31; + const Word bits_mask = (1U << bits) - 1; + value = both && dest < 32 + ? known((a & ~(bits_mask << dest)) | + (((b >> (imm & 31)) & bits_mask) << dest)) + : unknown; + break; + } + case 17: case 18: case 19: case 20: case 21: + value = both ? known(taken(d.op[lane]) ? ~0U : 0) : unknown; + break; + case 22: value = unknown; break; // A register + case 23: + if (imm < 2) break; + // Entered, or skipped with a count of 0. Its last instruction may start + // it over: reached already, that instruction now continues here too. + if (const Word end = pc + imm - 1; end < count) { + auto& starts = loop_starts[end]; + if (std::ranges::find(starts, pc + 1) == starts.end()) { + starts.push_back(pc + 1); + for (const Word end_index : {end * 2, end * 2 + 1}) { + if (states.contains(end_index)) work.push_back(end_index); + } + } + } + flow_count = 0; + if (!x.known || a != 0) flow[flow_count++] = {Flow::Next, 0, false}; + if (!x.known || a == 0) flow[flow_count++] = {Flow::Jump, pc + imm, false}; + break; + case 24: flow[0] = {Flow::Jump, pc + (imm & 0x7fff), false}; break; + case 25: case 26: case 27: case 28: case 29: { + const auto offset = std::int32_t(imm << 19) >> 19; + const Word target = Word(std::int32_t(pc) + offset); + const bool jumps = taken(d.op[lane]) == bool(imm & 0x8000); + flow_count = 0; + if (!both || jumps) { + const bool cancel_end = (imm & 0x4000) != 0; + flow[flow_count++] = offset == -4096 + ? Flow{Flow::Stop, 0, false} + : Flow{Flow::Branch, target, cancel_end}; + } + if (!both || !jumps) flow[flow_count++] = {Flow::Next, 0, false}; + break; + } + case 30: effects.data_ram = true; value = unknown; break; + case 31: effects.data_ram = true; break; + default: break; // 7: the load barrier + } + alu[lane] = value; + if (d.dst[lane] < 24) state.regs[d.dst[lane]] = value; + } + const auto output = [&](Word selector) -> Value { + switch (selector) { + case 1: return alu[0]; + case 2: return alu[1]; + case 5: return known(d.imm[0]); + case 6: return known(d.imm[1]); + case 8: return known(d.imm[0] >> 12); + case 9: return known(d.imm[1] >> 12); + case 10: return known((d.imm[0] << 16) | d.imm[1]); + default: return unknown; // FIFO loads (the rest fail the call) + } + }; + for (Word lane = 0; lane < 2; ++lane) { + if (!(mask & (4U << lane))) continue; + if (d.out_method[lane]) { + const Value address = output(d.out_method[lane]); + state.method = address.known ? known(address.value & 0xfff) : unknown; + state.increment = + address.known ? known((address.value >> 12) & 15) : unknown; + } + if (d.out_emit[lane]) { + if (state.method.known) effects.methods.push_back(state.method.value); + else effects.methods_known = false; + state.method = state.method.known && state.increment.known + ? known(state.method.value + state.increment.value) + : unknown; + } + } + if (last) continue; // An END before this instruction ends the call after it + // Lane 1's control flow overrides lane 0's; a loop's last instruction may also + // start it over. + const auto loop = loop_starts.find(pc); + const std::vector repeats = + loop != loop_starts.end() ? loop->second : std::vector{}; + for (std::size_t i = 0; i < flow_counts[0]; ++i) { + for (std::size_t j = 0; j < flow_counts[1]; ++j) { + Word next = pc + 1; + bool cancel_end = false, stopped = false; + for (const Flow& flow : {flows[0][i], flows[1][j]}) { + if (flow.kind == Flow::Jump || flow.kind == Flow::Branch) { + next = flow.pc; + } + if (flow.kind == Flow::Branch) cancel_end = flow.cancel_end; + stopped |= flow.kind == Flow::Stop; + } + if (stopped) continue; + const bool end_next = d.end && !cancel_end; + reach(next, end_next, state); + for (const Word repeat : repeats) reach(repeat, end_next, state); + } + } + } + } + std::ranges::sort(effects.methods); + effects.methods.erase(std::unique(effects.methods.begin(), effects.methods.end()), + effects.methods.end()); + return effects; + } + using DataRam = std::array; DataRam dram{}; // The data RAM DREAD/DWRITE use: a channel's own copy when the caller sets one. Macro code is From 0ba202649a7c612449d1f8e9e945e6c858065461 Mon Sep 17 00:00:00 2001 From: jpkto <255363096+jpkto@users.noreply.github.com> Date: Sun, 8 Nov 2026 00:19:08 +0100 Subject: [PATCH 025/105] TEST: check back macros against the old way (EDEN_NX2_BACK_MACRO_VERIFY) EDEN_NX2_BACK_MACRO_VERIFY= (64 without a number): the first n calls of each macro the back runs (FrontRunOnBack) also run the old way. The front drains the split, runs the call against its copies of the registers (the back's for those back macros may have written, exact once drained), its DMA reads made once the caches wrote the words out, and writes nothing (FrontRunTheOldWay). It then sends the call, whose back run records its writes (MacroEngine::split_record), drains again and compares the (method, value) lists. The first 16 differences are logged with both lists; there must be none. Every 10 s it logs the calls on the back, how many were checked and differed, and the front's reads and replays of registers back macros wrote, with how many of those waited for the back. Off, it costs a test of a static flag where it counts. The waits themselves, and the back's macro time, are in EDEN_NX2_GPU_PROFILE ("back register" and "back code", the back thread's "macro" zone). Co-Authored-By: Claude Opus 5.5 --- src/video_core/engines/maxwell_3d.cpp | 180 +++++++++++++++++++++++++- src/video_core/engines/maxwell_3d.h | 2 + src/video_core/macro.cpp | 3 + src/video_core/macro.h | 2 + 4 files changed, 185 insertions(+), 2 deletions(-) diff --git a/src/video_core/engines/maxwell_3d.cpp b/src/video_core/engines/maxwell_3d.cpp index 480f503bbf..535dea21c5 100644 --- a/src/video_core/engines/maxwell_3d.cpp +++ b/src/video_core/engines/maxwell_3d.cpp @@ -1060,7 +1060,9 @@ struct Maxwell3D::FrontState { std::vector writes; ///< The registers its code can write std::bitset write_set; ///< The same, for the back's check std::atomic escaped{}; ///< The back saw it write another register + u32 checked{}; ///< Calls EDEN_NX2_BACK_MACRO_VERIFY checked }; + std::vector> back_record; ///< A checked call's writes on the back /// By start in the macro code of back_generation, once called. std::vector> back_macros; u64 back_generation{~u64{0}}; @@ -1220,6 +1222,68 @@ bool BackWritable(const Maxwell3D& maxwell3d, u32 method) { } } +// EDEN_NX2_BACK_MACRO_VERIFY= (64 without a number): the first n calls of each macro the back +// runs (FrontRunOnBack) also run the old way, on the front once the split is drained +// (FrontRunTheOldWay), and the front compares the writes (method, value) both make; they must +// not differ. Every 10 s it logs the counts, with the front's reads and replays of registers +// such macros wrote and how many of them waited for the back. +u32 BackMacroChecks() { + static const u32 checks = [] { + const char* value = std::getenv("EDEN_NX2_BACK_MACRO_VERIFY"); + if (value == nullptr) { + return 0U; + } + const unsigned long calls = std::strtoul(value, nullptr, 10); + return calls != 0 ? static_cast(calls) : 64U; + }(); + return checks; +} + +struct BackMacroStats { + u64 calls{}; + u64 checked{}; + u64 differences{}; + u64 total_differences{}; + u64 reads{}; + u64 read_waits{}; + u64 replays{}; + u64 replay_waits{}; + std::chrono::steady_clock::time_point last_report{}; +}; + +BackMacroStats& BackStats() { + static BackMacroStats stats; + return stats; +} + +void ReportBackMacroStats() { + auto& stats = BackStats(); + const auto now = std::chrono::steady_clock::now(); + if (stats.last_report == std::chrono::steady_clock::time_point{}) { + stats.last_report = now; + return; + } + const double seconds = std::chrono::duration(now - stats.last_report).count(); + if (seconds < 10.0) { + return; + } + LOG_INFO(HW_GPU, + "Split back macros in {:.1f} s: {} calls, {} checked against the old way, {} of " + "those with other writes ({} in all); {} front reads of registers they wrote ({} " + "waited), {} replays of their shadow RAM entries ({} waited)", + seconds, stats.calls, stats.checked, stats.differences, stats.total_differences, + stats.reads, stats.read_waits, stats.replays, stats.replay_waits); + stats = BackMacroStats{.total_differences = stats.total_differences, .last_report = now}; +} + +std::string DescribeWrites(std::span> writes) { + std::string text; + for (size_t i = 0; i < std::min(writes.size(), 12); ++i) { + text += fmt::format("{}{:#x}={:#x}", i == 0 ? "" : ",", writes[i].first, writes[i].second); + } + return writes.size() > 12 ? text + ",..." : text; +} + } // Anonymous namespace Maxwell3D::FrontState& Maxwell3D::Front(Core::System& system, VideoCommon::GpuSplit& split) { @@ -1640,6 +1704,14 @@ bool Maxwell3D::FrontRunOnBack(Core::System& system, VideoCommon::GpuSplit& spli back.native_code_generation = engine.native_code_generation; fs.back_code_copied = true; } + const bool check = known->checked < BackMacroChecks(); + std::vector> old_writes; + if (check) [[unlikely]] { + ++known->checked; + old_writes = FrontRunTheOldWay(split, fs, start); + fs.back_record.clear(); + back.split_record = &fs.back_record; + } split.DeferWords( [this, system_ptr = &system, engine_ptr = &back, macro = known.get(), escapes = &fs.back_escapes](std::span parameters) { @@ -1665,16 +1737,115 @@ bool Maxwell3D::FrontRunOnBack(Core::System& system, VideoCommon::GpuSplit& spli } fs.back_owned[written] = true; } + if (BackMacroChecks() != 0) [[unlikely]] { + auto& stats = BackStats(); + ++stats.calls; + if (check) { + split.Drain(DrainSite::FrontVerify); + back.split_record = nullptr; + ++stats.checked; + if (old_writes != fs.back_record) { + ++stats.differences; + if (++stats.total_differences <= 16) { + LOG_WARNING(HW_GPU, + "Split back macro check: macro at {:#x} wrote {} the old way, {} " + "on the back", + start, DescribeWrites(old_writes), DescribeWrites(fs.back_record)); + } + } + } + ReportBackMacroStats(); + } return true; } +// EDEN_NX2_BACK_MACRO_VERIFY: the writes (method, value) the call FrontRunOnBack sends makes run +// the old way: on the front, against its copies of the registers, with the split drained first +// and its DMA reads made once the caches wrote the words out. Registers macros the back ran may +// have written are the back's, exact once drained; the call's own writes are what the back's +// CallMethod leaves (the shadow RAM applied). Writes nothing itself. +std::vector> Maxwell3D::FrontRunTheOldWay(VideoCommon::GpuSplit& split, + const FrontState& fs, u32 start) { + split.Drain(VideoCommon::DrainSite::FrontVerify); + std::vector> writes; + std::unordered_map values; + std::unordered_map entries; + const auto value_of = [&](u32 method) { + const auto it = values.find(method); + return it != values.end() ? it->second + : fs.back_regs[method] != 0 ? regs.reg_array[method] + : fs.regs[method]; + }; + const auto entry_of = [&](u32 method) { + const auto it = entries.find(method); + return it != entries.end() ? it->second + : fs.back_shadow[method] != 0 ? shadow_state.reg_array[method] + : fs.shadow[method]; + }; + std::vector fifo = fs.parameters; + const auto read = [&](u32 address) { + return address < Regs::NUM_REGS ? value_of(address) : 0U; + }; + const auto emit = [&](u32 address, u32 value) { + if (address >= MacroRegistersStart) { + return false; + } + writes.emplace_back(address, value); + u32 stored = value; + if (fs.control == Regs::ShadowRamControl::Track || + fs.control == Regs::ShadowRamControl::TrackWithFilter) { + entries[address] = value; + } else if (fs.control == Regs::ShadowRamControl::Replay) { + stored = entry_of(address); + } + values[address] = stored; + if (address == 0x158) { + // As MacroEngine::Execute reads on the back. + const GPUVAddr source = (GPUVAddr{value_of(0x154) & 0x1ffffff} << 32) | value_of(0x155); + const u32 words = std::min(value, 0x1000); + const size_t read_bytes = size_t{words} * sizeof(u32); + split.Run( + [this, source, read_bytes] { + VideoCommon::QueryDma::FlushPending(memory_manager, *rasterizer, source, + read_bytes); + if (memory_manager.IsMemoryDirty(source, read_bytes, + VideoCommon::CacheType::BufferCache)) { + memory_manager.FlushRegion(source, read_bytes, + VideoCommon::CacheType::BufferCache); + } + }, + VideoCommon::DrainSite::FrontVerify); + for (u32 i = 0; i < words; ++i) { + fifo.push_back(memory_manager.Read(source + u64{i} * sizeof(u32))); + } + } + return true; + }; + auto& engine = GetNativeMacroEngine(); +#ifdef ARCHITECTURE_x86_64 + if (!engine.native_jit) { + engine.native_jit = std::make_shared(); + } + engine.native_jit->Prepare(engine.native_code, engine.native_code_generation); + engine.native_jit->Execute(start, fifo, std::span{native_data_ram}, read, emit); +#else + engine.native_interpreter.Execute(engine.native_code, start, fifo, read, emit); +#endif + return writes; +} + // A register as the back holds it once it ran what the front sent: the front's copy, or the // back's once it ran the last macro there that can write it. No entry sent after that macro // writes the register (a front write takes it over first, a later macro there would be the one // waited for), so the back's value stays until the front writes the register again. u32 Maxwell3D::FrontRegister(VideoCommon::GpuSplit& split, FrontState& fs, u32 method) { if (fs.back_regs[method] != 0) { - split.WaitUntil(fs.back_regs[method], VideoCommon::DrainSite::BackRegister); + const bool waited = + split.WaitUntil(fs.back_regs[method], VideoCommon::DrainSite::BackRegister); + if (BackMacroChecks() != 0) [[unlikely]] { + ++BackStats().reads; + BackStats().read_waits += waited ? 1 : 0; + } fs.regs[method] = regs.reg_array[method]; fs.back_regs[method] = 0; fs.back_owned[method] = fs.back_shadow[method] != 0; @@ -1690,7 +1861,12 @@ bool Maxwell3D::FrontTakeOver(VideoCommon::GpuSplit& split, FrontState& fs, u32 using Control = Regs::ShadowRamControl; const bool tracked = fs.control == Control::Track || fs.control == Control::TrackWithFilter; if (fs.back_shadow[method] != 0 && fs.control == Control::Replay) { - split.WaitUntil(fs.back_shadow[method], VideoCommon::DrainSite::BackRegister); + const bool waited = + split.WaitUntil(fs.back_shadow[method], VideoCommon::DrainSite::BackRegister); + if (BackMacroChecks() != 0) [[unlikely]] { + ++BackStats().replays; + BackStats().replay_waits += waited ? 1 : 0; + } fs.shadow[method] = shadow_state.reg_array[method]; fs.back_shadow[method] = 0; } diff --git a/src/video_core/engines/maxwell_3d.h b/src/video_core/engines/maxwell_3d.h index 5ed7ef2e5f..8f38bf5ff5 100644 --- a/src/video_core/engines/maxwell_3d.h +++ b/src/video_core/engines/maxwell_3d.h @@ -3413,6 +3413,8 @@ class Maxwell3D final : public EngineInterface { void FrontRunMacro(Core::System& system, VideoCommon::GpuSplit& split, FrontState& state); bool FrontRunOnBack(Core::System& system, VideoCommon::GpuSplit& split, FrontState& state, const MacroEngine& engine, u32 start); + std::vector> FrontRunTheOldWay(VideoCommon::GpuSplit& split, + const FrontState& state, u32 start); u32 FrontRegister(VideoCommon::GpuSplit& split, FrontState& state, u32 method); bool FrontTakeOver(VideoCommon::GpuSplit& split, FrontState& state, u32 method); void FrontTrackInstanceLoop(FrontState& state, u32 method, u32 value); diff --git a/src/video_core/macro.cpp b/src/video_core/macro.cpp index 0743a8907f..d9fa574a1a 100644 --- a/src/video_core/macro.cpp +++ b/src/video_core/macro.cpp @@ -1401,6 +1401,9 @@ void MacroEngine::Execute(Core::System& system, Engines::Maxwell3D& maxwell3d, u if (split_writes != nullptr && !(*split_writes)[address]) [[unlikely]] { ++split_unexpected_writes; } + if (split_record != nullptr) [[unlikely]] { + split_record->emplace_back(address, value); + } if (address == 0x556) { // SET_RENDER_ENABLE_OVERRIDE: 0 use the render-enable condition, 1 always, // 2 never. Log a histogram every 20000 macro writes (pop-in diagnostics). diff --git a/src/video_core/macro.h b/src/video_core/macro.h index d213b86efd..b739d633f7 100644 --- a/src/video_core/macro.h +++ b/src/video_core/macro.h @@ -271,6 +271,8 @@ struct MacroEngine { /// writes to other registers are counted in split_unexpected_writes. const std::bitset<0xe00>* split_writes{}; u64 split_unexpected_writes{}; + /// EDEN_NX2_BACK_MACRO_VERIFY: where such a call records its writes (method, value). + std::vector>* split_record{}; }; struct NativeMacroContext { From 34656e8225acb5705050c27804c3df30f4a2f3d9 Mon Sep 17 00:00:00 2001 From: jpkto <255363096+jpkto@users.noreply.github.com> Date: Sun, 8 Nov 2026 00:21:31 +0100 Subject: [PATCH 026/105] video_core: command lists on GPU-written pages wait only for the words GPU work wrote In Ocarina of Time's Hyrule Field (EDEN_NX2_GPU_PROFILE, test46b and test47) the split's front waits for the host GPU 169-204 times a second, 550-765 us each (93-156 ms of every second), for the copy of a GPU-written command list page (BufferCache::Nx2ReadListCopy, the "list GPU wait" zone). The back has nothing queued meanwhile: across two boots its idle time moved one for one with these waits. Only pages whose last noted write is GPU work take that copy: a compute dispatch's storage write, or a copy engine copy of GPU-written bytes (DMACopy notes only the source ranges that are GPU-modified). The CPU, inline uploads and copies of bytes guest memory holds write guest memory and retire the copy instead. So the words GPU work wrote exist only on the host GPU, and those must wait. But the copy served the whole 4 KiB page: a list elsewhere on the page waited for it as well, and read from the copy the words the host had written, which the buffer it came from need not have had uploaded yet. - BufferCache::Nx2ListPage keeps the page's words GPU work wrote that guest memory lacks: set by every GPU write noted to the page (Nx2NoteListWrite, the whole range a dispatch or copy may write), cleared for the words the host writes whole (Nx2NoteListHostWrite) and for all of them when the front flushes the page (Nx2NoteListSynced, Nx2FlushListPages). - Once the front has flushed the page (before, GPU work may have written it unnoted, before it became a command list page), a list on none of those words reads guest memory and waits for no copy. A list on some of them reads those words from the copy and the others from guest memory. - The copy engine tells the command list pages what it writes to guest memory: DMACopy notes the copied bytes as a host write before it notes the GPU-written source ranges as GPU writes, and DMAClear notes the cleared words as an inline write does. Both left the page's copy current, so the next list there read the bytes from before. DMAClear also checked only a quarter of its range for registered buffers: it passed the word count as bytes. - A list on a GPU-written page waited for every dispatch and copy sent so far (GpuSplit::WaitForWriters). As for lists on other pages (Step, since dab34bfd8c), the work that rewrites it is ordered before the fetch only by a host-level wait between them: wait for the work before the last one (WaitForSyncedWriters). The profiler counts this wait as its own site, "list copy writers". - While the front waits for the GPU to finish a copy (BufferCacheRuntime::WaitSubmittedTick), the back runs only what the front published. When the front had not waited for the back before (the writers were done), the entries sent since its last publish, fewer than 256 ring words, waited out the copy as well: it publishes them first. Tests (buffer_cache_ports): lists beside the words a dispatch wrote read guest memory, lists on them read the copy and spanning ones both; pages the front has not flushed read every word from the copy; host writes give the words they cover whole back to guest memory and retire the copy; a flushed page reads guest memory until GPU work writes it again; copy engine clears give their words back. Co-Authored-By: Claude Opus 5.5 --- src/tests/video_core/buffer_cache_ports.cpp | 165 ++++++++++++++++++ src/video_core/buffer_cache/buffer_cache.h | 91 +++++++++- .../buffer_cache/buffer_cache_base.h | 39 ++++- src/video_core/dma_pusher.cpp | 11 +- src/video_core/gpu_split.cpp | 5 +- src/video_core/gpu_split.h | 7 +- .../renderer_vulkan/vk_buffer_cache.cpp | 10 +- 7 files changed, 301 insertions(+), 27 deletions(-) diff --git a/src/tests/video_core/buffer_cache_ports.cpp b/src/tests/video_core/buffer_cache_ports.cpp index 337a407f9d..5292a86108 100644 --- a/src/tests/video_core/buffer_cache_ports.cpp +++ b/src/tests/video_core/buffer_cache_ports.cpp @@ -92,6 +92,23 @@ struct Runtime { Staging UploadStagingBuffer(u64 size) { return DownloadStagingBuffer(size); } + void FreeDeferredStagingBuffer(Staging&) {} + /// Buffer to buffer, as when a new buffer takes over the ones it overlaps. + void CopyBuffer(Buffer& dst, Buffer& src, std::span copies, + bool) { + for (const auto& copy : copies) { + REQUIRE(copy.src_offset + copy.size <= src.bytes.size()); + REQUIRE(copy.dst_offset + copy.size <= dst.bytes.size()); + std::memcpy(dst.bytes.data() + copy.dst_offset, src.bytes.data() + copy.src_offset, + copy.size); + } + } + void ClearBuffer(Buffer& buffer, u32 offset, size_t size, u32 value) { + REQUIRE(offset + size <= buffer.bytes.size()); + for (size_t i = 0; i + sizeof(value) <= size; i += sizeof(value)) { + std::memcpy(buffer.bytes.data() + offset + i, &value, sizeof(value)); + } + } void CopyBuffer(Buffer* dst, Buffer& src, std::span copies, bool) { for (const auto& copy : copies) { @@ -282,6 +299,13 @@ struct BufferCacheTestAccess { static void MarkWritten(Cache& cache, BufferId id, DAddr address, u32 size) { cache.MarkWrittenBuffer(id, address, size); } + /// A dispatch or copy wrote [address, address + size) of buffer `id` (Nx2NoteListWrite). + static void ListWrite(Cache& cache, BufferId id, DAddr address, u64 size) { + cache.Nx2NoteListWrite(id, address, size); + } + static void UseGpuMemory(Cache& cache, Tegra::MemoryManager& memory) { + cache.gpu_memory = &memory; + } static size_t ReadDownloads(Cache& cache) { return cache.read_downloads.size(); } @@ -654,6 +678,147 @@ TEST_CASE("Command page readers remain ordered during batch completion", "[buffe REQUIRE(f.runtime.downloads == 1); } +namespace { + +/// Makes `page` a command list page for one test (nx2.h keeps them for the session). +struct ListPage { + explicit ListPage(DAddr page) { + Tegra::Nx2ClearGpuWrittenPages(); + Tegra::Nx2MarkCommandListPage(page); + } + ~ListPage() { + Tegra::Nx2ClearGpuWrittenPages(); + } + ListPage(const ListPage&) = delete; + ListPage& operator=(const ListPage&) = delete; +}; + +bool AllBytes(std::span bytes, u8 value) { + return std::ranges::all_of(bytes, [value](u8 byte) { return byte == value; }); +} + +} // namespace + +TEST_CASE("Command lists on words no GPU work wrote read guest memory, not the page's copy", + "[buffer_ports]") { + Fixture f; + const ListPage list_page{0x20000}; + const auto id = Access::Add(*f.cache, 0x20000, 0x10000); + std::ranges::fill(Access::Get(*f.cache, id).bytes, 0x77); + std::memset(f.bytes + 0x10000, 0x11, 0x1000); + // The page's first list drained the split and flushed the page (FlushGpuWrittenCommands). + f.cache->Nx2NoteListSynced(0x20000); + // A dispatch writes bytes 0x800-0x900 of the page; the copy taken after it is on its way. + f.cache->Nx2CollectListCopies(true); + Access::ListWrite(*f.cache, id, 0x20800, 0x100); + REQUIRE(f.cache->Nx2CopyListPages(f.runtime.CurrentTick())); + f.cache->Nx2CollectListCopies(false); + using Read = Cache::Nx2ListRead; + std::array list{}; + REQUIRE(f.cache->Nx2ReadListCopy(0x20100, list, true) == Read::Synced); + REQUIRE(f.cache->Nx2ReadListCopy(0x207f0, list, true) == Read::Synced); + // The GPU completes the copy. A list on the dispatch's words reads them from it; one that + // starts before them reads its other words from guest memory. + f.runtime.Finish(); + REQUIRE(f.cache->Nx2ReadListCopy(0x20800, list, true) == Read::Copied); + REQUIRE(AllBytes(list, 0x77)); + std::array spanning{}; + REQUIRE(f.cache->Nx2ReadListCopy(0x207f0, spanning, true) == Read::Copied); + REQUIRE(AllBytes(std::span{spanning}.first(16), 0x11)); + REQUIRE(AllBytes(std::span{spanning}.last(16), 0x77)); +} + +TEST_CASE("Command list pages the front has not flushed read every word from the copy", + "[buffer_ports]") { + Fixture f; + const ListPage list_page{0x20000}; + const auto id = Access::Add(*f.cache, 0x20000, 0x10000); + std::ranges::fill(Access::Get(*f.cache, id).bytes, 0x77); + std::memset(f.bytes + 0x10000, 0x11, 0x1000); + // GPU work may have written the page before it became a command list page: only the first + // write since is noted, and guest memory need not hold the rest. + f.cache->Nx2CollectListCopies(true); + Access::ListWrite(*f.cache, id, 0x20800, 0x100); + REQUIRE(f.cache->Nx2CopyListPages(f.runtime.CurrentTick())); + f.cache->Nx2CollectListCopies(false); + f.runtime.Finish(); + std::array list{}; + REQUIRE(f.cache->Nx2ReadListCopy(0x20100, list, true) == Cache::Nx2ListRead::Copied); + REQUIRE(AllBytes(list, 0x77)); + REQUIRE(f.cache->Nx2ReadListCopy(0x207f0, list, true) == Cache::Nx2ListRead::Copied); + REQUIRE(AllBytes(list, 0x77)); +} + +TEST_CASE("Host writes give command list words back to guest memory and retire the page's copy", + "[buffer_ports]") { + Fixture f; + const ListPage list_page{0x20000}; + const auto id = Access::Add(*f.cache, 0x20000, 0x10000); + f.cache->Nx2NoteListSynced(0x20000); + f.cache->Nx2CollectListCopies(true); + Access::ListWrite(*f.cache, id, 0x20800, 0x100); + REQUIRE(f.cache->Nx2CopyListPages(f.runtime.CurrentTick())); + f.cache->Nx2CollectListCopies(false); + f.runtime.Finish(); + using Read = Cache::Nx2ListRead; + std::array list{}; + REQUIRE(f.cache->Nx2ReadListCopy(0x20840, list, true) == Read::Copied); + // The CPU, an inline upload or a copy of bytes guest memory holds writes 0x800-0x840 whole, + // and two bytes of each of the words at 0x880 and 0x884. + f.cache->Nx2NoteListHostWrite(0x20800, 0x40); + f.cache->Nx2NoteListHostWrite(0x20882, 4); + REQUIRE(f.cache->Nx2ReadListCopy(0x20800, list, true) == Read::Synced); + REQUIRE(f.cache->Nx2ReadListCopy(0x20830, list, true) == Read::Synced); + // GPU-written words it did not write whole: the copy no longer holds the page. + REQUIRE(f.cache->Nx2ReadListCopy(0x20840, list, true) == Read::Unknown); + REQUIRE(f.cache->Nx2ReadListCopy(0x20880, list, true) == Read::Unknown); +} + +TEST_CASE("A flushed command list page reads guest memory until GPU work writes it again", + "[buffer_ports]") { + Fixture f; + const ListPage list_page{0x20000}; + const auto id = Access::Add(*f.cache, 0x20000, 0x10000); + f.cache->Nx2CollectListCopies(true); + Access::ListWrite(*f.cache, id, 0x20000, 0x1000); + REQUIRE(f.cache->Nx2CopyListPages(f.runtime.CurrentTick())); + using Read = Cache::Nx2ListRead; + std::array list{}; + REQUIRE(f.cache->Nx2ReadListCopy(0x20800, list, true) == Read::Copied); + // The front drained the split and flushed the page (DmaPusher::FlushGpuWrittenCommands). + f.cache->Nx2NoteListSynced(0x20000); + REQUIRE(f.cache->Nx2ReadListCopy(0x20800, list, true) == Read::Synced); + Access::ListWrite(*f.cache, id, 0x20900, 0x40); + REQUIRE(f.cache->Nx2CopyListPages(f.runtime.CurrentTick())); + f.cache->Nx2CollectListCopies(false); + f.runtime.Finish(); + REQUIRE(f.cache->Nx2ReadListCopy(0x20800, list, true) == Read::Synced); + REQUIRE(f.cache->Nx2ReadListCopy(0x20900, list, true) == Read::Copied); +} + +TEST_CASE("Copy engine clears give command list words back to guest memory", "[buffer_ports]") { + Fixture f; + const ListPage list_page{0x20000}; + const auto id = Access::Add(*f.cache, 0x20000, 0x10000); + Tegra::MemoryManager gpu_memory{f.system, f.device, 32, 1ULL << 31, 16}; + gpu_memory.BindRasterizer(&f.rasterizer); + gpu_memory.Map(0x200000, 0x20000, 0x10000, Tegra::PTEKind::PITCH, true); + Access::UseGpuMemory(*f.cache, gpu_memory); + f.cache->Nx2NoteListSynced(0x20000); + f.cache->Nx2CollectListCopies(true); + Access::ListWrite(*f.cache, id, 0x20800, 0x100); + REQUIRE(f.cache->Nx2CopyListPages(f.runtime.CurrentTick())); + f.cache->Nx2CollectListCopies(false); + f.runtime.Finish(); + // 16 words cleared (MaxwellDMA's remapped constant fill writes them to guest memory too). + REQUIRE(f.cache->DMAClear(0x200800, 16, 0)); + REQUIRE(AllBytes(std::span{Access::Get(*f.cache, id).bytes}.subspan(0x800, 0x40), 0)); + using Read = Cache::Nx2ListRead; + std::array list{}; + REQUIRE(f.cache->Nx2ReadListCopy(0x20830, list, true) == Read::Synced); + REQUIRE(f.cache->Nx2ReadListCopy(0x20840, list, true) == Read::Unknown); +} + TEST_CASE("Native uniform banks upload directly without overwriting GPU-owned bytes", "[buffer_ports]") { Fixture f; diff --git a/src/video_core/buffer_cache/buffer_cache.h b/src/video_core/buffer_cache/buffer_cache.h index 21e6092db7..279323c8f7 100644 --- a/src/video_core/buffer_cache/buffer_cache.h +++ b/src/video_core/buffer_cache/buffer_cache.h @@ -1422,6 +1422,11 @@ bool BufferCache

::DMACopy(GPUVAddr src_address, GPUVAddr dest_address, u64 am .size = amount, }}; + // Guest memory gets the copy below, which holds every byte the source has in guest memory: + // command list pages in the destination read those from it, and their copies are stale. The + // bytes copied from GPU-written source ranges are noted as GPU writes after this. + Nx2NoteListHostWrite(*cpu_dest_address, amount); + boost::container::small_vector, 4> tmp_intervals; auto mirror = [&](DAddr base_address, DAddr base_address_end) { const u64 size = base_address_end - base_address; @@ -1473,17 +1478,20 @@ bool BufferCache

::DMAClear(GPUVAddr dst_address, u64 amount, u32 value) { if (!cpu_dst_address) { return false; } - const bool dest_dirty = IsRegionRegistered(*cpu_dst_address, amount); + // `amount` counts words. + const size_t size = amount * sizeof(u32); + const bool dest_dirty = IsRegionRegistered(*cpu_dst_address, size); if (!dest_dirty) { return false; } - const size_t size = amount * sizeof(u32); // The caller writes the cleared words to guest memory after this: earlier reads' downloads // land first, so a reader of these bytes never finds neither there. FinishReadDownloads(*cpu_dst_address, size); ClearDownload(*cpu_dst_address, size); gpu_modified_ranges.Subtract(*cpu_dst_address, size); + // As an inline write: command list pages read the cleared words from guest memory. + Nx2NoteListHostWrite(*cpu_dst_address, size); const BufferId buffer = FindBuffer(*cpu_dst_address, static_cast(size), false); Buffer& dest_buffer = slot_buffers[buffer]; @@ -3074,6 +3082,11 @@ void BufferCache

::Nx2NoteListWrite(BufferId buffer_id, DAddr device_addr, u64 std::scoped_lock lock{nx2_list_mutex}; Nx2ListPage& entry = nx2_list_pages[page_addr]; ++entry.writes; + Nx2MarkListWords( + entry, (std::max)(device_addr, page_addr) - page_addr, + (std::min)(device_addr + size, page_addr + (DAddr{1} << page_bits)) - + page_addr, + true); if (!nx2_collect_list_copies && !entry.draw_written) { entry.draw_written = true; LOG_INFO(HW_GPU, "Command list page {:#x} is written by draws too: its " @@ -3187,8 +3200,11 @@ auto BufferCache

::Nx2ReadListCopy(DAddr device_addr, std::span out, bool return entry.synced_writes == entry.writes ? Nx2ListRead::Synced : Nx2ListRead::Unknown; } - // Pages the CPU writes the lists of (GPU-written once, long before) stay synced. - if (entry.synced_writes == entry.writes) { + // Pages the CPU writes the lists of (GPU-written once, long before) stay synced, and + // a list on none of the words GPU work wrote since the page was flushed is in guest + // memory too: it waits for no copy. + if (entry.synced_writes == entry.writes || + (entry.flushed && !Nx2HasListWords(entry, offset, offset + out.size()))) { return Nx2ListRead::Synced; } if (!current(entry)) { @@ -3196,7 +3212,7 @@ auto BufferCache

::Nx2ReadListCopy(DAddr device_addr, std::span out, bool } // The lists after the first of a batch find the copy done: read it at once. if (runtime.IsFree(entry.copy_tick)) { - std::memcpy(out.data(), entry.copy->mapped_span.data() + offset, out.size()); + Nx2ReadListWords(entry, device_addr, offset, out); return Nx2ListRead::Copied; } tick = entry.copy_tick; @@ -3210,17 +3226,71 @@ auto BufferCache

::Nx2ReadListCopy(DAddr device_addr, std::span out, bool it->second.copy_writes != writes) { return Nx2ListRead::Unknown; } - std::memcpy(out.data(), it->second.copy->mapped_span.data() + offset, out.size()); + Nx2ReadListWords(it->second, device_addr, offset, out); return Nx2ListRead::Copied; } } +template +void BufferCache

::Nx2ReadListWords(const Nx2ListPage& entry, DAddr device_addr, u64 offset, + std::span out) { + if constexpr (!IS_OPENGL) { + // The copy holds the words GPU work wrote. Guest memory holds the others once the page + // was flushed: the host wrote them, and the buffer the copy came from need not have had + // those writes uploaded. + const u8* const copy = entry.copy->mapped_span.data(); + const u64 end = offset + out.size(); + bool all_gpu = true; + for (u64 word = offset / 4; word < (end + 3) / 4 && all_gpu; ++word) { + all_gpu = entry.gpu_words.test(static_cast(word)); + } + if (all_gpu || !entry.flushed) { + std::memcpy(out.data(), copy + offset, out.size()); + return; + } + device_memory.ReadBlockUnsafe(device_addr, out.data(), out.size()); + for (u64 byte = offset; byte < end;) { + const u64 next = (std::min)((byte / 4 + 1) * 4, end); + if (entry.gpu_words.test(static_cast(byte / 4))) { + std::memcpy(out.data() + (byte - offset), copy + byte, next - byte); + } + byte = next; + } + } +} + +template +void BufferCache

::Nx2MarkListWords(Nx2ListPage& entry, u64 begin, u64 end, bool set) { + if (begin >= end) { + return; + } + // Words GPU work may have written in part count whole; the host clears only whole words. + const u64 first = set ? begin / 4 : (begin + 3) / 4; + const u64 last = (std::min)(set ? (end + 3) / 4 : end / 4, Nx2ListPageWords); + for (u64 word = first; word < last; ++word) { + entry.gpu_words.set(static_cast(word), set); + } +} + +template +bool BufferCache

::Nx2HasListWords(const Nx2ListPage& entry, u64 begin, u64 end) { + const u64 last = (std::min)((end + 3) / 4, Nx2ListPageWords); + for (u64 word = begin / 4; word < last; ++word) { + if (entry.gpu_words.test(static_cast(word))) { + return true; + } + } + return false; +} + template void BufferCache

::Nx2NoteListSynced(DAddr page) { if constexpr (!IS_OPENGL) { std::scoped_lock lock{nx2_list_mutex}; Nx2ListPage& entry = nx2_list_pages[page]; entry.synced_writes = entry.writes; + entry.gpu_words.reset(); + entry.flushed = true; } } @@ -3248,6 +3318,8 @@ void BufferCache

::Nx2FlushListPages(std::span pages) { const auto it = nx2_list_pages.find(page); if (it != nx2_list_pages.end() && it->second.writes == writes) { it->second.synced_writes = writes; + it->second.gpu_words.reset(); + it->second.flushed = true; } } } @@ -3269,7 +3341,8 @@ void BufferCache

::Nx2NoteListHostWrite(DAddr device_addr, u64 size) { continue; } std::scoped_lock lock{nx2_list_mutex}; - const auto it = nx2_list_pages.find(page << page_bits); + const DAddr page_addr = page << page_bits; + const auto it = nx2_list_pages.find(page_addr); if (it == nx2_list_pages.end()) { continue; } @@ -3279,6 +3352,10 @@ void BufferCache

::Nx2NoteListHostWrite(DAddr device_addr, u64 size) { if (synced) { entry.synced_writes = entry.writes; } + Nx2MarkListWords(entry, (std::max)(device_addr, page_addr) - page_addr, + (std::min)(device_addr + size, page_addr + (DAddr{1} << page_bits)) - + page_addr, + false); } } } diff --git a/src/video_core/buffer_cache/buffer_cache_base.h b/src/video_core/buffer_cache/buffer_cache_base.h index e600451e52..3dee3f4ec3 100644 --- a/src/video_core/buffer_cache/buffer_cache_base.h +++ b/src/video_core/buffer_cache/buffer_cache_base.h @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -45,6 +46,7 @@ #include "video_core/engines/maxwell_3d.h" #include "video_core/gpu_wait_stats.h" #include "video_core/memory_manager.h" +#include "video_core/nx2.h" #include "video_core/nx2_download_trace.h" #include "video_core/nx2_read_trace.h" #include "video_core/rasterizer_download_area.h" @@ -397,18 +399,21 @@ class BufferCache : public VideoCommon::ChannelSetupCaches out, bool past_sync); /// Front thread, the split drained: the command list page was flushed to guest memory. void Nx2NoteListSynced(DAddr page); void Nx2FlushListPages(std::span pages); - /// CPU and inline writes: they reach guest memory, so a synced page stays synced, but its copy - /// no longer holds the page. + /// CPU, inline and copy engine writes of bytes guest memory holds: they reach guest memory, + /// so a synced page stays synced and the words they cover are guest memory's again, but the + /// page's copy no longer holds the page. void Nx2NoteListHostWrite(DAddr device_addr, u64 size); /// Return true when a CPU region is modified from the GPU @@ -1002,8 +1007,9 @@ class BufferCache : public VideoCommon::ChannelSetupCaches().DownloadStagingBuffer(size_t{}, true)); + static constexpr size_t Nx2ListPageWords = (size_t{1} << Tegra::Nx2WrittenPageBits) / 4; struct Nx2ListPage { - u64 writes{}; ///< GPU writes noted to the page + u64 writes{}; ///< GPU and host writes noted to the page u64 copy_writes{}; ///< `writes` when the copy was made: the copy is current while equal u64 copy_tick{}; ///< GPU tick that completes the copy /// `writes` when the front last flushed the page: guest memory holds it while equal @@ -1016,8 +1022,25 @@ class BufferCache : public VideoCommon::ChannelSetupCaches gpu_words; + bool flushed{}; std::optional copy; }; + /// Sets (`set`) or clears the words of `entry.gpu_words` within [begin, end) bytes of its + /// page: every word the bytes touch when setting, only the words they cover when clearing. + static void Nx2MarkListWords(Nx2ListPage& entry, u64 begin, u64 end, bool set); + /// Whether GPU work wrote any word of [begin, end) bytes of the page guest memory lacks. + [[nodiscard]] static bool Nx2HasListWords(const Nx2ListPage& entry, u64 begin, u64 end); + /// Fills `out` with the page's bytes from `offset` (`device_addr`) on: the words of + /// `entry.gpu_words` from its copy, the others from guest memory. + void Nx2ReadListWords(const Nx2ListPage& entry, DAddr device_addr, u64 offset, + std::span out); std::mutex nx2_list_mutex; ///< nx2_list_pages: the split's front reads the copies std::unordered_map nx2_list_pages; /// Copies replaced while the GPU may still write them: freed once it is past their tick. diff --git a/src/video_core/dma_pusher.cpp b/src/video_core/dma_pusher.cpp index e914bdcb09..5cfccc5f00 100644 --- a/src/video_core/dma_pusher.cpp +++ b/src/video_core/dma_pusher.cpp @@ -669,9 +669,9 @@ bool DmaPusher::ReadGpuWrittenCommands(GPUVAddr address, u32 count) { // them to host memory right after the dispatch or copy that writes them and submits. The // front then waits only for the back to run that work and for the GPU to finish the copy, not // for everything queued and a cache flush (FlushGpuWrittenCommands: ~400 GPU round trips a - // second in a Mario Kart World race, during which the back had nothing to do). Pages - // FlushGpuWrittenCommands flushed whole and no GPU work wrote since are read from guest - // memory. + // second in a Mario Kart World race, during which the back had nothing to do). Lists on words + // no GPU work wrote since FlushGpuWrittenCommands flushed the page, or the host wrote them, + // are read from guest memory and wait for no copy (BufferCache::Nx2ReadListCopy). if (!Tegra::g_nx2_ampere.load(std::memory_order_relaxed) || count == 0 || rasterizer == nullptr) { return false; @@ -688,9 +688,12 @@ bool DmaPusher::ReadGpuWrittenCommands(GPUVAddr address, u32 count) { if (!written) { return false; } + // As for lists on pages no GPU work wrote (Step): the dispatch or copy that rewrites the + // list is ordered before the fetch only by a host-level wait between them, so the front waits + // for the work before the last one, not for compute and copy work sent after it. auto& split = VideoCommon::GpuSplit::Get(); if (split.Active()) { - split.WaitForWriters(); + split.WaitForSyncedWriters(VideoCommon::DrainSite::ListCopies); } bool past_sync = !split.Active(); using GpuWrittenList = VideoCore::RasterizerInterface::GpuWrittenList; diff --git a/src/video_core/gpu_split.cpp b/src/video_core/gpu_split.cpp index ff88a7251f..b2bbccae2e 100644 --- a/src/video_core/gpu_split.cpp +++ b/src/video_core/gpu_split.cpp @@ -233,9 +233,8 @@ bool GpuSplit::WaitForSync() { return WaitFor(std::min(last_sync_position, write_position), DrainSite::ListSync); } -bool GpuSplit::WaitForSyncedWriters() { - return WaitFor(std::min({last_writer_position, last_sync_position, write_position}), - DrainSite::ListWriters); +bool GpuSplit::WaitForSyncedWriters(DrainSite site) { + return WaitFor(std::min({last_writer_position, last_sync_position, write_position}), site); } void GpuSplit::NoteSemaphoreWrite(u64 device_address, u32 payload, bool known) { diff --git a/src/video_core/gpu_split.h b/src/video_core/gpu_split.h index 4eb62b807c..be799a4be3 100644 --- a/src/video_core/gpu_split.h +++ b/src/video_core/gpu_split.h @@ -50,6 +50,7 @@ enum class DrainSite : u32 { FrontVerify, ///< EDEN_NX2_FRONT_VERIFY BackRegister, ///< A register a macro the back ran may have written (Maxwell3D) BackCode, ///< The macro code changes under the macros the back has yet to run + ListCopies, ///< Lists on GPU-written pages: the dispatches and copies before the last sync Count, }; @@ -57,7 +58,7 @@ inline constexpr std::array(DrainSite::Count)> "other", "fetch drain", "acquire", "semaphore flush", "fence release", "list writers", "list sync", "list flush", "macro dma read", "macro dma writes", "macro dma flush", "gp sync fence", "map", "front start", "front verify", - "back register", "back code", + "back register", "back code", "list copy writers", }; // Splits the GPU thread in two (NX2_GPU_THREAD_SPLIT.md). The GPU thread (the front) takes the @@ -255,8 +256,8 @@ class GpuSplit { bool WaitForSync(); /// Waits until the back executed the compute and copy work sent before the last NoteSync: /// the writers a command list fetched now is ordered after. Returns whether any of it was - /// still queued. - bool WaitForSyncedWriters(); + /// still queued. `site`: whose wait it is, for EDEN_NX2_GPU_PROFILE. + bool WaitForSyncedWriters(DrainSite site = DrainSite::ListWriters); /// A semaphore release was just sent: once the back ran it, the word at `device_address` holds /// `payload`, or a value the front can't tell (`known` false: a reduction, a counter report). void NoteSemaphoreWrite(u64 device_address, u32 payload, bool known); diff --git a/src/video_core/renderer_vulkan/vk_buffer_cache.cpp b/src/video_core/renderer_vulkan/vk_buffer_cache.cpp index d471658892..0aadab8e8c 100644 --- a/src/video_core/renderer_vulkan/vk_buffer_cache.cpp +++ b/src/video_core/renderer_vulkan/vk_buffer_cache.cpp @@ -489,11 +489,17 @@ void BufferCacheRuntime::WaitSubmittedTick(u64 tick) { if (master.IsFree(tick)) { return; } + // The split's front waits for the GPU: the back runs meanwhile what the front sent, also the + // entries since the front's last publish, which it does not see yet (fewer than 256 words). + auto& split = VideoCommon::GpuSplit::Get(); + const bool front = !VideoCommon::GpuSplit::OnBackThread() && split.Active(); + if (front) { + split.Publish(); + } // The copy's submission may be deferred (Scheduler::DeferFlush): the split's front asks the // back for it rather than waiting for the back's next submission. If the back has stopped, // nothing else uses the scheduler and Wait submits it. - if (!scheduler.IsSubmitted(tick) && VideoCommon::GpuSplit::Get().Active() && - !VideoCommon::GpuSplit::OnBackThread()) { + if (!scheduler.IsSubmitted(tick) && front) { VideoCommon::Nx2Profile::Scope scope{VideoCommon::Nx2Profile::Zone::ListSubmitWait}; if (!scheduler.WaitSubmitted(tick)) { scheduler.Wait(tick); From 3a665f9f27b139257ba54def39007720c487d86f Mon Sep 17 00:00:00 2001 From: jpkto <255363096+jpkto@users.noreply.github.com> Date: Sun, 8 Nov 2026 00:22:01 +0100 Subject: [PATCH 027/105] TEST: command list read stats and a check against the old reads (EDEN_NX2_LIST_STATS, EDEN_NX2_LIST_VERIFY) EDEN_NX2_LIST_STATS=1 logs every 10 s from the front, for the command lists on GPU-written pages (DmaPusher::ReadGpuWrittenCommands): - "list stats": the lists and their sizes; for each page read, how Nx2ReadListCopy served it (page synced, words clean, copy ready, copy waited, unknown); the copy waits' time and spread; the reads and waits by the page's last writer (dispatch, copy, draw); how many copies held the bytes guest memory already had (the GPU rewrote the same words); and at copy waits: whether the back had work and how much of it the front had not published yet, whether other channels had lists queued for the GPU thread (which could run while the channel waits for the copy), and how long before the wait the back had recorded the copy. - "list stats writers": the dispatches (by shader hash) and copy engine copies (by source GPU address) that last wrote the pages read, with their reads, waits and writes to command list pages. - "list stats pages": the pages read, the bytes the lists read on them, and the bytes and writer of their last write. EDEN_NX2_LIST_VERIFY=1 reads each such list again as the commit before this one did: after every dispatch and copy sent so far (WaitForWriters), from the pages' whole copies (Nx2ListStats::t_reference), or drained and flushed. When the two differ it also drains and flushes, which tells the stale read. "list verify" counts lists and differences every 10 s, the first 32 differences are logged with the list's pages, and the front parses the words read last. The writers are tagged in RasterizerVulkan::DispatchCompute and AccelerateDMA::BufferCopy, the queued lists counted in ThreadManager. Test (buffer_cache_ports): the read as before takes the whole copy. Rebased onto upstream: EDEN_NX2_LIST_STATS and EDEN_NX2_LIST_VERIFY are plain flags read again by VideoCore::CreateGPU, as upstream reads its diagnostic switches since #714, so each works from nx2_switches.txt too, not only from the environment. Co-Authored-By: Claude Opus 5.5 --- src/tests/video_core/buffer_cache_ports.cpp | 29 + src/video_core/CMakeLists.txt | 1 + src/video_core/buffer_cache/buffer_cache.h | 22 +- src/video_core/dma_pusher.cpp | 99 +++ src/video_core/dma_pusher.h | 3 + src/video_core/gpu_split.h | 9 + src/video_core/gpu_thread.cpp | 4 + src/video_core/nx2_list_stats.h | 580 ++++++++++++++++++ .../renderer_vulkan/vk_rasterizer.cpp | 5 + src/video_core/video_core.cpp | 2 + 10 files changed, 751 insertions(+), 3 deletions(-) create mode 100644 src/video_core/nx2_list_stats.h diff --git a/src/tests/video_core/buffer_cache_ports.cpp b/src/tests/video_core/buffer_cache_ports.cpp index 5292a86108..4da78ed02f 100644 --- a/src/tests/video_core/buffer_cache_ports.cpp +++ b/src/tests/video_core/buffer_cache_ports.cpp @@ -24,6 +24,7 @@ #include "video_core/buffer_cache/memory_tracker_base.h" #include "video_core/control/channel_state.h" #include "video_core/gpu.h" +#include "video_core/nx2_list_stats.h" #include "video_core/renderer_null/null_rasterizer.h" namespace { @@ -819,6 +820,34 @@ TEST_CASE("Copy engine clears give command list words back to guest memory", "[b REQUIRE(f.cache->Nx2ReadListCopy(0x20840, list, true) == Read::Unknown); } +TEST_CASE("EDEN_NX2_LIST_VERIFY reads command lists as before, from the page's whole copy", + "[buffer_ports]") { + Fixture f; + const ListPage list_page{0x20000}; + const auto id = Access::Add(*f.cache, 0x20000, 0x10000); + std::ranges::fill(Access::Get(*f.cache, id).bytes, 0x77); + std::memset(f.bytes + 0x10000, 0x11, 0x1000); + f.cache->Nx2NoteListSynced(0x20000); + f.cache->Nx2CollectListCopies(true); + Access::ListWrite(*f.cache, id, 0x20800, 0x100); + REQUIRE(f.cache->Nx2CopyListPages(f.runtime.CurrentTick())); + f.cache->Nx2CollectListCopies(false); + f.runtime.Finish(); + using Read = Cache::Nx2ListRead; + std::array list{}; + { + // Whatever words GPU work wrote: once it wrote the page, every list there reads the copy. + const Tegra::Nx2ListStats::ReferenceScope reference; + REQUIRE(f.cache->Nx2ReadListCopy(0x20100, list, true) == Read::Copied); + REQUIRE(AllBytes(list, 0x77)); + REQUIRE(f.cache->Nx2ReadListCopy(0x207f0, list, true) == Read::Copied); + REQUIRE(AllBytes(list, 0x77)); + } + REQUIRE(f.cache->Nx2ReadListCopy(0x20100, list, true) == Read::Synced); + REQUIRE(f.cache->Nx2ReadListCopy(0x207f0, list, true) == Read::Copied); + REQUIRE(AllBytes(std::span{list}.first(16), 0x11)); +} + TEST_CASE("Native uniform banks upload directly without overwriting GPU-owned bytes", "[buffer_ports]") { Fixture f; diff --git a/src/video_core/CMakeLists.txt b/src/video_core/CMakeLists.txt index 3a1a413cd8..c64b800b19 100644 --- a/src/video_core/CMakeLists.txt +++ b/src/video_core/CMakeLists.txt @@ -119,6 +119,7 @@ add_library(video_core STATIC nx2_debug_control.cpp nx2_debug_control.h nx2_drain_check.h + nx2_list_stats.h nx2_profile.h nx2_perf.h nx2_download_trace.h diff --git a/src/video_core/buffer_cache/buffer_cache.h b/src/video_core/buffer_cache/buffer_cache.h index 279323c8f7..00cc702ba4 100644 --- a/src/video_core/buffer_cache/buffer_cache.h +++ b/src/video_core/buffer_cache/buffer_cache.h @@ -27,6 +27,7 @@ #include "video_core/host1x/gpu_device_memory_manager.h" #include "video_core/nx2.h" #include "video_core/nx2_cmdpage_diag.h" +#include "video_core/nx2_list_stats.h" #include "video_core/nx2_perf.h" #include "video_core/nx2_profile.h" #include "video_core/texture_cache/util.h" @@ -3101,6 +3102,9 @@ void BufferCache

::Nx2NoteListWrite(BufferId buffer_id, DAddr device_addr, u64 Tegra::Nx2CmdPageDiag::NoteWrite(page_addr, device_addr, size, !nx2_collect_list_copies); } + if (Tegra::Nx2ListStats::Any()) { + Tegra::Nx2ListStats::NoteWrite(page_addr, device_addr, size); + } } ++page; } @@ -3140,6 +3144,9 @@ bool BufferCache

::Nx2CopyListPages(u64 tick) { buffer.MarkUsage(copy.src_offset, copy.size); runtime.CopyBuffer(staging.buffer, buffer, std::span{©, 1}, true); recorded = true; + if (Tegra::Nx2ListStats::Any()) { + Tegra::Nx2ListStats::NoteCopy(page); + } std::scoped_lock lock{nx2_list_mutex}; Nx2ListPage& entry = nx2_list_pages[page]; if (entry.copy) { @@ -3204,7 +3211,11 @@ auto BufferCache

::Nx2ReadListCopy(DAddr device_addr, std::span out, bool // a list on none of the words GPU work wrote since the page was flushed is in guest // memory too: it waits for no copy. if (entry.synced_writes == entry.writes || - (entry.flushed && !Nx2HasListWords(entry, offset, offset + out.size()))) { + (!Tegra::Nx2ListStats::t_reference && entry.flushed && + !Nx2HasListWords(entry, offset, offset + out.size()))) { + if (entry.synced_writes != entry.writes) { + Tegra::Nx2ListStats::NoteServed(Tegra::Nx2ListStats::Served::WordsClean); + } return Nx2ListRead::Synced; } if (!current(entry)) { @@ -3219,7 +3230,12 @@ auto BufferCache

::Nx2ReadListCopy(DAddr device_addr, std::span out, bool writes = entry.copy_writes; } // Submitted with the work that wrote the page (Nx2CopyListPages's caller flushes). + const u64 wait_start = Tegra::Nx2ListStats::Any() ? Tegra::Nx2ListStats::NowNs() : 0; runtime.WaitSubmittedTick(tick); + if (Tegra::Nx2ListStats::Any()) { + Tegra::Nx2ListStats::NoteServed(Tegra::Nx2ListStats::Served::CopyWaited, wait_start, + Tegra::Nx2ListStats::NowNs() - wait_start); + } std::scoped_lock lock{nx2_list_mutex}; const auto it = nx2_list_pages.find(page); if (it == nx2_list_pages.end() || !current(it->second) || it->second.copy_tick != tick || @@ -3237,14 +3253,14 @@ void BufferCache

::Nx2ReadListWords(const Nx2ListPage& entry, DAddr device_add if constexpr (!IS_OPENGL) { // The copy holds the words GPU work wrote. Guest memory holds the others once the page // was flushed: the host wrote them, and the buffer the copy came from need not have had - // those writes uploaded. + // those writes uploaded. EDEN_NX2_LIST_VERIFY's read as before takes the whole copy. const u8* const copy = entry.copy->mapped_span.data(); const u64 end = offset + out.size(); bool all_gpu = true; for (u64 word = offset / 4; word < (end + 3) / 4 && all_gpu; ++word) { all_gpu = entry.gpu_words.test(static_cast(word)); } - if (all_gpu || !entry.flushed) { + if (all_gpu || !entry.flushed || Tegra::Nx2ListStats::t_reference) { std::memcpy(out.data(), copy + offset, out.size()); return; } diff --git a/src/video_core/dma_pusher.cpp b/src/video_core/dma_pusher.cpp index 5cfccc5f00..f8386b2aad 100644 --- a/src/video_core/dma_pusher.cpp +++ b/src/video_core/dma_pusher.cpp @@ -9,6 +9,8 @@ #include #include #include +#include +#include #include #include @@ -30,6 +32,7 @@ #include "video_core/nx2.h" #include "video_core/nx2_cmdpage_diag.h" #include "video_core/nx2_drain_check.h" +#include "video_core/nx2_list_stats.h" #include "video_core/rasterizer_interface.h" namespace Tegra { @@ -37,6 +40,32 @@ namespace Tegra { constexpr u32 MacroRegistersStart = 0xE00; constexpr u32 ComputeInline = 0x6D; +namespace { + +/// EDEN_NX2_LIST_STATS and _VERIFY: the front read `words` of a command list at `address` +/// (`device`) on a GPU-written page, as `where` tells; see Nx2ListStats::NoteRead. +void NoteListRead(MemoryManager& memory, s32 channel, GPUVAddr address, u64 device, + std::span words, VideoCore::RasterizerInterface::GpuWrittenList where, + bool after_sync, u64 backlog, u64 unpublished) { + using GpuWrittenList = VideoCore::RasterizerInterface::GpuWrittenList; + using Nx2ListStats::Served; + std::optional copy_equal; + if (Nx2ListStats::g_enabled && where == GpuWrittenList::Copied) { + std::vector guest(words.size()); + memory.ReadBlockUnsafe(address, guest.data(), guest.size()); + copy_equal = std::ranges::equal(guest, words); + } + const Served result = where == GpuWrittenList::Copied ? Served::CopyReady + : where == GpuWrittenList::Synced ? Served::PageSynced + : Served::Unknown; + constexpr u64 page_mask = Nx2ListStats::PageBytes - 1; + Nx2ListStats::NoteRead(device & ~page_mask, static_cast(device & page_mask), + static_cast(words.size()), result, after_sync, copy_equal, backlog, + unpublished, channel); +} + +} // Anonymous namespace + bool DmaPusher::IsMemoryWriter(const Engines::EngineInterface& engine, Engines::EngineTypes type, u32 method) { if (type == Engines::EngineTypes::KeplerCompute) { @@ -697,6 +726,9 @@ bool DmaPusher::ReadGpuWrittenCommands(GPUVAddr address, u32 count) { } bool past_sync = !split.Active(); using GpuWrittenList = VideoCore::RasterizerInterface::GpuWrittenList; + if (Nx2ListStats::Any()) [[unlikely]] { + Nx2ListStats::BeginList(count); + } command_headers.resize_destructive(count); u8* const out = reinterpret_cast(command_headers.data()); for (GPUVAddr chunk = address; chunk < end;) { @@ -708,7 +740,11 @@ bool DmaPusher::ReadGpuWrittenCommands(GPUVAddr address, u32 count) { } GpuWrittenList where = GpuWrittenList::Synced; if (Tegra::Nx2IsGpuWritten(*device)) { + const bool stats = Nx2ListStats::g_enabled && split.Active(); + const u64 backlog = stats ? split.Backlog() : 0; + const u64 unpublished = stats ? split.Unpublished() : 0; where = rasterizer->ReadGpuWrittenCommandList(*device, part, past_sync); + const bool after_sync = where == GpuWrittenList::NeedsSync; if (where == GpuWrittenList::NeedsSync) { const auto wait_start = Nx2CmdPageDiag::g_enabled ? std::chrono::steady_clock::now() @@ -741,6 +777,10 @@ bool DmaPusher::ReadGpuWrittenCommands(GPUVAddr address, u32 count) { } } } + if (Nx2ListStats::Any()) [[unlikely]] { + NoteListRead(memory_manager, channel_state.bind_id, chunk, *device, part, where, + after_sync, backlog, unpublished); + } } if (where == GpuWrittenList::Unknown) { return false; @@ -750,9 +790,68 @@ bool DmaPusher::ReadGpuWrittenCommands(GPUVAddr address, u32 count) { } chunk = chunk_end; } + if (Nx2ListStats::g_verify) [[unlikely]] { + VerifyGpuWrittenCommands(address, count); + } return true; } +void DmaPusher::VerifyGpuWrittenCommands(GPUVAddr address, u32 count) { + // The list again as the front read lists before it tracked the words GPU work wrote: after + // every dispatch and copy sent so far, from the pages' whole copies, or drained and flushed. + using GpuWrittenList = VideoCore::RasterizerInterface::GpuWrittenList; + const size_t bytes = size_t{count} * sizeof(u32); + std::vector parsed(count); + std::memcpy(parsed.data(), command_headers.data(), bytes); + auto& split = VideoCommon::GpuSplit::Get(); + if (split.Active()) { + split.WaitForWriters(); + } + bool past_sync = !split.Active(); + std::vector reference(count); + bool reference_drained = false; + { + const Nx2ListStats::ReferenceScope reference_read; + u8* const out = reinterpret_cast(reference.data()); + const GPUVAddr end = address + bytes; + for (GPUVAddr chunk = address; chunk < end && !reference_drained;) { + const GPUVAddr chunk_end = (std::min)(end, (chunk & ~GPUVAddr{0xfff}) + 0x1000); + const std::span part{out + (chunk - address), + static_cast(chunk_end - chunk)}; + const auto device = memory_manager.GpuToCpuAddress(chunk); + GpuWrittenList where = device ? GpuWrittenList::Synced : GpuWrittenList::Unknown; + if (device && Tegra::Nx2IsGpuWritten(*device)) { + where = rasterizer->ReadGpuWrittenCommandList(*device, part, past_sync); + if (where == GpuWrittenList::NeedsSync) { + split.WaitForSync(); + past_sync = true; + where = rasterizer->ReadGpuWrittenCommandList(*device, part, true); + } + } + if (where == GpuWrittenList::Synced) { + memory_manager.ReadBlockUnsafe(chunk, part.data(), part.size()); + } + reference_drained = where == GpuWrittenList::Unknown; + chunk = chunk_end; + } + } + const auto read_drained = [&](std::vector& words) { + FlushGpuWrittenCommands(address, bytes); + memory_manager.ReadBlockUnsafe(address, words.data(), bytes); + }; + if (reference_drained) { + read_drained(reference); + } + std::vector drained; + if (!reference_drained && reference != parsed) { + drained.resize(count); + read_drained(drained); + } + Nx2ListStats::NoteVerify(address, parsed, reference, drained, reference_drained); + // The front parses the words read last. + std::memcpy(command_headers.data(), drained.empty() ? reference.data() : drained.data(), bytes); +} + bool DmaPusher::FlushGpuWrittenCommands(GPUVAddr address, size_t size) { if (!g_nx2_ampere.load(std::memory_order_relaxed) || size == 0) { return false; diff --git a/src/video_core/dma_pusher.h b/src/video_core/dma_pusher.h index 227a97cdcd..f0f2094802 100644 --- a/src/video_core/dma_pusher.h +++ b/src/video_core/dma_pusher.h @@ -241,6 +241,9 @@ class DmaPusher final { /// since. False when not all are current (or the list isn't GPU-written): the caller drains, /// flushes and reads guest memory. bool ReadGpuWrittenCommands(GPUVAddr address, u32 count); + /// EDEN_NX2_LIST_VERIFY: reads the list ReadGpuWrittenCommands just read again as before + /// (Nx2ListStats) and leaves the words read last in command_headers. + void VerifyGpuWrittenCommands(GPUVAddr address, u32 count); /// Brings command list words the GPU wrote (still only in host buffers) to guest memory. bool FlushGpuWrittenCommands(GPUVAddr address, size_t size); diff --git a/src/video_core/gpu_split.h b/src/video_core/gpu_split.h index be799a4be3..f1766759a1 100644 --- a/src/video_core/gpu_split.h +++ b/src/video_core/gpu_split.h @@ -271,6 +271,15 @@ class GpuSplit { u64 device_address) const; /// Whether compute or copy work sent so far is still queued (EDEN_NX2_DRAIN_CHECK). [[nodiscard]] bool WritersPending() const; + /// Front thread: the ring words sent that the back had not run when it last stored its + /// position (every 4096 words of a batch and at its end) (EDEN_NX2_LIST_STATS). + [[nodiscard]] u64 Backlog() const { + return write_position - consumed.load(std::memory_order_acquire); + } + /// Front thread: the ring words sent that the back does not see yet (EDEN_NX2_LIST_STATS). + [[nodiscard]] u64 Unpublished() const { + return write_position - published_local; + } /// The entry just sent (Writes::Noted) writes [address, address + size) of `memory`. void NoteWrite(const Tegra::MemoryManager& memory, GPUVAddr address, u64 size); diff --git a/src/video_core/gpu_thread.cpp b/src/video_core/gpu_thread.cpp index 8c2e5973d8..81f7d0e4b7 100644 --- a/src/video_core/gpu_thread.cpp +++ b/src/video_core/gpu_thread.cpp @@ -25,6 +25,7 @@ #include "video_core/gpu_split.h" #include "video_core/gpu_wait_stats.h" #include "video_core/nx2_drain_check.h" +#include "video_core/nx2_list_stats.h" #include "video_core/nx2_profile.h" #include "video_core/host1x/host1x.h" #include "video_core/renderer_base.h" @@ -119,6 +120,7 @@ void ThreadManager::StartThread(VideoCore::RendererBase& renderer, Core::Fronten auto* const rasterizer_ptr = renderer.ReadRasterizer(); if (auto* submit_list = std::get_if(&next.data)) { Profile::Scope scope{Profile::Zone::Push}; + Tegra::Nx2ListStats::NoteListQueued(submit_list->channel, -1); scheduler.Push(system.GPU(), submit_list->channel, std::move(submit_list->entries)); } else if (std::holds_alternative(next.data)) { split.Defer([gpu = &system.GPU()] { gpu->TickWork(); }); @@ -149,6 +151,7 @@ void ThreadManager::StartThread(VideoCore::RendererBase& renderer, Core::Fronten } if (auto* submit_list = std::get_if(&next.data)) { Profile::Scope scope{Profile::Zone::Push}; + Tegra::Nx2ListStats::NoteListQueued(submit_list->channel, -1); scheduler.Push(system.GPU(), submit_list->channel, std::move(submit_list->entries)); } else if (std::holds_alternative(next.data)) { Profile::Scope scope{Profile::Zone::Tick}; @@ -171,6 +174,7 @@ void ThreadManager::StartThread(VideoCore::RendererBase& renderer, Core::Fronten } void ThreadManager::SubmitList(s32 channel, Tegra::CommandList&& entries, bool is_async) { + Tegra::Nx2ListStats::NoteListQueued(channel, 1); PushCommand(SubmitListCommand(channel, std::move(entries)), false, is_async); } diff --git a/src/video_core/nx2_list_stats.h b/src/video_core/nx2_list_stats.h new file mode 100644 index 0000000000..47532469de --- /dev/null +++ b/src/video_core/nx2_list_stats.h @@ -0,0 +1,580 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "common/common_types.h" +#include "common/logging.h" +#include "video_core/nx2.h" + +// EDEN_NX2_LIST_STATS: what the front's reads of command lists on GPU-written pages +// (DmaPusher::ReadGpuWrittenCommands) wait for. For each list its size; for each of its pages how +// BufferCache::Nx2ReadListCopy served it, the GPU work that last wrote the page (a dispatch's +// shader, a copy engine copy's source, or draws) and whether the copy held other bytes than guest +// memory; for each copy the front waited for: how long, how long before the back had recorded it, +// how much the back still had to run (and of that, not yet published) and whether other channels +// had lists queued. Every 10 s the front logs the counts. +// +// EDEN_NX2_LIST_VERIFY: each list read so is read again as the front read lists before it tracked +// the words GPU work wrote (after every dispatch and copy sent so far, from the pages' whole +// copies, or drained and flushed) and, when the two differ, drained and flushed as well, which +// tells the stale read. Differences are counted and the first ones logged; the front parses the +// words read last. +namespace Tegra::Nx2ListStats { + +// Plain flags: checked at every GPU-written list and buffer write mark. +inline bool g_enabled = std::getenv("EDEN_NX2_LIST_STATS") != nullptr; +inline bool g_verify = std::getenv("EDEN_NX2_LIST_VERIFY") != nullptr; + +/// Called by VideoCore::CreateGPU: static initialization runs before the frontend loads +/// nx2_switches.txt, so a switch set only there takes effect from that point on. +inline void ReadSwitches() { + g_enabled = std::getenv("EDEN_NX2_LIST_STATS") != nullptr; + g_verify = std::getenv("EDEN_NX2_LIST_VERIFY") != nullptr; +} + +[[nodiscard]] inline bool Any() { + return g_enabled || g_verify; +} + +[[nodiscard]] inline u64 NowNs() { + return static_cast(std::chrono::duration_cast( + std::chrono::steady_clock::now().time_since_epoch()) + .count()); +} + +inline constexpr u64 PageBytes = u64{1} << Nx2WrittenPageBits; + +/// How BufferCache::Nx2ReadListCopy served a list's bytes on one page. +enum class Served : u32 { + PageSynced, ///< From guest memory: the front flushed the page, no GPU work wrote it since + WordsClean, ///< From guest memory: no GPU work wrote the list's words since the flush + CopyReady, ///< From the page's copy, which the GPU had finished + CopyWaited, ///< From the page's copy, once the GPU finished it + Unknown, ///< Nowhere: the front drains and flushes + Count, +}; + +inline constexpr std::array(Served::Count)> served_names{ + "page synced", "words clean", "copy ready", "copy waited", "unknown", +}; + +/// The GPU work that last wrote a command list page (BufferCache::Nx2NoteListWrite). +enum class Writer : u32 { + None, ///< No write noted since the stats began + Dispatch, ///< A compute dispatch: the id is its shader's hash + Copy, ///< A copy engine copy: the id is its source GPU address + Draw, ///< Draws and the other GPU writes + Count, +}; + +inline constexpr std::array(Writer::Count)> writer_names{ + "none", + "dispatch", + "copy", + "draw", +}; + +struct WriterTag { + Writer writer{Writer::Draw}; + u64 id{}; +}; + +/// The back: the dispatch or copy the GPU writes noted now come from. +inline thread_local WriterTag t_writer{}; + +/// The back, around a dispatch or a copy engine copy: tags the GPU writes noted meanwhile. +class WriterScope { +public: + explicit WriterScope(Writer writer, u64 id) : previous{t_writer} { + t_writer = WriterTag{writer, id}; + } + ~WriterScope() { + t_writer = previous; + } + WriterScope(const WriterScope&) = delete; + WriterScope& operator=(const WriterScope&) = delete; + +private: + WriterTag previous; +}; + +/// What Nx2ReadListCopy told about its last read where its result does not (NoteServed). +struct ServedDetail { + bool valid{}; + Served served{Served::Unknown}; + u64 wait_start_ns{}; ///< CopyWaited: when the front started to wait + u64 waited_ns{}; +}; +inline thread_local ServedDetail t_served{}; + +/// The front, in Nx2ReadListCopy: how it served the read. +inline void NoteServed(Served served, u64 wait_start_ns = 0, u64 waited_ns = 0) { + if (Any()) { + t_served = ServedDetail{true, served, wait_start_ns, waited_ns}; + } +} + +/// The front, while EDEN_NX2_LIST_VERIFY reads a list as before: Nx2ReadListCopy serves guest +/// memory only for pages flushed and written by no GPU work since, else the page's whole copy. +inline thread_local bool t_reference{}; + +class ReferenceScope { +public: + ReferenceScope() { + t_reference = true; + } + ~ReferenceScope() { + t_reference = false; + } + ReferenceScope(const ReferenceScope&) = delete; + ReferenceScope& operator=(const ReferenceScope&) = delete; +}; + +inline constexpr size_t ChannelSlots = 16; +/// Command lists submitted to the GPU thread (ThreadManager::SubmitList) that it has not taken +/// yet, by channel. +inline std::array, ChannelSlots> g_queued_lists{}; + +[[nodiscard]] inline size_t ChannelSlot(s32 channel) { + return static_cast(static_cast(channel)) % ChannelSlots; +} + +/// Any thread: a list of `channel` was queued for the GPU thread (`delta` 1) or taken (-1). +inline void NoteListQueued(s32 channel, s32 delta) { + if (g_enabled) { + g_queued_lists[ChannelSlot(channel)].fetch_add(delta, std::memory_order_relaxed); + } +} + +/// Whether lists of channels other than `channel` wait for the GPU thread. +[[nodiscard]] inline bool OtherChannelsQueued(s32 channel) { + const size_t own = ChannelSlot(channel); + for (size_t slot = 0; slot < ChannelSlots; ++slot) { + if (slot != own && g_queued_lists[slot].load(std::memory_order_relaxed) > 0) { + return true; + } + } + return false; +} + +struct Waits { + u64 reads{}; + u64 waits{}; + u64 wait_ns{}; + + void Add(bool waited, u64 ns) { + ++reads; + if (waited) { + ++waits; + wait_ns += ns; + } + } +}; + +struct PageInfo { + WriterTag writer{Writer::None, 0}; + u32 write_begin{}; ///< The last GPU write's bytes on the page + u32 write_end{}; + u64 write_size{}; ///< The last GPU write's whole size + u64 copy_ns{}; ///< When the back recorded the page's last copy + Waits waits; ///< This period's reads + u32 list_begin{~u32{0}}; ///< The bytes of the page this period's lists read + u32 list_end{}; +}; + +struct WriterStats { + WriterTag writer; + Waits waits; ///< This period's reads of the pages it wrote last + u64 writes{}; ///< This period's GPU writes to command list pages + u64 bytes{}; ///< Their bytes on those pages +}; + +/// How a list differed from its read as before (EDEN_NX2_LIST_VERIFY). +enum class Difference : u32 { + OldStale, ///< The drained read matches the new read: the read as before was stale + NewStale, ///< The drained read matches the read as before: the new read was stale + AllDiffer, ///< All three differ + Drained, ///< The read as before drained and flushed: the new read was stale + Count, +}; + +inline constexpr std::array(Difference::Count)> difference_names{ + "the read as before was stale", + "the new read was stale", + "all three differ", + "the new read differs from the drained one", +}; + +/// One page of the list the front reads, for EDEN_NX2_LIST_VERIFY's log. +struct ListPart { + u64 page{}; + u32 offset{}; + u32 size{}; + Served served{Served::Unknown}; +}; + +inline constexpr std::array list_size_limits{16, 64, 256, 1024, 4096}; +inline constexpr std::array wait_limits_us{100, 250, 500, 1000}; + +/// The counts of one logging period. +struct Period { + u64 lists{}; + u64 list_words{}; + std::array list_sizes{}; + std::array(Served::Count)> served{}; + std::array(Writer::Count)> by_writer{}; + u64 after_sync{}; ///< Reads that first waited for the work before the last sync + u64 wait_ns{}; + u64 wait_max_ns{}; + std::array wait_sizes{}; + u64 copies_compared{}; + u64 copies_equal{}; + u64 waits_backlog{}; ///< Copy waits the back had work for + u64 backlog_words{}; + u64 waits_unpublished{}; ///< Copy waits with work the back did not see yet + u64 unpublished_words{}; + u64 waits_other_channels{}; + u64 waits_recorded{}; ///< Copy waits whose copy's record time is known + u64 since_recorded_ns{}; + + u64 verified{}; + u64 verified_drained{}; ///< Read as before by draining and flushing + u64 differ{}; + u64 differ_words{}; + std::array(Difference::Count)> differences{}; +}; + +struct State { + std::mutex mutex; + std::unordered_map pages; + std::unordered_map writers; + std::vector parts; ///< The pages of the list the front reads + Period period; + u64 verified_total{}; + u64 differ_total{}; + u32 differ_logged{}; + std::chrono::steady_clock::time_point last_log{std::chrono::steady_clock::now()}; +}; + +inline State& Get() { + static State state; + return state; +} + +[[nodiscard]] inline u64 WriterKey(const WriterTag& tag) { + return (static_cast(tag.writer) << 60) ^ tag.id; +} + +[[nodiscard]] inline std::string WriterText(const WriterTag& tag) { + switch (tag.writer) { + case Writer::Dispatch: + return fmt::format("dispatch {:#018x}", tag.id); + case Writer::Copy: + return fmt::format("copy from {:#x}", tag.id); + default: + return writer_names[static_cast(tag.writer)]; + } +} + +/// The index of the first of `limits` `value` does not exceed, else limits.size(). +[[nodiscard]] inline size_t Bucket(u64 value, std::span limits) { + size_t bucket = 0; + while (bucket < limits.size() && value > limits[bucket]) { + ++bucket; + } + return bucket; +} + +[[nodiscard]] inline double Ms(u64 ns) { + return static_cast(ns) / 1e6; +} + +[[nodiscard]] inline double Average(u64 total, u64 count, double unit) { + return count == 0 ? 0.0 : static_cast(total) / static_cast(count) / unit; +} + +/// The back: GPU work wrote [device_addr, device_addr + size) over the command list page at +/// `page_addr` (BufferCache::Nx2NoteListWrite). +inline void NoteWrite(u64 page_addr, u64 device_addr, u64 size) { + const u64 begin = (std::max)(device_addr, page_addr) - page_addr; + const u64 end = (std::min)(device_addr + size, page_addr + PageBytes) - page_addr; + const WriterTag tag = t_writer; + State& state = Get(); + std::scoped_lock lock{state.mutex}; + PageInfo& page = state.pages[page_addr]; + page.writer = tag; + page.write_begin = static_cast(begin); + page.write_end = static_cast(end); + page.write_size = size; + WriterStats& stats = state.writers[WriterKey(tag)]; + stats.writer = tag; + ++stats.writes; + stats.bytes += end - begin; +} + +/// The back recorded the copy of the command list page at `page_addr` (Nx2CopyListPages). +inline void NoteCopy(u64 page_addr) { + const u64 now = NowNs(); + State& state = Get(); + std::scoped_lock lock{state.mutex}; + state.pages[page_addr].copy_ns = now; +} + +inline void LogStats(const State& state, double seconds) { + const Period& counts = state.period; + const auto writer_reads = [&counts](Writer kind) { + const Waits& entry = counts.by_writer[static_cast(kind)]; + return fmt::format("{} {}/{}/{:.1f}", writer_names[static_cast(kind)], entry.reads, + entry.waits, Ms(entry.wait_ns)); + }; + const u64 copy_waits = counts.served[static_cast(Served::CopyWaited)]; + LOG_INFO( + HW_GPU, + "list stats {:.1f}s: {} lists of {} words (up to 16 words {}, 64 {}, 256 {}, 1024 {}, " + "4096 {}, more {}) | page reads: page synced {}, words clean {}, copy ready {}, copy " + "waited {}, unknown {}; after a sync wait {} | copy waits {:.1f} ms, average {:.0f} " + "us, max {:.0f} us (up to 100 us {}, 250 {}, 500 {}, 1000 {}, more {}) | by the " + "page's last writer (reads/waits/ms): {}, {}, {}, {} | copies equal to guest memory " + "{} of {} | at copy waits: the back had work {} (average {:.0f} words), not yet " + "published {} (average {:.0f} words), other channels had lists queued {}, the copy " + "was recorded {:.0f} us before on average ({} known)", + seconds, counts.lists, counts.list_words, counts.list_sizes[0], counts.list_sizes[1], + counts.list_sizes[2], counts.list_sizes[3], counts.list_sizes[4], counts.list_sizes[5], + counts.served[0], counts.served[1], counts.served[2], counts.served[3], counts.served[4], + counts.after_sync, Ms(counts.wait_ns), Average(counts.wait_ns, copy_waits, 1e3), + static_cast(counts.wait_max_ns) / 1e3, counts.wait_sizes[0], counts.wait_sizes[1], + counts.wait_sizes[2], counts.wait_sizes[3], counts.wait_sizes[4], + writer_reads(Writer::Dispatch), writer_reads(Writer::Copy), writer_reads(Writer::Draw), + writer_reads(Writer::None), counts.copies_equal, counts.copies_compared, + counts.waits_backlog, Average(counts.backlog_words, counts.waits_backlog, 1.0), + counts.waits_unpublished, Average(counts.unpublished_words, counts.waits_unpublished, 1.0), + counts.waits_other_channels, Average(counts.since_recorded_ns, counts.waits_recorded, 1e3), + counts.waits_recorded); + + std::vector writers; + for (const auto& entry : state.writers) { + writers.push_back(&entry.second); + } + std::ranges::sort(writers, [](const WriterStats* lhs, const WriterStats* rhs) { + return std::tie(lhs->waits.wait_ns, lhs->waits.reads, lhs->writes) > + std::tie(rhs->waits.wait_ns, rhs->waits.reads, rhs->writes); + }); + std::string writers_text; + for (size_t i = 0; i < writers.size() && i < 8; ++i) { + const WriterStats& stats = *writers[i]; + writers_text += fmt::format( + "{}{}: {} reads, {} waits in {:.1f} ms, {} writes of {} bytes on list pages", + writers_text.empty() ? "" : "; ", WriterText(stats.writer), stats.waits.reads, + stats.waits.waits, Ms(stats.waits.wait_ns), stats.writes, stats.bytes); + } + if (!writers_text.empty()) { + LOG_INFO(HW_GPU, "list stats writers: {}", writers_text); + } + + std::vector> pages; + for (const auto& entry : state.pages) { + if (entry.second.waits.reads != 0) { + pages.emplace_back(entry.first, &entry.second); + } + } + std::ranges::sort(pages, [](const auto& lhs, const auto& rhs) { + return std::tie(lhs.second->waits.wait_ns, lhs.second->waits.reads) > + std::tie(rhs.second->waits.wait_ns, rhs.second->waits.reads); + }); + std::string pages_text; + for (size_t i = 0; i < pages.size() && i < 8; ++i) { + const PageInfo& info = *pages[i].second; + pages_text += fmt::format( + "{}{:#x}: {} reads, {} waits in {:.1f} ms, lists at {:#x}-{:#x}, last written " + "{:#x}-{:#x} (of {} bytes) by {}", + pages_text.empty() ? "" : "; ", pages[i].first, info.waits.reads, info.waits.waits, + Ms(info.waits.wait_ns), info.list_begin, info.list_end, info.write_begin, + info.write_end, info.write_size, WriterText(info.writer)); + } + if (!pages_text.empty()) { + LOG_INFO(HW_GPU, "list stats pages: {}", pages_text); + } +} + +inline void LogVerify(const State& state, double seconds) { + const Period& counts = state.period; + LOG_INFO(HW_GPU, + "list verify {:.1f}s: {} lists read again ({} drained and flushed), {} differ in {} " + "words: {} {}, {} {}, {} {}, {} {}; since the start {} of {} differ", + seconds, counts.verified, counts.verified_drained, counts.differ, counts.differ_words, + difference_names[0], counts.differences[0], difference_names[1], counts.differences[1], + difference_names[2], counts.differences[2], difference_names[3], counts.differences[3], + state.differ_total, state.verified_total); +} + +/// The front: logs every 10 s. +inline void MaybeLog() { + State& state = Get(); + const auto now = std::chrono::steady_clock::now(); + std::scoped_lock lock{state.mutex}; + const double seconds = std::chrono::duration(now - state.last_log).count(); + if (seconds < 10.0) { + return; + } + state.last_log = now; + if (g_enabled) { + LogStats(state, seconds); + } + if (g_verify) { + LogVerify(state, seconds); + } + state.period = {}; + state.writers.clear(); + for (auto& entry : state.pages) { + entry.second.waits = {}; + entry.second.list_begin = ~u32{0}; + entry.second.list_end = 0; + } +} + +/// The front starts reading a list of `words` on GPU-written pages. +inline void BeginList(u32 words) { + MaybeLog(); + t_served = {}; + State& state = Get(); + std::scoped_lock lock{state.mutex}; + state.parts.clear(); + ++state.period.lists; + state.period.list_words += words; + ++state.period.list_sizes[Bucket(words, list_size_limits)]; +} + +/// The front read `size` bytes at `offset` of the command list page at `page_addr`. `result`: as +/// Nx2ReadListCopy's result tells (PageSynced, CopyReady or Unknown; NoteServed tells more). +/// `after_sync`: it first waited for the work before the last sync (a page draws write). +/// `copy_equal`: for a read from the copy, whether guest memory held the same bytes. `backlog` +/// and `unpublished`: the ring words the back had not run before the read, and of them those it +/// did not see. `channel`: the list's. +inline void NoteRead(u64 page_addr, u32 offset, u32 size, Served result, bool after_sync, + std::optional copy_equal, u64 backlog, u64 unpublished, s32 channel) { + const ServedDetail detail = t_served; + t_served = {}; + const Served served = result == Served::Unknown || !detail.valid ? result : detail.served; + const bool waited = served == Served::CopyWaited; + const u64 wait_ns = waited ? detail.waited_ns : 0; + const bool other_channels = waited && OtherChannelsQueued(channel); + State& state = Get(); + std::scoped_lock lock{state.mutex}; + state.parts.push_back(ListPart{page_addr, offset, size, served}); + if (!g_enabled) { + return; + } + Period& counts = state.period; + ++counts.served[static_cast(served)]; + counts.after_sync += after_sync ? 1 : 0; + PageInfo& page = state.pages[page_addr]; + page.waits.Add(waited, wait_ns); + page.list_begin = (std::min)(page.list_begin, offset); + page.list_end = (std::max)(page.list_end, offset + size); + counts.by_writer[static_cast(page.writer.writer)].Add(waited, wait_ns); + if (page.writer.writer != Writer::None) { + WriterStats& stats = state.writers[WriterKey(page.writer)]; + stats.writer = page.writer; + stats.waits.Add(waited, wait_ns); + } + if (copy_equal) { + ++counts.copies_compared; + counts.copies_equal += *copy_equal ? 1 : 0; + } + if (!waited) { + return; + } + counts.wait_ns += wait_ns; + counts.wait_max_ns = (std::max)(counts.wait_max_ns, wait_ns); + ++counts.wait_sizes[Bucket(wait_ns / 1000, wait_limits_us)]; + if (backlog != 0) { + ++counts.waits_backlog; + counts.backlog_words += backlog; + } + if (unpublished != 0) { + ++counts.waits_unpublished; + counts.unpublished_words += unpublished; + } + counts.waits_other_channels += other_channels ? 1 : 0; + if (page.copy_ns != 0 && page.copy_ns <= detail.wait_start_ns) { + ++counts.waits_recorded; + counts.since_recorded_ns += detail.wait_start_ns - page.copy_ns; + } +} + +/// EDEN_NX2_LIST_VERIFY: the front read the list at `address` as `parsed`, then as before as +/// `reference` (`reference_drained`: by draining and flushing) and, when those differ, drained +/// and flushed as `drained` (else empty). +inline void NoteVerify(u64 address, std::span parsed, std::span reference, + std::span drained, bool reference_drained) { + u64 differing = 0; + size_t first = parsed.size(); + for (size_t i = 0; i < parsed.size() && i < reference.size(); ++i) { + if (parsed[i] != reference[i]) { + ++differing; + first = (std::min)(first, i); + } + } + State& state = Get(); + std::scoped_lock lock{state.mutex}; + Period& counts = state.period; + ++counts.verified; + ++state.verified_total; + counts.verified_drained += reference_drained ? 1 : 0; + if (differing == 0) { + return; + } + Difference kind = Difference::Drained; + if (!reference_drained) { + if (std::ranges::equal(drained, parsed)) { + kind = Difference::OldStale; + } else if (std::ranges::equal(drained, reference)) { + kind = Difference::NewStale; + } else { + kind = Difference::AllDiffer; + } + } + ++counts.differ; + ++state.differ_total; + counts.differ_words += differing; + ++counts.differences[static_cast(kind)]; + if (state.differ_logged >= 32) { + return; + } + ++state.differ_logged; + std::string parts_text; + for (const ListPart& part : state.parts) { + const auto it = state.pages.find(part.page); + const PageInfo info = it != state.pages.end() ? it->second : PageInfo{}; + parts_text += fmt::format("{}{:#x}+{:#x} ({} bytes) {}, last written {:#x}-{:#x} by {}", + parts_text.empty() ? "" : "; ", part.page, part.offset, part.size, + served_names[static_cast(part.served)], info.write_begin, + info.write_end, WriterText(info.writer)); + } + LOG_WARNING(HW_GPU, + "list verify: list at {:#x} ({} words) differs from its read as before in {} " + "words, the first word {}: {:#010x}, as before {:#010x}{}: {}; its pages: {}", + address, parsed.size(), differing, first, parsed[first], reference[first], + drained.size() > first ? fmt::format(", drained {:#010x}", drained[first]) + : std::string{}, + difference_names[static_cast(kind)], parts_text); +} + +} // namespace Tegra::Nx2ListStats diff --git a/src/video_core/renderer_vulkan/vk_rasterizer.cpp b/src/video_core/renderer_vulkan/vk_rasterizer.cpp index a085d7600d..ac669b3169 100644 --- a/src/video_core/renderer_vulkan/vk_rasterizer.cpp +++ b/src/video_core/renderer_vulkan/vk_rasterizer.cpp @@ -74,6 +74,7 @@ #include "video_core/nx2.h" #include "video_core/nx2_debug_control.h" #include "video_core/nx2_drain_check.h" +#include "video_core/nx2_list_stats.h" #include "video_core/nx2_profile.h" namespace Vulkan { @@ -1757,6 +1758,8 @@ void RasterizerVulkan::DispatchCompute() { } std::scoped_lock lock{texture_cache.mutex, buffer_cache.mutex}; std::optional byte_addressed_shadow; + const Tegra::Nx2ListStats::WriterScope list_writer{Tegra::Nx2ListStats::Writer::Dispatch, + pipeline->ShaderHash()}; buffer_cache.Nx2CollectListCopies(true); SCOPE_EXIT { buffer_cache.Nx2CollectListCopies(false); @@ -3124,6 +3127,8 @@ bool AccelerateDMA::BufferClear(GPUVAddr src_address, u64 amount, u32 value) { bool AccelerateDMA::BufferCopy(GPUVAddr src_address, GPUVAddr dest_address, u64 amount) { std::scoped_lock lock{buffer_cache.mutex}; + const Tegra::Nx2ListStats::WriterScope list_writer{Tegra::Nx2ListStats::Writer::Copy, + src_address}; buffer_cache.Nx2CollectListCopies(true); const bool copied = buffer_cache.DMACopy(src_address, dest_address, amount); if (buffer_cache.Nx2CopyListPages(scheduler.CurrentTick())) { diff --git a/src/video_core/video_core.cpp b/src/video_core/video_core.cpp index 2dbfd80646..f7a8ec608c 100644 --- a/src/video_core/video_core.cpp +++ b/src/video_core/video_core.cpp @@ -16,6 +16,7 @@ #include "video_core/nx2_cmdpage_diag.h" #include "video_core/nx2_download_trace.h" #include "video_core/nx2_drain_check.h" +#include "video_core/nx2_list_stats.h" #include "video_core/nx2_profile.h" #include "video_core/nx2_read_trace.h" #include "video_core/renderer_base.h" @@ -58,6 +59,7 @@ void CreateGPU(std::optional& gpu, Core::Frontend::EmuWindow& emu_wi VideoCommon::Nx2DownloadTrace::ReadSwitch(); VideoCommon::Nx2DrainCheck::ReadSwitch(); VideoCommon::ReadTrace::ReadSwitch(); + Tegra::Nx2ListStats::ReadSwitches(); const auto nvdec_value = Settings::values.nvdec_emulation.GetValue(); const bool use_nvdec = nvdec_value != Settings::NvdecEmulation::Off; From 188baa1448745ecbf30cdc2cdaca82de5b00461f Mon Sep 17 00:00:00 2001 From: jpkto <255363096+jpkto@users.noreply.github.com> Date: Sun, 8 Nov 2026 00:17:53 +0100 Subject: [PATCH 028/105] video_core: decide Switch 2 acquires from the split's semaphore view Semaphore acquires and the polls of suspended ones no longer drain the split. Fix 2 only checked greater-or-equal acquires without the drain, and drained for the rest, for words shaders write, for releases it couldn't value, and whenever the value wasn't met yet. In Ocarina of Time that was about 4,800 drains a second at the acquire site (182 ms/s on the GPU thread), most of them at words QMD releases advance. Now the front decides each acquire (equal, greater-or-equal, mask) from what it can see: - The last release sent to the word, while the back hasn't run it: met passes (with NoteSync, as before), not met suspends. - A release the front can't value (a 3D counter report, a reduction it can't compute): a positioned wait for that release only, then memory. - Otherwise memory. Greater-or-equal holds once met, since the words only advance. Another operation holds only for the value the word has when the channel reaches it: compute and copy work sent before the acquire may release it unseen (QMD releases), so a met value waits for that work first (a positioned wait, not a drain). - Words on GPU-written pages that the buffer cache holds GPU data for: memory still decides greater-or-equal acquires it meets. Otherwise the back flushes the word, the channel suspends, and the polls read memory once the flush ran. This keeps the MustFlushRegion check for semaphores shaders write. An unmet acquire registers its waiter and suspends the channel. A release the back runs latches it (Nx2SemaphoreWaiters::OnWrite). The polls decide the same way without draining, so other releases resume it from memory or the view. The fence work is a plain Defer. It is sent once the back has run everything before the acquire and the acquire still waits, which is where the drain path sent it. Acquires that the queued work releases send none. SEM_EXECUTE acquires never blocked; they drop their drain too, keep the NoteSync it implied, and are checked for their warning only. High GPU accuracy, delayed fences, macros on the back and unmapped semaphores keep the drain path. Compute report reductions are valued on the front when the word's value before them is known: the last release sent to it, or memory when no compute or copy work is queued. That holds only for reports sent directly, since a macro's own launches count as work only once the macro ran. GpuSplit gains a position API: Position, Ran, LastWriterPosition, WaitForPosition, and the position of a queued semaphore write. The GPU thread's poll loop counts a resumed channel as progress, as its comment says, so it keeps polling fast while channels move. Tests: a Catch2 case and the standalone split harness (real back thread) cover the positioned wait for a queued release, which leaves the work after it running. Rebased: the back macros commit before this one added the same GpuSplit::Position(), so this commit keeps that one instead of adding a second (two definitions did not compile, here and in the next commit, until the commit after them merged the position waits). Co-Authored-By: Claude Opus 5.5 --- src/tests/video_core/maxwell_fast_method.cpp | 63 ++++ src/video_core/control/scheduler.cpp | 7 +- src/video_core/control/scheduler.h | 4 +- src/video_core/engines/kepler_compute.cpp | 36 ++- src/video_core/engines/kepler_compute.h | 3 +- src/video_core/engines/puller.cpp | 309 ++++++++++++------- src/video_core/engines/puller.h | 16 + src/video_core/gpu_split.cpp | 2 +- src/video_core/gpu_split.h | 27 +- src/video_core/gpu_thread.cpp | 4 +- 10 files changed, 342 insertions(+), 129 deletions(-) diff --git a/src/tests/video_core/maxwell_fast_method.cpp b/src/tests/video_core/maxwell_fast_method.cpp index 582ebb3bfd..9aba1837c1 100644 --- a/src/tests/video_core/maxwell_fast_method.cpp +++ b/src/tests/video_core/maxwell_fast_method.cpp @@ -1020,6 +1020,69 @@ TEST_CASE("Split remembers the semaphore writes its back has not run yet", REQUIRE_FALSE(split.QueuedSemaphoreWriteAt(0x2000).has_value()); } +TEST_CASE("Split waits for a semaphore write's position, not for the work sent after it", + "[video_core][gpu_split_dependencies]") { + using Access = VideoCommon::GpuSplitTestAccess; + Fixture f; + f.system.Initialize(); + VideoCommon::GpuSplit split; + std::array, 2> release{}; + Access::Start(split, f.system); + SCOPE_EXIT { + split.Stop(); + }; + // Runs before Stop: a failed check must not leave the back in the entries below. + SCOPE_EXIT { + release[0] = true; + release[1] = true; + }; + const auto blocking = [&split](std::atomic& flag) { + split.Defer([&flag] { + while (!flag.load()) { + std::this_thread::yield(); + } + }); + }; + // A release the front can't value (a reduction, a counter) sent by compute work, behind an + // entry the back is still in, then more work behind another. + blocking(release[0]); + split.Defer([] {}); + split.NoteSemaphoreWrite(0x1000, 0, false); + split.NoteWriter(); + const u64 release_position = split.Position(); + REQUIRE(split.LastWriterPosition() == release_position); + blocking(release[1]); + split.Publish(); + const auto queued = split.QueuedSemaphoreWriteAt(0x1000); + REQUIRE(queued.has_value()); + REQUIRE(queued->position == release_position); + REQUIRE_FALSE(split.Ran(release_position)); + std::atomic done{}; + bool waited{}; + std::thread waiter([&] { + waited = split.WaitForPosition(release_position, VideoCommon::DrainSite::Acquire); + done = true; + }); + std::this_thread::sleep_for(std::chrono::milliseconds{20}); + REQUIRE_FALSE(done.load()); + release[0] = true; + const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds{5}; + while (!done.load() && std::chrono::steady_clock::now() < deadline) { + std::this_thread::sleep_for(std::chrono::milliseconds{1}); + } + REQUIRE(done.load()); + waiter.join(); + REQUIRE(waited); + // The release ran; the work after it still runs. + REQUIRE(split.Ran(release_position)); + REQUIRE_FALSE(split.QueuedSemaphoreWriteAt(0x1000).has_value()); + REQUIRE_FALSE(split.Ran(split.Position())); + release[1] = true; + split.Drain(VideoCommon::DrainSite::Other); + REQUIRE(split.Ran(split.Position())); + REQUIRE_FALSE(split.WaitForPosition(release_position, VideoCommon::DrainSite::Acquire)); +} + TEST_CASE("Split restart resets writer positions without replaying old releases", "[video_core][gpu_split_dependencies]") { using Access = VideoCommon::GpuSplitTestAccess; diff --git a/src/video_core/control/scheduler.cpp b/src/video_core/control/scheduler.cpp index 97d2dcd482..77af0af522 100644 --- a/src/video_core/control/scheduler.cpp +++ b/src/video_core/control/scheduler.cpp @@ -55,8 +55,9 @@ void Scheduler::Push(GPU& gpu, s32 channel, CommandList&& entries) { } } -void Scheduler::ResumePending(GPU& gpu) { +bool Scheduler::ResumePending(GPU& gpu) { // Running one channel can release the semaphore another waits for: retry until none moves. + bool resumed = false; for (bool progress = true; progress;) { progress = false; for (const auto& state : pending_channels) { @@ -65,7 +66,7 @@ void Scheduler::ResumePending(GPU& gpu) { // pending: resuming it again counted as progress forever and the GPU thread never // reached its stop request (closing the game hung). if (!dma_pusher.system.IsPoweredOn()) { - return; + return resumed; } if (!dma_pusher.HasPendingAcquire() || !dma_pusher.CanResumeAcquire()) { continue; @@ -73,12 +74,14 @@ void Scheduler::ResumePending(GPU& gpu) { BindChannel(gpu, state->bind_id); dma_pusher.DispatchCalls(); progress = true; + resumed = true; } } std::erase_if(pending_channels, [](const auto& state) { return !state->payload->dma_pusher.HasPendingAcquire(); }); any_pending.store(!pending_channels.empty(), std::memory_order_release); + return resumed; } void Scheduler::DeclareChannel(std::shared_ptr new_channel) { diff --git a/src/video_core/control/scheduler.h b/src/video_core/control/scheduler.h index b1aaf46b60..1e6824c8a0 100644 --- a/src/video_core/control/scheduler.h +++ b/src/video_core/control/scheduler.h @@ -26,8 +26,8 @@ class Scheduler { public: void Push(GPU& gpu, s32 channel, CommandList&& entries); - /// Runs channels whose pending semaphore acquire is now met. - void ResumePending(GPU& gpu); + /// Runs channels whose pending semaphore acquire is now met. Returns whether one ran. + bool ResumePending(GPU& gpu); [[nodiscard]] bool HasPendingChannels() const { return !pending_channels.empty(); } diff --git a/src/video_core/engines/kepler_compute.cpp b/src/video_core/engines/kepler_compute.cpp index 311f6dbb53..62c43dd7d3 100644 --- a/src/video_core/engines/kepler_compute.cpp +++ b/src/video_core/engines/kepler_compute.cpp @@ -1527,12 +1527,12 @@ void KeplerCompute::FrontCallMethod(Core::System& system, VideoCommon::GpuSplit& FrontApply(fs, method, argument); split.Method(this, method, argument, is_last_call, segment); if (method == 0x6c3) { - FrontNoteReport(split, fs, argument); + FrontNoteReport(split, fs, argument, false); } } -void KeplerCompute::FrontNoteReport(VideoCommon::GpuSplit& split, const FrontState& fs, - u32 control) { +void KeplerCompute::FrontNoteReport(VideoCommon::GpuSplit& split, const FrontState& fs, u32 control, + bool from_macro) { // SET_REPORT_SEMAPHORE_D was sent: what ProcessNativeReportSemaphore will write on the back. const u32 operation = control & 3; if (operation != 0 && operation != 2) { @@ -1544,10 +1544,30 @@ void KeplerCompute::FrontNoteReport(VideoCommon::GpuSplit& split, const FrontSta if (!device) { return; } - // A reduction combines the payload with the word's value then. + // A reduction combines the payload with the word's value then. The front knows that value + // when no compute or copy work sent since it may write the word (QMD releases): from the last + // release sent to the word if it is known, or from memory if nothing queued may change it. + // A macro's own launches count as work only after it ran (DmaPusher::NoteWriter). const bool reduction = ((control >> 3) & 1) != 0; - split.NoteSemaphoreWrite(*device, fs.registers[0x6c2], !reduction); - if (!reduction && ((control >> 28) & 1) == 0) { + if (reduction) { + const u32 payload = fs.registers[0x6c2]; + const u32 op = (control >> 9) & 7; + std::optional base; + if (!from_macro) { + if (const auto queued = split.QueuedSemaphoreWriteAt(*device)) { + if (queued->known && queued->position >= split.LastWriterPosition()) { + base = queued->payload; + } + } else if (!split.WritersPending() && !Nx2MayBeGpuWritten(*device, sizeof(u32))) { + base = memory_manager.Read(address); + } + } + split.NoteSemaphoreWrite(*device, base ? ReduceSemaphore(op, false, *base, payload) : 0, + base.has_value()); + return; + } + split.NoteSemaphoreWrite(*device, fs.registers[0x6c2], true); + if (((control >> 28) & 1) == 0) { // 64 bits (a timestamp follows). split.NoteSemaphoreWrite(*device + 4, 0, true); } @@ -1571,7 +1591,7 @@ void KeplerCompute::FrontCallMultiMethod(Core::System& system, VideoCommon::GpuS split.MultiMethod(this, method, base_start, amount, methods_pending, segment); if (method == 0x6c3) { for (u32 i = 0; i < amount; ++i) { - FrontNoteReport(split, fs, base_start[i]); + FrontNoteReport(split, fs, base_start[i], false); } } } @@ -1596,7 +1616,7 @@ void KeplerCompute::FrontRunMacro(Core::System& system, VideoCommon::GpuSplit& s FrontApply(fs, address, value); split.MacroWrite(this, address, value); if (address == 0x6c3) { - FrontNoteReport(split, fs, value); + FrontNoteReport(split, fs, value, true); } return true; }; diff --git a/src/video_core/engines/kepler_compute.h b/src/video_core/engines/kepler_compute.h index 6e99ffc239..6ceeec90f6 100644 --- a/src/video_core/engines/kepler_compute.h +++ b/src/video_core/engines/kepler_compute.h @@ -274,7 +274,8 @@ class KeplerCompute final : public EngineInterface { FrontState& Front(VideoCommon::GpuSplit& split); void ReleaseFront(); void FrontApply(FrontState& state, u32 method, u32 argument); - void FrontNoteReport(VideoCommon::GpuSplit& split, const FrontState& state, u32 control); + void FrontNoteReport(VideoCommon::GpuSplit& split, const FrontState& state, u32 control, + bool from_macro); MacroEngine& FrontMacroEngine(FrontState& state); std::span FrontMacroPositions(FrontState& state); void FrontRunMacro(Core::System& system, VideoCommon::GpuSplit& split, FrontState& state); diff --git a/src/video_core/engines/puller.cpp b/src/video_core/engines/puller.cpp index b0eafabb93..8a9e8ec514 100644 --- a/src/video_core/engines/puller.cpp +++ b/src/video_core/engines/puller.cpp @@ -107,56 +107,104 @@ bool ReleasesLandAtOnce() { return !delay_fence && !Settings::IsGPULevelHigh(); } -enum class AcquireCheck { - Met, ///< By the word in memory - MetByQueued, ///< By a release sent to the word that the back has not run yet - NotMet, ///< Nothing sent to the word can meet it yet - Drain, ///< Drain the split and read the word as before +// Whether the front decides Switch 2 acquires from its view of the semaphore words (AskView) +// instead of draining the split: the back writes each release to memory as it runs it, and the +// front sees every release it sends, its macros' included. +bool ViewApplies() { + const auto& split = VideoCommon::GpuSplit::Get(); + return split.Active() && split.FrontMacros() && ReleasesLandAtOnce(); +} + +// What the front can tell of an acquire without waiting for the back. +struct ViewAnswer { + enum class Kind { + Met, ///< By the word in memory + MetByQueued, ///< By the last release sent to the word, which the back has not run yet + NotMet, ///< Nothing sent to the word meets it yet + Unknown, ///< Memory tells once the back ran the work before `position` + Flush, ///< Shaders wrote the word: the buffer cache may hold it newer than memory + }; + Kind kind; + u64 position{}; }; -// Checks a greater-or-equal acquire without draining the split. The words Switch 2 syncs on only -// advance (NVN2 even releases QMD completions as maxima, KeplerCompute::ProcessNativeMethod), so -// a value that meets one now still does once the back ran the queued work. Only the last release -// the front sent to the word decides instead, when the back may not have run it yet. -AcquireCheck CheckWithoutDrain(DmaPusher& dma_pusher, const Puller::PendingAcquire& acquire) { - auto& split = VideoCommon::GpuSplit::Get(); - if (!acquire.WaitsToReach() || !split.Active() || !split.FrontMacros() || - !ReleasesLandAtOnce()) { - return AcquireCheck::Drain; - } - const auto device = dma_pusher.memory_manager.GpuToCpuAddress(acquire.address); - // A word the buffer cache may hold newer than memory is read through the back. - if (!device || Nx2MayBeGpuWritten(*device, sizeof(u32))) { - return AcquireCheck::Drain; +ViewAnswer AskView(DmaPusher& dma_pusher, const Puller::PendingAcquire& acquire, u64 device) { + using Kind = ViewAnswer::Kind; + const auto& split = VideoCommon::GpuSplit::Get(); + const auto memory = [&] { return dma_pusher.memory_manager.Read(acquire.address); }; + // The words Switch 2 syncs on only advance (NVN2 even releases QMD completions as maxima, + // KeplerCompute::ProcessNativeMethod): memory that meets a greater-or-equal acquire now still + // does once the back ran the queued work, also where the buffer cache holds it newer. + if (Nx2MayBeGpuWritten(device, sizeof(u32)) && + dma_pusher.rasterizer->MustFlushRegion(device, sizeof(u32), + VideoCommon::CacheType::BufferCache)) { + return {acquire.WaitsToReach() && acquire.Matches(memory()) ? Kind::Met : Kind::Flush}; + } + // Another acquire holds only for the value the word has when the channel reaches it. Compute + // and copy work sent before it may still release the word without the front's knowing (QMD + // releases): a value that meets it stands once the back ran that work. + const bool settled = acquire.WaitsToReach() || split.Ran(acquire.writers_before); + // The last release the front sent to the word decides while the back hasn't run it. + if (const auto queued = split.QueuedSemaphoreWriteAt(device)) { + if (!queued->known) { + return {Kind::Unknown, queued->position}; + } + if (!acquire.Matches(queued->payload)) { + return {Kind::NotMet}; + } + if (settled || queued->position >= acquire.writers_before) { + return {Kind::MetByQueued}; + } + return {Kind::Unknown, acquire.writers_before}; } - if (const auto queued = split.QueuedSemaphoreWriteAt(*device)) { - return queued->known && acquire.Matches(queued->payload) ? AcquireCheck::MetByQueued - : AcquireCheck::Drain; + if (!acquire.Matches(memory())) { + return {Kind::NotMet}; } - return acquire.Matches(dma_pusher.memory_manager.Read(acquire.address)) - ? AcquireCheck::Met - : AcquireCheck::NotMet; + return settled ? ViewAnswer{Kind::Met} : ViewAnswer{Kind::Unknown, acquire.writers_before}; } -// EDEN_NX2_DRAIN_CHECK: what the drain path decides instead, to verify a decision made without it -// (a release the back ran latches `waiter`, a suspended acquire's). -bool MetAfterDrain(DmaPusher& dma_pusher, const Puller::PendingAcquire& acquire, - const void* waiter) { - DrainBack(); - return acquire.Matches(ReadSemaphoreWord(dma_pusher, acquire.address)) || - (waiter != nullptr && Nx2SemaphoreWaiters::Get().Latched(waiter)); +// Has the back flush the word out of the buffer cache after the work sent before: memory holds +// it once the back ran the flush (PendingAcquire::flush_position). +void RequestFlush(DmaPusher& dma_pusher, Puller::PendingAcquire& acquire, u64 device) { + auto& split = VideoCommon::GpuSplit::Get(); + split.Defer([rasterizer = dma_pusher.rasterizer, device] { + rasterizer->FlushRegion(device, sizeof(u32), VideoCommon::CacheType::BufferCache); + }); + acquire.flush_position = split.Position(); } -// Whether a pass without the drain stands. Verified, a pass the drained word contradicts takes -// the drain path instead. -bool PassStands(DmaPusher& dma_pusher, const Puller::PendingAcquire& acquire, bool queued, - const void* waiter) { - if (!VideoCommon::Nx2DrainCheck::g_enabled) { - return true; +// The fence work an acquire that still waits after the drain sends (ReleaseFences), sent once the +// back ran everything before the acquire: the release that work runs then counts as it did (the +// waiter is registered), and acquires the queued work releases send none. +void MaybeReleaseFences(DmaPusher& dma_pusher, Puller::PendingAcquire& acquire) { + if (acquire.fences_released || !VideoCommon::GpuSplit::Get().Ran(acquire.sent_at)) { + return; + } + acquire.fences_released = true; + OnBack([rasterizer = dma_pusher.rasterizer] { rasterizer->ReleaseFences(); }); +} + +// Logs once an acquire that waits for more than 2 s. `current`: the word as read after a drain, +// or nothing to read it from memory. +void ReportLongWait(DmaPusher& dma_pusher, Puller::PendingAcquire& acquire, + std::optional current) { + if (acquire.reported || + std::chrono::steady_clock::now() - acquire.since <= std::chrono::seconds{2}) { + return; } - const bool met = MetAfterDrain(dma_pusher, acquire, waiter); - VideoCommon::Nx2DrainCheck::NotePass(waiter != nullptr, queued, met); - return met; + acquire.reported = true; + const auto device = dma_pusher.memory_manager.GpuToCpuAddress(acquire.address); + const GPUVAddr words = acquire.address & ~GPUVAddr{0xf}; + LOG_WARNING( + HW_GPU, + "Channel {} waits on semaphore {:#x} (op {}, {:#x}, now {:#x}) for 2 s; device " + "{:#x}, buffer cache holds GPU data: {}, words {:08x} {:08x} {:08x} {:08x}", + dma_pusher.channel_state.bind_id, acquire.address, acquire.operation, acquire.value, + current.value_or(dma_pusher.memory_manager.Read(acquire.address)), device.value_or(0), + device && dma_pusher.rasterizer->MustFlushRegion(*device, 16, VideoCommon::CacheType::All), + dma_pusher.memory_manager.Read(words), dma_pusher.memory_manager.Read(words + 4), + dma_pusher.memory_manager.Read(words + 8), + dma_pusher.memory_manager.Read(words + 12)); } } // Anonymous namespace @@ -182,40 +230,9 @@ bool Puller::CanResumeAcquire(DmaPusher& dma_pusher) { if (!pending_acquire) { return true; } - switch (CheckWithoutDrain(dma_pusher, *pending_acquire)) { - case AcquireCheck::MetByQueued: - if (!PassStands(dma_pusher, *pending_acquire, true, this)) { - break; - } - // What the front reads after the acquire waits for the work sent before the release. - VideoCommon::GpuSplit::Get().NoteSync(); - return true; - case AcquireCheck::Met: - if (!PassStands(dma_pusher, *pending_acquire, false, this)) { - break; - } - return true; - case AcquireCheck::NotMet: - // A release the back ran latches the waiter. Otherwise poll again later, without waiting - // for the back: until it runs one, nothing sent to the word meets it. After 2 s, the - // drain below reports the wait. - if (Nx2SemaphoreWaiters::Get().Latched(this)) { - return true; - } - if (VideoCommon::Nx2DrainCheck::g_enabled) [[unlikely]] { - const bool met = MetAfterDrain(dma_pusher, *pending_acquire, this); - VideoCommon::Nx2DrainCheck::NoteKeptWaiting(met); - if (met) { - return true; - } - } - if (pending_acquire->reported || - std::chrono::steady_clock::now() - pending_acquire->since <= std::chrono::seconds{2}) { - return false; - } - break; - case AcquireCheck::Drain: - break; + if (const auto device = dma_pusher.memory_manager.GpuToCpuAddress(pending_acquire->address); + device && ViewApplies()) { + return PollWithoutDrain(dma_pusher, *device); } const bool check = VideoCommon::Nx2DrainCheck::g_enabled; const bool met_before = @@ -241,20 +258,43 @@ bool Puller::CanResumeAcquire(DmaPusher& dma_pusher) { if (pending_acquire->Matches(current)) { return true; } - if (!pending_acquire->reported && - std::chrono::steady_clock::now() - pending_acquire->since > std::chrono::seconds{2}) { - pending_acquire->reported = true; - const auto device = dma_pusher.memory_manager.GpuToCpuAddress(pending_acquire->address); - LOG_WARNING(HW_GPU, "Channel {} waits on semaphore {:#x} (op {}, {:#x}, now {:#x}) for 2 s; " - "device {:#x}, buffer cache holds GPU data: {}, words {:08x} {:08x} {:08x} {:08x}", - dma_pusher.channel_state.bind_id, pending_acquire->address, - pending_acquire->operation, pending_acquire->value, current, device.value_or(0), - device && dma_pusher.rasterizer->MustFlushRegion(*device, 16, VideoCommon::CacheType::All), - dma_pusher.memory_manager.Read(pending_acquire->address & ~GPUVAddr{0xf}), - dma_pusher.memory_manager.Read((pending_acquire->address & ~GPUVAddr{0xf}) + 4), - dma_pusher.memory_manager.Read((pending_acquire->address & ~GPUVAddr{0xf}) + 8), - dma_pusher.memory_manager.Read((pending_acquire->address & ~GPUVAddr{0xf}) + 12)); + ReportLongWait(dma_pusher, *pending_acquire, current); + return false; +} + +bool Puller::PollWithoutDrain(DmaPusher& dma_pusher, u64 device) { + using Kind = ViewAnswer::Kind; + PendingAcquire& acquire = *pending_acquire; + auto& split = VideoCommon::GpuSplit::Get(); + // A release the back ran latches the waiter, even if a later one changed the word again. + if (Nx2SemaphoreWaiters::Get().Latched(this)) { + return true; + } + if (acquire.flush_position && split.Ran(*acquire.flush_position)) { + acquire.flush_position.reset(); } + if (!acquire.flush_position) { + const ViewAnswer answer = AskView(dma_pusher, acquire, device); + switch (answer.kind) { + case Kind::Met: + return true; + case Kind::MetByQueued: + // What the front reads after the acquire waits for the work sent before the release. + split.NoteSync(); + return true; + case Kind::Flush: + RequestFlush(dma_pusher, acquire, device); + break; + case Kind::NotMet: + case Kind::Unknown: + // Poll again later, without waiting for the back: until it runs a release that meets + // the acquire (which latches the waiter) or the work before `position`, nothing the + // front sees meets it. + break; + } + } + MaybeReleaseFences(dma_pusher, acquire); + ReportLongWait(dma_pusher, acquire, std::nullopt); return false; } @@ -393,14 +433,7 @@ void Puller::ProcessSemaphoreTriggerMethod(DmaPusher& dma_pusher) { return; } VideoCommon::Nx2Profile::Add(VideoCommon::Nx2Profile::Counter::AcquireChecks, 1); - if (const AcquireCheck without_drain = CheckWithoutDrain(dma_pusher, acquire); - (without_drain == AcquireCheck::Met || without_drain == AcquireCheck::MetByQueued) && - PassStands(dma_pusher, acquire, without_drain == AcquireCheck::MetByQueued, nullptr)) { - if (without_drain == AcquireCheck::MetByQueued) { - // What the front reads after the acquire waits for the work sent before the - // release (GpuSplit::NoteSync). - VideoCommon::GpuSplit::Get().NoteSync(); - } + if (AcquireWithoutDrain(dma_pusher, acquire)) { return; } const bool check = VideoCommon::Nx2DrainCheck::g_enabled; @@ -460,6 +493,47 @@ void Puller::ProcessSemaphoreTriggerMethod(DmaPusher& dma_pusher) { } } +bool Puller::AcquireWithoutDrain(DmaPusher& dma_pusher, PendingAcquire& acquire) { + using Kind = ViewAnswer::Kind; + const auto device = dma_pusher.memory_manager.GpuToCpuAddress(acquire.address); + if (!device || !ViewApplies()) { + return false; + } + auto& split = VideoCommon::GpuSplit::Get(); + acquire.writers_before = split.LastWriterPosition(); + acquire.sent_at = split.Position(); + ViewAnswer answer = AskView(dma_pusher, acquire, *device); + // The value is in memory once the back ran that work, before what was sent after it. A + // release the front can't value, then the writers before the acquire: two waits at most. + for (u32 waits = 0; answer.kind == Kind::Unknown && waits < 2; ++waits) { + split.WaitForPosition(answer.position, VideoCommon::DrainSite::Acquire); + answer = AskView(dma_pusher, acquire, *device); + } + switch (answer.kind) { + case Kind::Met: + return true; + case Kind::MetByQueued: + // What the front reads after the acquire waits for the work sent before the release + // (GpuSplit::NoteSync). + split.NoteSync(); + return true; + case Kind::Flush: + RequestFlush(dma_pusher, acquire, *device); + break; + case Kind::NotMet: + case Kind::Unknown: + break; + } + // Suspended: a release the back runs latches the waiter, registered before any fence work, + // and the polls see the others in memory or in the view (PollWithoutDrain). + Nx2SemaphoreWaiters::Get().Add(this, *device, acquire.value, acquire.operation); + MaybeReleaseFences(dma_pusher, acquire); + pending_acquire = acquire; + dma_pusher.suspensions.fetch_add(1, std::memory_order_release); + VideoCommon::Nx2Profile::Add(VideoCommon::Nx2Profile::Counter::AcquireWaits, 1); + return true; +} + void Puller::ProcessSemaphoreRelease(DmaPusher& dma_pusher) { const GPUVAddr sequence_address{regs.semaphore_address.SemaphoreAddress()}; const u32 payload = regs.semaphore_release; @@ -500,21 +574,40 @@ void Puller::ProcessSemaphoreExecute(DmaPusher& dma_pusher, u32 argument) { NoteRelease(dma_pusher, address, payload, false); return; } - // Acquires: check once after flushing pending fences; never block the GPU thread on them. - DrainBack(); - ReleaseFences(dma_pusher); - const u32 word = dma_pusher.memory_manager.Read(address); - const bool ready = operation == 0 ? word == payload - : operation == 2 ? word >= payload - : operation == 3 ? static_cast(word - payload) >= 0 - : operation == 4 ? (word & payload) != 0 - : operation == 5 ? (word & payload) == 0 - : true; - if (!ready) { + // Acquires: never block the GPU thread on them; they are checked for the warning below only. + const auto ready = [operation, payload](u32 value) { + return operation == 0 ? value == payload + : operation == 2 ? value >= payload + : operation == 3 ? static_cast(value - payload) >= 0 + : operation == 4 ? (value & payload) != 0 + : operation == 5 ? (value & payload) == 0 + : true; + }; + std::optional word; + if (const auto device = dma_pusher.memory_manager.GpuToCpuAddress(address); + device && ViewApplies()) { + // The last release sent to the word tells while the back hasn't run it. What the front + // reads after the acquire waits for the work sent before it, as after a drain. + auto& split = VideoCommon::GpuSplit::Get(); + split.NoteSync(); + OnBack([rasterizer = dma_pusher.rasterizer] { rasterizer->ReleaseFences(); }); + if (const auto queued = split.QueuedSemaphoreWriteAt(*device)) { + if (queued->known) { + word = queued->payload; + } + } else { + word = dma_pusher.memory_manager.Read(address); + } + } else { + DrainBack(); + ReleaseFences(dma_pusher); + word = dma_pusher.memory_manager.Read(address); + } + if (word && !ready(*word)) { static std::atomic reported{}; if (reported.fetch_add(1) < 8) { LOG_WARNING(HW_GPU, "SEM_EXECUTE op {} at {:#x}: {:#x} vs {:#x} not met, continuing", - operation, address, word, payload); + operation, address, *word, payload); } } } diff --git a/src/video_core/engines/puller.h b/src/video_core/engines/puller.h index fe6874d789..7695fd1cad 100644 --- a/src/video_core/engines/puller.h +++ b/src/video_core/engines/puller.h @@ -79,6 +79,17 @@ class Puller final { u32 operation; std::chrono::steady_clock::time_point since; bool reported; + /// GpuSplit::LastWriterPosition when the acquire was sent: compute and copy work before + /// it may release the word without the front's knowing (QMD releases). + u64 writers_before{}; + /// A flush of the word out of the buffer cache, sent to the split's back: until the back + /// ran it, memory may hold an older value than the cache. + std::optional flush_position{}; + /// GpuSplit::Position when the acquire was sent, and whether the fence work was sent + /// since: once the back ran everything before the acquire and it still waits, as after + /// the drain this replaces. + u64 sent_at{}; + bool fences_released{}; [[nodiscard]] bool Matches(u32 current) const; /// Whether it waits for the word to reach `value` (greater or equal). @@ -148,6 +159,11 @@ class Puller final { void ProcessSemaphoreRelease(DmaPusher& dma_pusher); void ProcessSemaphoreTriggerMethod(DmaPusher& dma_pusher); void ProcessSemaphoreExecute(DmaPusher& dma_pusher, u32 argument); + /// Decides a Switch 2 acquire from the split's view of the semaphore words: passes it, or + /// suspends the channel on it. False when the view doesn't apply (drain as before). + bool AcquireWithoutDrain(DmaPusher& dma_pusher, PendingAcquire& acquire); + /// CanResumeAcquire for an acquire the view decides, at `device`. + bool PollWithoutDrain(DmaPusher& dma_pusher, u64 device); [[nodiscard]] bool ExecuteMethodOnEngine(DmaPusher& dma_pusher, u32 method); std::optional pending_acquire; diff --git a/src/video_core/gpu_split.cpp b/src/video_core/gpu_split.cpp index b2bbccae2e..f0800c9945 100644 --- a/src/video_core/gpu_split.cpp +++ b/src/video_core/gpu_split.cpp @@ -256,7 +256,7 @@ std::optional GpuSplit::QueuedSemaphoreWriteAt( it->second.position <= consumed.load(std::memory_order_acquire)) { return std::nullopt; } - return QueuedSemaphoreWrite{it->second.payload, it->second.known}; + return QueuedSemaphoreWrite{it->second.payload, it->second.known, it->second.position}; } bool GpuSplit::WritersPending() const { diff --git a/src/video_core/gpu_split.h b/src/video_core/gpu_split.h index f1766759a1..d5a587a351 100644 --- a/src/video_core/gpu_split.h +++ b/src/video_core/gpu_split.h @@ -228,10 +228,6 @@ class GpuSplit { void Publish(); /// Waits until the back executed everything sent so far. void Drain(DrainSite site = DrainSite::Other); - /// The ring position after the entries sent so far, for WaitUntil. - [[nodiscard]] u64 Position() const { - return write_position; - } /// Waits until the back executed the entries sent before `position`. Returns whether any of /// them was still queued. bool WaitUntil(u64 position, DrainSite site) { @@ -246,11 +242,29 @@ class GpuSplit { /// Returns whether any of it was still queued. bool WaitForWriters(); /// A host-level wait was just sent (a syncpoint acquire, a host WFI, a GP entry with its sync - /// bit; semaphore acquires drain, or note it when a release still queued meets them): the - /// guest's GPU work before it is complete before the commands after it are fetched. + /// bit; semaphore acquires note it when a release still queued meets them): the guest's GPU + /// work before it is complete before the commands after it are fetched. void NoteSync() { last_sync_position = write_position; } + /// The position after everything sent so far, for Ran and WaitForPosition. + [[nodiscard]] u64 Position() const { + return write_position; + } + /// Whether the back executed everything sent before `position`. It may tell so a little + /// late: the back stores its position every few thousand words and when it runs out of work. + [[nodiscard]] bool Ran(u64 position) const { + return consumed.load(std::memory_order_acquire) >= position; + } + /// The position after the last compute or copy work sent (NoteWriter). + [[nodiscard]] u64 LastWriterPosition() const { + return (std::min)(last_writer_position, write_position); + } + /// Waits until the back executed everything sent before `position`, not what came after. + /// Returns whether any of it was still queued. + bool WaitForPosition(u64 position, DrainSite site) { + return WaitFor((std::min)(position, write_position), site); + } /// Waits until the back executed the work sent before the last NoteSync. Returns whether any /// of it was still queued. bool WaitForSync(); @@ -264,6 +278,7 @@ class GpuSplit { struct QueuedSemaphoreWrite { u32 payload; bool known; + u64 position; ///< Once the back ran this (Ran), the word holds the write }; /// The last write sent to the semaphore word at `device_address`, if the back may not have run /// it yet. diff --git a/src/video_core/gpu_thread.cpp b/src/video_core/gpu_thread.cpp index 81f7d0e4b7..9bebdde539 100644 --- a/src/video_core/gpu_thread.cpp +++ b/src/video_core/gpu_thread.cpp @@ -85,7 +85,9 @@ void ThreadManager::StartThread(VideoCore::RendererBase& renderer, Core::Fronten // Suspended channels wait on semaphores the CPU releases: poll them meanwhile. { Profile::Scope scope{Profile::Zone::ResumePending}; - scheduler.ResumePending(system.GPU()); + if (scheduler.ResumePending(system.GPU())) { + last_progress = std::chrono::steady_clock::now(); + } } if (!state.queue.TryPop(next)) { // The CPU usually releases the semaphore soon after: poll for half a From 557a13242c2248198f860dfa3f70de82138926ca Mon Sep 17 00:00:00 2001 From: jpkto <255363096+jpkto@users.noreply.github.com> Date: Sun, 8 Nov 2026 00:26:16 +0100 Subject: [PATCH 029/105] TEST: verify and count the semaphore view's acquire decisions EDEN_NX2_DRAIN_CHECK now checks every decision the semaphore view makes against the drain path: the front drains and reads the word as before anyway. This replaces fix 2's checks, whose code the previous commit replaced. In this mode a pass that the drained word doesn't meet is corrected (the channel waits), and a wait that the drained word meets ends. The 10 s "Drain checks" line ends with: semaphore view: acquires passed N (by memory M, by a release still queued Q, after a positioned wait W), not met after the drain X, suspended S (met after the drain D); polls resumed R (...), not met after the drain Y, kept waiting K (met after the drain Z) X and Y must stay 0. D and Z count acquires and polls that a drain would have passed or resumed sooner. EDEN_NX2_ACQUIRE_STATS counts the decisions without changing what runs (use it without the drain check, whose drains change the timing). Every 10 s it logs: - equal, greater-or-equal and mask acquires: passed by memory, by a queued release, after a positioned wait, suspended, or drained; - the suspended ones by word: on GPU-written pages (and held newer by the buffer cache), and by the releases the front sent there (puller, 3D reports, compute reports, copy engine, or none: QMD releases or the CPU); - what resumed them: a latch by its writer (payload releases, query results, QMD reductions, QMD maxima), memory, a queued release, or the drain check; also how many after a flush; - how long they waited: total, maximum, and how many over 1 ms. Nx2SemaphoreWaiters records which writer latched a waiter for this. EDEN_NX2_GPU_PROFILE gets an "acquire poll" zone around the poll in Scheduler::ResumePending. "resume pending" then holds the resumed channels' own command processing, which is most of it: a channel suspended on an acquire parses the lists it was sent meanwhile inside ResumePending. In test47 that was 165 ms/s at 50,000 calls/s, while "push" was 3.6 ms/s. Rebased onto upstream: EDEN_NX2_ACQUIRE_STATS is a plain flag read again by VideoCore::CreateGPU, as upstream reads its diagnostic switches since #714, so it works from nx2_switches.txt too, not only from the environment. Co-Authored-By: Claude Opus 5.5 --- src/video_core/control/scheduler.cpp | 12 +- src/video_core/dma_pusher.cpp | 2 + src/video_core/engines/kepler_compute.cpp | 9 +- src/video_core/engines/maxwell_3d.cpp | 3 + src/video_core/engines/puller.cpp | 108 ++++++++- src/video_core/engines/puller.h | 1 + src/video_core/gpu_thread.cpp | 1 + src/video_core/nx2.h | 29 ++- src/video_core/nx2_drain_check.h | 277 +++++++++++++++++++--- src/video_core/nx2_profile.h | 3 +- src/video_core/query_cache/query_cache.h | 8 +- src/video_core/video_core.cpp | 1 + 12 files changed, 412 insertions(+), 42 deletions(-) diff --git a/src/video_core/control/scheduler.cpp b/src/video_core/control/scheduler.cpp index 77af0af522..03031422f2 100644 --- a/src/video_core/control/scheduler.cpp +++ b/src/video_core/control/scheduler.cpp @@ -14,6 +14,7 @@ #include "video_core/control/scheduler.h" #include "video_core/gpu.h" #include "video_core/gpu_split.h" +#include "video_core/nx2_profile.h" namespace Tegra::Control { @@ -68,7 +69,16 @@ bool Scheduler::ResumePending(GPU& gpu) { if (!dma_pusher.system.IsPoweredOn()) { return resumed; } - if (!dma_pusher.HasPendingAcquire() || !dma_pusher.CanResumeAcquire()) { + if (!dma_pusher.HasPendingAcquire()) { + continue; + } + // TEST: the poll apart from the resumed channel's commands (EDEN_NX2_GPU_PROFILE). + bool can_resume; + { + VideoCommon::Nx2Profile::Scope scope{VideoCommon::Nx2Profile::Zone::AcquirePoll}; + can_resume = dma_pusher.CanResumeAcquire(); + } + if (!can_resume) { continue; } BindChannel(gpu, state->bind_id); diff --git a/src/video_core/dma_pusher.cpp b/src/video_core/dma_pusher.cpp index f8386b2aad..377c9d4328 100644 --- a/src/video_core/dma_pusher.cpp +++ b/src/video_core/dma_pusher.cpp @@ -626,6 +626,8 @@ void DmaPusher::NoteDmaSemaphore(u32 method, u32 argument) { } auto& split = VideoCommon::GpuSplit::Get(); split.NoteSemaphoreWrite(*device, dma_semaphore[2], true); + VideoCommon::Nx2AcquireStats::NoteRelease(*device, + VideoCommon::Nx2AcquireStats::Releaser::Copy); if (type == SemaphoreType::RELEASE_FOUR_WORD_SEMAPHORE) { // The payload as 64 bits (a timestamp follows). split.NoteSemaphoreWrite(*device + 4, 0, true); diff --git a/src/video_core/engines/kepler_compute.cpp b/src/video_core/engines/kepler_compute.cpp index 62c43dd7d3..020045db29 100644 --- a/src/video_core/engines/kepler_compute.cpp +++ b/src/video_core/engines/kepler_compute.cpp @@ -25,6 +25,7 @@ #include "video_core/fire_emblem_upscale.h" #include "video_core/nx2_profile.h" #include "video_core/nx2.h" +#include "video_core/nx2_drain_check.h" #include "video_core/engines/fermi_2d.h" #include "video_core/engines/kepler_compute.h" #include "video_core/engines/native_upscale.h" @@ -374,7 +375,8 @@ bool KeplerCompute::ProcessNativeMethod(Core::System& system, u32 method, u32 ar memory_manager.Read(address), payload); memory_manager.Write(address, value); if (const auto device = memory_manager.GpuToCpuAddress(address)) { - Tegra::Nx2SemaphoreWaiters::Get().OnWrite(*device, value); + Tegra::Nx2SemaphoreWaiters::Get().OnWrite( + *device, value, Tegra::Nx2SemaphoreWaiters::Source::QmdReduction); } static const u64 watch = [] { const char* text = std::getenv("EDEN_NX2_SEM_WATCH"); @@ -398,7 +400,8 @@ bool KeplerCompute::ProcessNativeMethod(Core::System& system, u32 method, u32 ar const u32 value = (std::max)(memory_manager.Read(address), payload); memory_manager.Write(address, value); if (const auto device = memory_manager.GpuToCpuAddress(address)) { - Tegra::Nx2SemaphoreWaiters::Get().OnWrite(*device, value); + Tegra::Nx2SemaphoreWaiters::Get().OnWrite( + *device, value, Tegra::Nx2SemaphoreWaiters::Source::QmdMax); } }); } else { @@ -1544,6 +1547,8 @@ void KeplerCompute::FrontNoteReport(VideoCommon::GpuSplit& split, const FrontSta if (!device) { return; } + VideoCommon::Nx2AcquireStats::NoteRelease( + *device, VideoCommon::Nx2AcquireStats::Releaser::ReportCompute); // A reduction combines the payload with the word's value then. The front knows that value // when no compute or copy work sent since it may write the word (QMD releases): from the last // release sent to the word if it is known, or from memory if nothing queued may change it. diff --git a/src/video_core/engines/maxwell_3d.cpp b/src/video_core/engines/maxwell_3d.cpp index 535dea21c5..da159a719d 100644 --- a/src/video_core/engines/maxwell_3d.cpp +++ b/src/video_core/engines/maxwell_3d.cpp @@ -29,6 +29,7 @@ #include "video_core/memory_manager.h" #include "video_core/nx2.h" #include "video_core/nx2_cmdpage_diag.h" +#include "video_core/nx2_drain_check.h" #include "video_core/nx2_perf.h" #include "video_core/nx2_profile.h" #include "video_core/query_cache/frame_trace.h" @@ -1429,6 +1430,8 @@ void Maxwell3D::FrontNoteReport(VideoCommon::GpuSplit& split, const FrontState& } const bool payload = report.query.report == Regs::ReportSemaphore::Report::Payload; split.NoteSemaphoreWrite(*device, report.payload, payload); + VideoCommon::Nx2AcquireStats::NoteRelease(*device, + VideoCommon::Nx2AcquireStats::Releaser::Report3D); if (report.query.short_query == 0) { // 64 bits (a timestamp follows). split.NoteSemaphoreWrite(*device + 4, 0, payload); diff --git a/src/video_core/engines/puller.cpp b/src/video_core/engines/puller.cpp index 8a9e8ec514..c0097eb6d5 100644 --- a/src/video_core/engines/puller.cpp +++ b/src/video_core/engines/puller.cpp @@ -91,6 +91,8 @@ void NoteRelease(DmaPusher& dma_pusher, GPUVAddr address, u32 payload, bool four return; } split.NoteSemaphoreWrite(*device, payload, true); + VideoCommon::Nx2AcquireStats::NoteRelease(*device, + VideoCommon::Nx2AcquireStats::Releaser::Puller); if (four_words) { // The payload as 64 bits (a timestamp follows). split.NoteSemaphoreWrite(*device + 4, 0, true); @@ -171,6 +173,77 @@ void RequestFlush(DmaPusher& dma_pusher, Puller::PendingAcquire& acquire, u64 de rasterizer->FlushRegion(device, sizeof(u32), VideoCommon::CacheType::BufferCache); }); acquire.flush_position = split.Position(); + acquire.flushed = true; +} + +// EDEN_NX2_DRAIN_CHECK: what the drain path decides instead, to verify the view (a release the +// back ran latches `waiter`, a suspended acquire's). +bool MetAfterDrain(DmaPusher& dma_pusher, const Puller::PendingAcquire& acquire, + const void* waiter) { + DrainBack(); + return acquire.Matches(ReadSemaphoreWord(dma_pusher, acquire.address)) || + (waiter != nullptr && Nx2SemaphoreWaiters::Get().Latched(waiter)); +} + +// TEST: counts the view's decision on an acquire (EDEN_NX2_ACQUIRE_STATS) and checks it against +// the drain path (EDEN_NX2_DRAIN_CHECK): a pass the drained word doesn't meet suspends instead, +// a suspension the drained word meets passes. +ViewAnswer CheckAcquire(DmaPusher& dma_pusher, const Puller::PendingAcquire& acquire, + ViewAnswer answer, bool waited, u64 device) { + using Kind = ViewAnswer::Kind; + namespace Check = VideoCommon::Nx2DrainCheck; + namespace Stats = VideoCommon::Nx2AcquireStats; + const bool pass = answer.kind == Kind::Met || answer.kind == Kind::MetByQueued; + const auto how = waited ? Check::ViewPass::Waited + : answer.kind == Kind::MetByQueued ? Check::ViewPass::Queued + : Check::ViewPass::Memory; + if (Stats::g_enabled) { + const auto decision = !pass ? Stats::Decision::Suspended + : how == Check::ViewPass::Waited ? Stats::Decision::Waited + : how == Check::ViewPass::Queued ? Stats::Decision::Queued + : Stats::Decision::Memory; + Stats::NoteAcquire(acquire.operation, decision, device, waited, + Nx2MayBeGpuWritten(device, sizeof(u32)), answer.kind == Kind::Flush); + } + if (Check::g_enabled) { + const bool met = MetAfterDrain(dma_pusher, acquire, nullptr); + if (pass) { + Check::NoteViewPass(false, how, met); + if (!met) { + return {Kind::NotMet}; + } + } else { + Check::NoteViewWait(false, met); + if (met) { + return {Kind::Met}; + } + } + } + return answer; +} + +// TEST: whether the view's resume of a poll stands (EDEN_NX2_DRAIN_CHECK). +bool PollPassStands(DmaPusher& dma_pusher, const Puller::PendingAcquire& acquire, + const void* waiter, VideoCommon::Nx2DrainCheck::ViewPass how) { + if (!VideoCommon::Nx2DrainCheck::g_enabled) { + return true; + } + const bool met = MetAfterDrain(dma_pusher, acquire, waiter); + VideoCommon::Nx2DrainCheck::NoteViewPass(true, how, met); + return met; +} + +// TEST: a suspended acquire resumes (EDEN_NX2_ACQUIRE_STATS). +void NoteResumed(const void* waiter, const Puller::PendingAcquire& acquire, + VideoCommon::Nx2AcquireStats::Resume how) { + namespace Stats = VideoCommon::Nx2AcquireStats; + if (!Stats::g_enabled) { + return; + } + const auto source = how == Stats::Resume::Latch ? Nx2SemaphoreWaiters::Get().LatchSource(waiter) + : Stats::Source::Unknown; + Stats::NoteResume(how, source, acquire.flushed, + std::chrono::steady_clock::now() - acquire.since); } // The fence work an acquire that still waits after the drain sends (ReleaseFences), sent once the @@ -264,23 +337,38 @@ bool Puller::CanResumeAcquire(DmaPusher& dma_pusher) { bool Puller::PollWithoutDrain(DmaPusher& dma_pusher, u64 device) { using Kind = ViewAnswer::Kind; + using Resume = VideoCommon::Nx2AcquireStats::Resume; + using ViewPass = VideoCommon::Nx2DrainCheck::ViewPass; PendingAcquire& acquire = *pending_acquire; auto& split = VideoCommon::GpuSplit::Get(); // A release the back ran latches the waiter, even if a later one changed the word again. if (Nx2SemaphoreWaiters::Get().Latched(this)) { + NoteResumed(this, acquire, Resume::Latch); return true; } if (acquire.flush_position && split.Ran(*acquire.flush_position)) { acquire.flush_position.reset(); } + // TEST: a resume the drained word contradicts was checked already. + bool checked = false; if (!acquire.flush_position) { const ViewAnswer answer = AskView(dma_pusher, acquire, device); switch (answer.kind) { case Kind::Met: + if (!PollPassStands(dma_pusher, acquire, this, ViewPass::Memory)) { + checked = true; + break; + } + NoteResumed(this, acquire, Resume::Memory); return true; case Kind::MetByQueued: + if (!PollPassStands(dma_pusher, acquire, this, ViewPass::Queued)) { + checked = true; + break; + } // What the front reads after the acquire waits for the work sent before the release. split.NoteSync(); + NoteResumed(this, acquire, Resume::Queued); return true; case Kind::Flush: RequestFlush(dma_pusher, acquire, device); @@ -293,6 +381,14 @@ bool Puller::PollWithoutDrain(DmaPusher& dma_pusher, u64 device) { break; } } + if (VideoCommon::Nx2DrainCheck::g_enabled && !checked) [[unlikely]] { + const bool met = MetAfterDrain(dma_pusher, acquire, this); + VideoCommon::Nx2DrainCheck::NoteViewWait(true, met); + if (met) { + NoteResumed(this, acquire, Resume::Drain); + return true; + } + } MaybeReleaseFences(dma_pusher, acquire); ReportLongWait(dma_pusher, acquire, std::nullopt); return false; @@ -436,6 +532,11 @@ void Puller::ProcessSemaphoreTriggerMethod(DmaPusher& dma_pusher) { if (AcquireWithoutDrain(dma_pusher, acquire)) { return; } + if (VideoCommon::Nx2AcquireStats::g_enabled) [[unlikely]] { + VideoCommon::Nx2AcquireStats::NoteAcquire( + acquire.operation, VideoCommon::Nx2AcquireStats::Decision::Drained, 0, false, false, + false); + } const bool check = VideoCommon::Nx2DrainCheck::g_enabled; const bool met_before = check && acquire.Matches(dma_pusher.memory_manager.Read(acquire.address)); @@ -505,10 +606,15 @@ bool Puller::AcquireWithoutDrain(DmaPusher& dma_pusher, PendingAcquire& acquire) ViewAnswer answer = AskView(dma_pusher, acquire, *device); // The value is in memory once the back ran that work, before what was sent after it. A // release the front can't value, then the writers before the acquire: two waits at most. - for (u32 waits = 0; answer.kind == Kind::Unknown && waits < 2; ++waits) { + u32 waits = 0; + for (; answer.kind == Kind::Unknown && waits < 2; ++waits) { split.WaitForPosition(answer.position, VideoCommon::DrainSite::Acquire); answer = AskView(dma_pusher, acquire, *device); } + if (VideoCommon::Nx2DrainCheck::g_enabled || VideoCommon::Nx2AcquireStats::g_enabled) + [[unlikely]] { + answer = CheckAcquire(dma_pusher, acquire, answer, waits != 0, *device); + } switch (answer.kind) { case Kind::Met: return true; diff --git a/src/video_core/engines/puller.h b/src/video_core/engines/puller.h index 7695fd1cad..1f9c070b6f 100644 --- a/src/video_core/engines/puller.h +++ b/src/video_core/engines/puller.h @@ -90,6 +90,7 @@ class Puller final { /// the drain this replaces. u64 sent_at{}; bool fences_released{}; + bool flushed{}; ///< TEST (EDEN_NX2_ACQUIRE_STATS): a flush was sent for it [[nodiscard]] bool Matches(u32 current) const; /// Whether it waits for the word to reach `value` (greater or equal). diff --git a/src/video_core/gpu_thread.cpp b/src/video_core/gpu_thread.cpp index 9bebdde539..2ad7942e5e 100644 --- a/src/video_core/gpu_thread.cpp +++ b/src/video_core/gpu_thread.cpp @@ -81,6 +81,7 @@ void ThreadManager::StartThread(VideoCore::RendererBase& renderer, Core::Fronten split.MaybeStart(system); WaitStats::MaybeReport(split.Active()); VideoCommon::Nx2DrainCheck::MaybeReport(); + VideoCommon::Nx2AcquireStats::MaybeReport(); if (scheduler.HasPendingChannels()) { // Suspended channels wait on semaphores the CPU releases: poll them meanwhile. { diff --git a/src/video_core/nx2.h b/src/video_core/nx2.h index e85e6142e7..b9c5c86387 100644 --- a/src/video_core/nx2.h +++ b/src/video_core/nx2.h @@ -369,6 +369,16 @@ class Nx2SemaphoreWaiters { static constexpr u32 AcquireGequal = 0x4; static constexpr u32 AcquireMask = 0x8; + /// TEST (EDEN_NX2_ACQUIRE_STATS): the writer that latched a waiter (OnWrite). + enum class Source : u8 { + Unknown, + PayloadRelease, ///< The query cache's payload releases (puller, 3D, compute, copy, QMD) + QueryResult, ///< A report the query cache wrote once its fence passed + QmdReduction, + QmdMax, + Count, + }; + static Nx2SemaphoreWaiters& Get() { static Nx2SemaphoreWaiters waiters; return waiters; @@ -377,7 +387,7 @@ class Nx2SemaphoreWaiters { void Add(const void* owner, u64 device_address, u32 value, u32 operation) { std::scoped_lock lk{mutex}; std::erase_if(waiters, [owner](const Waiter& w) { return w.owner == owner; }); - waiters.push_back({owner, device_address, value, operation, false}); + waiters.push_back({owner, device_address, value, operation, false, Source::Unknown}); count.store(waiters.size(), std::memory_order_release); } @@ -399,8 +409,19 @@ class Nx2SemaphoreWaiters { [owner](const Waiter& w) { return w.owner == owner && w.latched; }); } - /// A GPU write of `value` to the u32 at `device_address`. - void OnWrite(u64 device_address, u32 value) { + /// TEST: what latched `owner`'s waiter. + [[nodiscard]] Source LatchSource(const void* owner) { + std::scoped_lock lk{mutex}; + for (const Waiter& w : waiters) { + if (w.owner == owner && w.latched) { + return w.source; + } + } + return Source::Unknown; + } + + /// A GPU write of `value` to the u32 at `device_address` (by `source`, TEST). + void OnWrite(u64 device_address, u32 value, Source source = Source::Unknown) { if (count.load(std::memory_order_acquire) == 0) { return; } @@ -408,6 +429,7 @@ class Nx2SemaphoreWaiters { for (auto& w : waiters) { if (w.address == device_address && !w.latched && Matches(w, value)) { w.latched = true; + w.source = source; } } } @@ -419,6 +441,7 @@ class Nx2SemaphoreWaiters { u32 value; u32 operation; bool latched; + Source source; }; static bool Matches(const Waiter& w, u32 current) { diff --git a/src/video_core/nx2_drain_check.h b/src/video_core/nx2_drain_check.h index 85b63bab81..7906c5bdd9 100644 --- a/src/video_core/nx2_drain_check.h +++ b/src/video_core/nx2_drain_check.h @@ -9,12 +9,14 @@ #include #include #include +#include #include #include #include "common/common_types.h" #include "common/logging.h" +#include "video_core/nx2.h" // EDEN_NX2_DRAIN_CHECK: what the split GPU thread's front would see without some of its waits for // the back, counted without changing what runs and logged every 10 s by the front: @@ -24,9 +26,10 @@ // - semaphore acquires, and the polls of suspended ones (Puller): whether the value met the // acquire before the front drained the split and after, by whether the GPU had released a // semaphore at that address before (any engine, RasterizerInterface::Query with a payload); -// - acquires and polls decided without the drain (Puller CheckWithoutDrain), verified: the front -// drains and reads the word as before anyway, and counts the decisions the drain contradicts. -// A pass the drained word does not meet must stay 0; such a decision is corrected. +// - acquires and polls the semaphore view decides (Puller AskView), verified: the front drains +// and reads the word as before anyway, and counts the decisions the drain contradicts. A pass +// the drained word does not meet must stay 0; such a pass is corrected (the channel waits), and +// a wait the drained word meets ends (counted: a drain would have passed or resumed it). namespace VideoCommon::Nx2DrainCheck { // A plain flag: the front checks it at every command list and acquire. @@ -43,12 +46,14 @@ struct State { std::atomic lists_written{}; /// Indexed by poll * 8 + released * 4 + met before the drain * 2 + met after it. std::array, 16> acquires{}; - /// Passes without the drain, by poll * 2 + met by a release still queued, and those the - /// drained word did not meet. - std::array, 4> passes{}; - std::array, 4> wrong_passes{}; - std::atomic polls_kept{}; ///< Polls that kept waiting without the drain - std::atomic polls_kept_met{}; ///< Of those, met after the drain + /// Passes the semaphore view decided, by poll * 3 + ViewPass, and those the drained word did + /// not meet. + std::array, 6> view_passes{}; + std::array, 6> view_wrong{}; + /// Acquires the view suspended and polls it kept waiting (by poll), and of those the ones + /// the drained word met. + std::array, 2> view_waits{}; + std::array, 2> view_waits_met{}; std::mutex released_mutex; std::unordered_set released; ///< Device addresses of the semaphores the GPU released std::chrono::steady_clock::time_point last_report{}; ///< The front's @@ -80,23 +85,31 @@ inline void NoteAcquire(bool poll, bool released, bool met_before, bool met_afte Get().acquires[index].fetch_add(1, std::memory_order_relaxed); } -/// An acquire (or a poll, `poll`) passed without the drain, met by a release still queued -/// (`queued`) or by memory; whether the word read after a drain met it too. -inline void NotePass(bool poll, bool queued, bool met_after_drain) { - const size_t index = (poll ? 2U : 0U) + (queued ? 1U : 0U); +/// How the semaphore view passed an acquire or resumed a poll. +enum class ViewPass : u32 { + Memory, ///< Met in memory + Queued, ///< Met by the last release sent to the word, still queued + Waited, ///< Met after a positioned wait (GpuSplit::WaitForPosition) +}; + +/// The view passed an acquire (or resumed a poll, `poll`); whether the word read after a drain +/// met it too. +inline void NoteViewPass(bool poll, ViewPass how, bool met_after_drain) { + const size_t index = (poll ? 3U : 0U) + static_cast(how); State& state = Get(); - state.passes[index].fetch_add(1, std::memory_order_relaxed); + state.view_passes[index].fetch_add(1, std::memory_order_relaxed); if (!met_after_drain) { - state.wrong_passes[index].fetch_add(1, std::memory_order_relaxed); + state.view_wrong[index].fetch_add(1, std::memory_order_relaxed); } } -/// A poll kept waiting without the drain; whether the word read after a drain met it. -inline void NoteKeptWaiting(bool met_after_drain) { +/// The view suspended an acquire (or kept a poll waiting, `poll`); whether the word read after +/// a drain met it. +inline void NoteViewWait(bool poll, bool met_after_drain) { State& state = Get(); - state.polls_kept.fetch_add(1, std::memory_order_relaxed); + state.view_waits[poll ? 1 : 0].fetch_add(1, std::memory_order_relaxed); if (met_after_drain) { - state.polls_kept_met.fetch_add(1, std::memory_order_relaxed); + state.view_waits_met[poll ? 1 : 0].fetch_add(1, std::memory_order_relaxed); } } @@ -143,12 +156,20 @@ inline void MaybeReport() { neither); }; const auto passes = [&](bool poll) { - const size_t base = poll ? 2U : 0U; - const u64 by_memory = take(state.passes[base]); - const u64 by_queued = take(state.passes[base + 1]); - const u64 wrong = take(state.wrong_passes[base]) + take(state.wrong_passes[base + 1]); - return fmt::format("{} (by a release still queued {}), not met after the drain {}", - by_memory + by_queued, by_queued, wrong); + const size_t base = poll ? 3U : 0U; + const u64 by_memory = take(state.view_passes[base]); + const u64 by_queued = take(state.view_passes[base + 1]); + const u64 waited = take(state.view_passes[base + 2]); + const u64 wrong = take(state.view_wrong[base]) + take(state.view_wrong[base + 1]) + + take(state.view_wrong[base + 2]); + return fmt::format("{} (by memory {}, by a release still queued {}, after a positioned " + "wait {}), not met after the drain {}", + by_memory + by_queued + waited, by_memory, by_queued, waited, wrong); + }; + const auto waits = [&](bool poll) { + const u64 count = take(state.view_waits[poll ? 1 : 0]); + const u64 met = take(state.view_waits_met[poll ? 1 : 0]); + return fmt::format("{} (met after the drain {})", count, met); }; const u64 unwaited = take(state.lists_unwaited); const u64 written = take(state.lists_written); @@ -157,17 +178,209 @@ inline void MaybeReport() { const std::string polls_never = group(true, false); const std::string polls_released = group(true, true); const std::string acquire_passes = passes(false); + const std::string acquire_waits = waits(false); const std::string poll_passes = passes(true); - const u64 kept = take(state.polls_kept); - const u64 kept_met = take(state.polls_kept_met); + const std::string poll_waits = waits(true); LOG_INFO(HW_GPU, "Drain checks in {:.1f} s: command lists that did not wait for writers sent after the " - "last sync {}, GPU-written by those writers {} | acquires at addresses the GPU never " - "released {}, at ones it released {} | polls of suspended acquires at addresses never " - "released {}, at released ones {} | without the drain: acquires passed {}; polls " - "resumed {}, kept waiting {} (met after the drain {})", + "last sync {}, GPU-written by those writers {} | drained acquires at addresses the " + "GPU never released {}, at ones it released {} | drained polls of suspended acquires " + "at addresses never released {}, at released ones {} | semaphore view: acquires " + "passed {}, suspended {}; polls resumed {}, kept waiting {}", seconds, unwaited, written, acquires_never, acquires_released, polls_never, - polls_released, acquire_passes, poll_passes, kept, kept_met); + polls_released, acquire_passes, acquire_waits, poll_passes, poll_waits); } } // namespace VideoCommon::Nx2DrainCheck + +// EDEN_NX2_ACQUIRE_STATS: how the split's front decides Switch 2 semaphore acquires (Puller), every +// 10 s, without changing what runs: by operation, which of them still wait, at which words (on +// GPU-written pages, by the releases the front sent there), what resumes them and how long they +// waited. Front thread only, but for the writer that latched a waiter (Nx2SemaphoreWaiters). +namespace VideoCommon::Nx2AcquireStats { + +// A plain flag: the front checks it at every acquire and release it sends. +inline bool g_enabled = std::getenv("EDEN_NX2_ACQUIRE_STATS") != nullptr; + +/// Called by VideoCore::CreateGPU: static initialization runs before the frontend loads +/// nx2_switches.txt, so a switch set only there takes effect from that point on. +inline void ReadSwitch() { + g_enabled = std::getenv("EDEN_NX2_ACQUIRE_STATS") != nullptr; +} + +/// How the front decided an acquire. +enum class Decision : u32 { + Memory, ///< Passed: met in memory + Queued, ///< Passed: met by the last release sent to the word, still queued + Waited, ///< Passed after a positioned wait + Suspended, ///< The channel waits + Drained, ///< The semaphore view doesn't apply: drained as before + Count, +}; + +/// Who the front sent releases to a word for (State::noted bits). +enum class Releaser : u32 { + Puller, ///< Semaphore methods (WriteLong, SEM_EXECUTE release) + Report3D, ///< 3D report semaphores + ReportCompute, ///< Compute report semaphores + Copy, ///< Copy engine semaphores + Count, +}; + +/// What resumed a suspended acquire. +enum class Resume : u32 { + Latch, ///< A release the back ran latched the waiter (by its Source) + Memory, ///< Met in memory: the CPU, or a GPU write that latches no waiter + Queued, ///< Met by the last release sent to the word, still queued + Drain, ///< EDEN_NX2_DRAIN_CHECK's drain met it first + Count, +}; + +using Source = Tegra::Nx2SemaphoreWaiters::Source; + +struct State { + /// By operation (equal, greater or equal, mask) and Decision. + std::array(Decision::Count)>, 3> acquires{}; + u64 positioned_waits{}; + u64 suspended_gpu_written{}; ///< Words on GPU-written pages + u64 suspended_flush{}; ///< Of those, words the buffer cache held newer than memory + std::array(Releaser::Count)> suspended_noted{}; + u64 suspended_never_noted{}; ///< No release the front sent: QMD releases, the CPU + std::array(Resume::Count)> resumes{}; + std::array(Source::Count)> latches{}; + u64 resumes_after_flush{}; + u64 wait_ns{}; + u64 wait_max_ns{}; + u64 waits_over_1ms{}; + std::unordered_map noted; ///< Releaser bits by word + std::chrono::steady_clock::time_point last_report{}; +}; + +inline State& Get() { + static State state; + return state; +} + +[[nodiscard]] inline size_t OperationIndex(u32 operation) { + switch (operation) { + case Tegra::Nx2SemaphoreWaiters::AcquireEqual: + return 0; + case Tegra::Nx2SemaphoreWaiters::AcquireGequal: + return 1; + default: + return 2; + } +} + +/// The front sent a release to the word at `device_address`. +inline void NoteRelease(u64 device_address, Releaser releaser) { + if (!g_enabled) { + return; + } + State& state = Get(); + if (state.noted.size() < 65536) { + state.noted[device_address & ~u64{3}] |= static_cast(1U << static_cast(releaser)); + } +} + +/// An acquire of `operation` (Puller GpuSemaphoreOperation) at `device_address`, decided so. +/// Suspended: whether its word is on a GPU-written page, and the buffer cache held it newer. +inline void NoteAcquire(u32 operation, Decision decision, u64 device_address, bool waited, + bool gpu_written, bool flush) { + State& state = Get(); + ++state.acquires[OperationIndex(operation)][static_cast(decision)]; + if (waited) { + ++state.positioned_waits; + } + if (decision != Decision::Suspended) { + return; + } + if (gpu_written) { + ++state.suspended_gpu_written; + } + if (flush) { + ++state.suspended_flush; + } + const auto it = state.noted.find(device_address & ~u64{3}); + if (it == state.noted.end()) { + ++state.suspended_never_noted; + return; + } + for (u32 releaser = 0; releaser < static_cast(Releaser::Count); ++releaser) { + if ((it->second >> releaser) & 1) { + ++state.suspended_noted[releaser]; + } + } +} + +/// A suspended acquire resumed, after `waited`. +inline void NoteResume(Resume how, Source latch_source, bool after_flush, + std::chrono::steady_clock::duration waited) { + State& state = Get(); + ++state.resumes[static_cast(how)]; + if (how == Resume::Latch) { + ++state.latches[static_cast(latch_source)]; + } + if (after_flush) { + ++state.resumes_after_flush; + } + const u64 ns = static_cast(std::max( + 0, std::chrono::duration_cast(waited).count())); + state.wait_ns += ns; + state.wait_max_ns = std::max(state.wait_max_ns, ns); + if (ns > 1'000'000) { + ++state.waits_over_1ms; + } +} + +/// The front, between commands. +inline void MaybeReport() { + if (!g_enabled) { + return; + } + State& state = Get(); + const auto now = std::chrono::steady_clock::now(); + if (state.last_report == std::chrono::steady_clock::time_point{}) { + state.last_report = now; + return; + } + const double seconds = std::chrono::duration(now - state.last_report).count(); + if (seconds < 10.0) { + return; + } + state.last_report = now; + const auto by_operation = [&](size_t operation) { + const auto& counts = state.acquires[operation]; + u64 total = 0; + for (const u64 count : counts) { + total += count; + } + return fmt::format("{} (passed by memory {}, by a release still queued {}, after a " + "positioned wait {}; suspended {}; drained {})", + total, counts[0], counts[1], counts[2], counts[3], counts[4]); + }; + const auto& noted = state.suspended_noted; + const auto& resumes = state.resumes; + const auto& latches = state.latches; + const u64 resumed = resumes[0] + resumes[1] + resumes[2] + resumes[3]; + LOG_INFO(HW_GPU, + "Acquires in {:.1f} s (EDEN_NX2_ACQUIRE_STATS): equal {}, greater or equal {}, mask " + "{}; positioned waits {} | suspended at words on GPU-written pages {} (held newer by " + "the buffer cache {}), with releases sent by the puller {}, 3D reports {}, compute " + "reports {}, the copy engine {}, none (QMD releases, the CPU) {} | resumed {}: " + "latched {} (payload releases {}, query results {}, QMD reductions {}, QMD maxima {}, " + "other {}), by memory {}, by a queued release {}, by the drain check {}, after a " + "flush {} | waited {:.1f} ms, at most {:.3f} ms, over 1 ms {}", + seconds, by_operation(0), by_operation(1), by_operation(2), state.positioned_waits, + state.suspended_gpu_written, state.suspended_flush, noted[0], noted[1], noted[2], + noted[3], state.suspended_never_noted, resumed, resumes[0], latches[1], latches[2], + latches[3], latches[4], latches[0], resumes[1], resumes[2], resumes[3], + state.resumes_after_flush, static_cast(state.wait_ns) / 1e6, + static_cast(state.wait_max_ns) / 1e6, state.waits_over_1ms); + auto noted_words = std::move(state.noted); + state = State{}; + state.noted = std::move(noted_words); + state.last_report = now; +} + +} // namespace VideoCommon::Nx2AcquireStats diff --git a/src/video_core/nx2_profile.h b/src/video_core/nx2_profile.h index 8d3c1e3d94..58ed087c79 100644 --- a/src/video_core/nx2_profile.h +++ b/src/video_core/nx2_profile.h @@ -62,6 +62,7 @@ enum class Zone : u32 { WriterWait, ListSubmitWait, ListGpuWait, + AcquirePoll, ///< TEST: a suspended acquire's poll (CanResumeAcquire in ResumePending) Count, }; @@ -73,7 +74,7 @@ inline constexpr std::array(Zone::Count)> zone_ "descriptor acquire", "bind stage buffers", "push descriptors", "render targets", "configure draw", "dynamic state", "draw record", "texture upload", "fence", "fence skipped", "method", "split drain", "split wait", - "writer wait", "list submit wait", "list GPU wait", + "writer wait", "list submit wait", "list GPU wait", "acquire poll", }; // A plain flag: Enabled() runs several times per method write (no guard to check). diff --git a/src/video_core/query_cache/query_cache.h b/src/video_core/query_cache/query_cache.h index a62f8c924c..5144764827 100644 --- a/src/video_core/query_cache/query_cache.h +++ b/src/video_core/query_cache/query_cache.h @@ -325,7 +325,10 @@ void QueryCacheBase::CounterReport(GPUVAddr addr, QueryType counter_type u32 value = static_cast(query_base->value); std::memcpy(pointer, &value, sizeof(value)); } - Tegra::Nx2SemaphoreWaiters::Get().OnWrite(cpu_addr, static_cast(query_base->value)); + Tegra::Nx2SemaphoreWaiters::Get().OnWrite( + cpu_addr, static_cast(query_base->value), + counter_type == QueryType::Payload ? Tegra::Nx2SemaphoreWaiters::Source::PayloadRelease + : Tegra::Nx2SemaphoreWaiters::Source::QueryResult); if (const u64 trace = QueryTrace::Sample(QueryTrace::Event::Write)) { const bool wide = True(query_base->flags & QueryFlagBits::HasTimestamp); LOG_INFO(HW_GPU, @@ -352,7 +355,8 @@ void QueryCacheBase::CounterReport(GPUVAddr addr, QueryType counter_type } else { std::memcpy(pointer, &payload, sizeof(payload)); } - Tegra::Nx2SemaphoreWaiters::Get().OnWrite(cpu_addr, payload); + Tegra::Nx2SemaphoreWaiters::Get().OnWrite( + cpu_addr, payload, Tegra::Nx2SemaphoreWaiters::Source::PayloadRelease); if (const u64 trace = QueryTrace::Sample(QueryTrace::Event::Write)) { LOG_INFO(HW_GPU, "DKB_QUERY write n={} path=immediate as={} gpu={:#x} device={:#x} type={} " diff --git a/src/video_core/video_core.cpp b/src/video_core/video_core.cpp index f7a8ec608c..73831ee5e1 100644 --- a/src/video_core/video_core.cpp +++ b/src/video_core/video_core.cpp @@ -60,6 +60,7 @@ void CreateGPU(std::optional& gpu, Core::Frontend::EmuWindow& emu_wi VideoCommon::Nx2DrainCheck::ReadSwitch(); VideoCommon::ReadTrace::ReadSwitch(); Tegra::Nx2ListStats::ReadSwitches(); + VideoCommon::Nx2AcquireStats::ReadSwitch(); const auto nvdec_value = Settings::values.nvdec_emulation.GetValue(); const bool use_nvdec = nvdec_value != Settings::NvdecEmulation::Off; From 87a44cceceabff3d2bfaf7a0ebbc08c6f9812ad2 Mon Sep 17 00:00:00 2001 From: jpkto <255363096+jpkto@users.noreply.github.com> Date: Sun, 8 Nov 2026 00:32:52 +0100 Subject: [PATCH 030/105] video_core: share one ring position wait between back macros and acquires The back macros and the semaphore view each added GpuSplit::Position(), and a positioned wait with the same body (WaitUntil, WaitForPosition). Keep the semaphore view's pair and wait for back-owned registers and macro code through WaitForPosition. Rebased: the semaphore view commit no longer adds a second Position(), so this one removes only WaitUntil. Co-Authored-By: Claude Opus 5.5 --- src/video_core/engines/maxwell_3d.cpp | 6 +++--- src/video_core/gpu_split.h | 5 ----- 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/src/video_core/engines/maxwell_3d.cpp b/src/video_core/engines/maxwell_3d.cpp index da159a719d..44b84b9a2d 100644 --- a/src/video_core/engines/maxwell_3d.cpp +++ b/src/video_core/engines/maxwell_3d.cpp @@ -1641,7 +1641,7 @@ bool Maxwell3D::FrontRunOnBack(Core::System& system, VideoCommon::GpuSplit& spli }; if (fs.back_generation != engine.native_code_generation) [[unlikely]] { // The macros sent before run the code they were sent with, and check their records. - split.WaitUntil(fs.back_last, DrainSite::BackCode); + split.WaitForPosition(fs.back_last, DrainSite::BackCode); if (fs.back_escapes.load(std::memory_order_relaxed) != fs.back_escapes_seen) { take_escapes(); } @@ -1844,7 +1844,7 @@ std::vector> Maxwell3D::FrontRunTheOldWay(VideoCommon::GpuSp u32 Maxwell3D::FrontRegister(VideoCommon::GpuSplit& split, FrontState& fs, u32 method) { if (fs.back_regs[method] != 0) { const bool waited = - split.WaitUntil(fs.back_regs[method], VideoCommon::DrainSite::BackRegister); + split.WaitForPosition(fs.back_regs[method], VideoCommon::DrainSite::BackRegister); if (BackMacroChecks() != 0) [[unlikely]] { ++BackStats().reads; BackStats().read_waits += waited ? 1 : 0; @@ -1865,7 +1865,7 @@ bool Maxwell3D::FrontTakeOver(VideoCommon::GpuSplit& split, FrontState& fs, u32 const bool tracked = fs.control == Control::Track || fs.control == Control::TrackWithFilter; if (fs.back_shadow[method] != 0 && fs.control == Control::Replay) { const bool waited = - split.WaitUntil(fs.back_shadow[method], VideoCommon::DrainSite::BackRegister); + split.WaitForPosition(fs.back_shadow[method], VideoCommon::DrainSite::BackRegister); if (BackMacroChecks() != 0) [[unlikely]] { ++BackStats().replays; BackStats().replay_waits += waited ? 1 : 0; diff --git a/src/video_core/gpu_split.h b/src/video_core/gpu_split.h index d5a587a351..3938944a5b 100644 --- a/src/video_core/gpu_split.h +++ b/src/video_core/gpu_split.h @@ -228,11 +228,6 @@ class GpuSplit { void Publish(); /// Waits until the back executed everything sent so far. void Drain(DrainSite site = DrainSite::Other); - /// Waits until the back executed the entries sent before `position`. Returns whether any of - /// them was still queued. - bool WaitUntil(u64 position, DrainSite site) { - return WaitFor((std::min)(position, write_position), site); - } /// Compute or copy work was just sent: what can write GPU-written command lists. void NoteWriter() { From e11c916aca94c63ac0836ae9b9a1a53d0c4f6c13 Mon Sep 17 00:00:00 2001 From: jpkto <255363096+jpkto@users.noreply.github.com> Date: Sun, 8 Nov 2026 01:57:24 +0100 Subject: [PATCH 031/105] TEST: time the host GPU's work outside draws and dispatches (EDEN_NX2_GPU_COST) The host GPU is busy about 10 ms of a 20 ms frame in Ocarina of Time, and only about 3.4 ms of that is timed draws and dispatches. EDEN_NX2_GPU_COST=1 shows where the rest goes. It needs EDEN_DKB_FRAME_TRACE=1, whose frames it times. In a sampled submission both command buffers record through a copy of the dispatch table whose work entries (draws, dispatches, render pass begins and ends, barriers, copies, clears, fills, query commands, conditional rendering, transform feedback) write a bottom-of-pipe timestamp after the real call. Binds and dynamic state write none. The timestamps of a command buffer, from a top-of-pipe one at the submission's start to one at its end, split its GPU time into intervals. Each interval ends with one Vulkan command: the time from the end of the work before it to the end of that command's own. Each interval goes to the command's kind and to the source line that recorded it: - Scheduler::Record and RecordWithUploadBuffer take the caller's std::source_location, kept per command while the variable is set; - a render pass's end carries the line that ended it; - its begin carries the line that ended the pass before; - a submission's commands carry the line that asked for it; - BufferCacheRuntime::CopyBuffer passes its caller's line on, so its render pass ends and copies count at the upload, download or copy that asked. After each render pass end, the trace also notes what ran before the next begin (only barriers, copies, dispatches, queries, clears) and whether that begin resumed the same framebuffer. Each upload that can't go to the upload command buffer is also classified by where its pages were last used: done on the GPU, submitted but not done, or in the command buffer being recorded. Every 10 seconds, over the frames sampled in that time, per frame: - "GPU cost": busy time, then each category with its kinds; - "GPU cost by site": the 28 source lines and kinds with the most time; - "Render pass breaks at ": for the 14 lines that ended the most passes, the draws in each pass, how many had storage writes, the next begin, what ran between, and the GPU time of the end, the work between and the next begin; - "Uploads": uploads a second by where they ran and why. The "Render pass ends" lines of EDEN_NX2_GPU_PROFILE also name the function now. The trace only reads: the commands, their order and their synchronization stay as they are. With the variable unset nothing changes. Co-Authored-By: Claude Opus 5.5 --- src/video_core/CMakeLists.txt | 2 + .../renderer_vulkan/vk_buffer_cache.cpp | 132 ++- .../renderer_vulkan/vk_buffer_cache.h | 20 +- .../renderer_vulkan/vk_gpu_cost.cpp | 1026 +++++++++++++++++ src/video_core/renderer_vulkan/vk_gpu_cost.h | 72 ++ .../renderer_vulkan/vk_scheduler.cpp | 92 +- src/video_core/renderer_vulkan/vk_scheduler.h | 78 +- 7 files changed, 1354 insertions(+), 68 deletions(-) create mode 100644 src/video_core/renderer_vulkan/vk_gpu_cost.cpp create mode 100644 src/video_core/renderer_vulkan/vk_gpu_cost.h diff --git a/src/video_core/CMakeLists.txt b/src/video_core/CMakeLists.txt index c64b800b19..a4e9c6b30f 100644 --- a/src/video_core/CMakeLists.txt +++ b/src/video_core/CMakeLists.txt @@ -210,6 +210,8 @@ add_library(video_core STATIC renderer_vulkan/vk_fence_manager.h renderer_vulkan/vk_frame_dump.cpp renderer_vulkan/vk_frame_dump.h + renderer_vulkan/vk_gpu_cost.cpp + renderer_vulkan/vk_gpu_cost.h renderer_vulkan/vk_graphics_pipeline.cpp renderer_vulkan/vk_graphics_pipeline.h renderer_vulkan/vk_multi_range_buffer.cpp diff --git a/src/video_core/renderer_vulkan/vk_buffer_cache.cpp b/src/video_core/renderer_vulkan/vk_buffer_cache.cpp index 0aadab8e8c..70d69e797c 100644 --- a/src/video_core/renderer_vulkan/vk_buffer_cache.cpp +++ b/src/video_core/renderer_vulkan/vk_buffer_cache.cpp @@ -6,11 +6,16 @@ #include #include +#include #include #include +#include #include +#include #include +#include + #include "video_core/buffer_cache/buffer_cache_base.h" #include "video_core/nx2_perf.h" #include "video_core/nx2_profile.h" @@ -18,6 +23,7 @@ #include "video_core/renderer_vulkan/vertex_repack.h" #include "video_core/renderer_vulkan/maxwell_to_vk.h" +#include "video_core/renderer_vulkan/vk_gpu_cost.h" #include "video_core/renderer_vulkan/vk_scheduler.h" #include "video_core/renderer_vulkan/vk_staging_buffer_pool.h" #include "video_core/renderer_vulkan/vk_update_descriptor.h" @@ -28,6 +34,9 @@ namespace Vulkan { namespace { +/// TEST (EDEN_NX2_GPU_COST): Buffer::MarkUsage notes page ticks, CanReorderUpload classifies. +const bool g_trace_uploads = GpuCost::Enabled(); + void HostReadBarrier(vk::CommandBuffer cmdbuf) { const VkMemoryBarrier barrier{ .sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER, @@ -138,6 +147,34 @@ Buffer::Buffer(BufferCacheRuntime& runtime, DAddr cpu_addr_, u64 size_bytes_, void Buffer::MarkUsage(u64 offset, u64 size) noexcept { tracker.Track(offset, size); last_usage_tick = scheduler->CurrentTick(); + if (g_trace_uploads) [[unlikely]] { + const std::array use{offset, size, last_usage_tick}; + if (std::ranges::find(noted_uses, use) != noted_uses.end() || size == 0) { + return; + } + noted_uses[noted_cursor++ % noted_uses.size()] = use; + if (use_ticks.empty()) { + use_ticks.resize(static_cast(SizeBytes() >> 12) + 1); + } + const size_t last = (std::min)(static_cast((offset + size - 1) >> 12), + use_ticks.size() - 1); + for (size_t page = static_cast(offset >> 12); page <= last; ++page) { + use_ticks[page] = last_usage_tick; + } + } +} + +u64 Buffer::NewestUseTick(u64 offset, u64 size) const noexcept { + u64 newest = 0; + if (size == 0 || use_ticks.empty()) { + return newest; + } + const size_t last = + (std::min)(static_cast((offset + size - 1) >> 12), use_ticks.size() - 1); + for (size_t page = static_cast(offset >> 12); page <= last; ++page) { + newest = (std::max)(newest, use_ticks[page]); + } + return newest; } VkBufferView Buffer::View(u32 offset, u32 size, VideoCore::Surface::PixelFormat format) { @@ -582,12 +619,69 @@ bool BufferCacheRuntime::CanReorderUpload(const Buffer& buffer, std::ranges::all_of(copies, [&](const VideoCommon::BufferCopy& copy) { return !buffer.IsRegionUsed(copy.dst_offset, copy.size); }); + if (g_trace_uploads) [[unlikely]] { + TraceUpload(buffer, copies, can_use_upload_cmdbuf); + } return can_use_upload_cmdbuf; } +void BufferCacheRuntime::TraceUpload(const Buffer& buffer, + std::span copies, + bool reordered) { + // TEST (EDEN_NX2_GPU_COST): where the destination's last use was when an upload has to run + // in order (main command buffer, ends the render pass). The usage bits only clear when the + // whole buffer is idle at a frame's end (TickFrame). + enum Class : size_t { Reordered, LastUseDone, InFlight, ThisCommandBuffer, NoTick, Count }; + static constexpr std::array names{ + "reorderable (upload command buffer)", "in order, last use of its pages done", + "in order, last use submitted, not done", "in order, used in this command buffer", + "in order, no use noted"}; + struct Totals { + std::array count{}; + std::array bytes{}; + std::chrono::steady_clock::time_point start = std::chrono::steady_clock::now(); + }; + static std::mutex mutex; + static Totals totals; + u64 newest = 0; + u64 bytes = 0; + for (const VideoCommon::BufferCopy& copy : copies) { + newest = (std::max)(newest, buffer.NewestUseTick(copy.dst_offset, copy.size)); + bytes += copy.size; + } + const auto done = [&] { + if (scheduler.IsFree(newest)) { + return true; + } + scheduler.GetMasterSemaphore().Refresh(); + return scheduler.IsFree(newest); + }; + const Class kind = reordered ? Reordered + : newest == 0 ? NoTick + : newest >= scheduler.CurrentTick() ? ThisCommandBuffer + : !done() ? InFlight + : LastUseDone; + std::scoped_lock lock{mutex}; + ++totals.count[kind]; + totals.bytes[kind] += bytes; + const auto now = std::chrono::steady_clock::now(); + if (now - totals.start < std::chrono::seconds{10}) { + return; + } + const double seconds = std::chrono::duration(now - totals.start).count(); + std::string line; + for (size_t i = 0; i < Count; ++i) { + line += fmt::format("{}{}: {:.0f}/s {:.0f} KiB/s", i == 0 ? "" : ", ", names[i], + static_cast(totals.count[i]) / seconds, + static_cast(totals.bytes[i]) / 1024.0 / seconds); + } + LOG_INFO(Render_Vulkan, "Uploads in {:.1f} s: {}", seconds, line); + totals = {}; +} + void BufferCacheRuntime::CopyBuffer(VkBuffer dst_buffer, VkBuffer src_buffer, std::span copies, bool barrier, - bool can_reorder_upload) { + bool can_reorder_upload, std::source_location location) { if (dst_buffer == VK_NULL_HANDLE || src_buffer == VK_NULL_HANDLE) { return; } @@ -608,25 +702,29 @@ void BufferCacheRuntime::CopyBuffer(VkBuffer dst_buffer, VkBuffer src_buffer, boost::container::small_vector vk_copies(copies.size()); std::ranges::transform(copies, vk_copies.begin(), MakeBufferCopy); if (src_buffer == staging_pool.StreamBuf() && can_reorder_upload) { - scheduler.RecordWithUploadBuffer([src_buffer, dst_buffer, vk_copies]( - vk::CommandBuffer, vk::CommandBuffer upload_cmdbuf) { - upload_cmdbuf.CopyBuffer(src_buffer, dst_buffer, vk_copies); - }); + scheduler.RecordWithUploadBuffer( + [src_buffer, dst_buffer, vk_copies](vk::CommandBuffer, + vk::CommandBuffer upload_cmdbuf) { + upload_cmdbuf.CopyBuffer(src_buffer, dst_buffer, vk_copies); + }, + location); return; } - scheduler.RequestOutsideRenderPassOperationContext(); - scheduler.Record([src_buffer, dst_buffer, vk_copies, barrier](vk::CommandBuffer cmdbuf) { - if (barrier) { - cmdbuf.PipelineBarrier(vk::PIPELINE_STAGE_GRAPHICS_COMPUTE_TRANSFER, - VK_PIPELINE_STAGE_TRANSFER_BIT, 0, READ_BARRIER); - } - cmdbuf.CopyBuffer(src_buffer, dst_buffer, vk_copies); - if (barrier) { - cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_TRANSFER_BIT, - vk::PIPELINE_STAGE_GRAPHICS_COMPUTE, 0, WRITE_BARRIER); - } - }); + scheduler.RequestOutsideRenderPassOperationContext(location); + scheduler.Record( + [src_buffer, dst_buffer, vk_copies, barrier](vk::CommandBuffer cmdbuf) { + if (barrier) { + cmdbuf.PipelineBarrier(vk::PIPELINE_STAGE_GRAPHICS_COMPUTE_TRANSFER, + VK_PIPELINE_STAGE_TRANSFER_BIT, 0, READ_BARRIER); + } + cmdbuf.CopyBuffer(src_buffer, dst_buffer, vk_copies); + if (barrier) { + cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_TRANSFER_BIT, + vk::PIPELINE_STAGE_GRAPHICS_COMPUTE, 0, WRITE_BARRIER); + } + }, + location); } void BufferCacheRuntime::PreCopyBarrier() { diff --git a/src/video_core/renderer_vulkan/vk_buffer_cache.h b/src/video_core/renderer_vulkan/vk_buffer_cache.h index b98cd89b5f..f042a1a8b7 100644 --- a/src/video_core/renderer_vulkan/vk_buffer_cache.h +++ b/src/video_core/renderer_vulkan/vk_buffer_cache.h @@ -6,7 +6,10 @@ #pragma once +#include #include +#include +#include #include @@ -59,6 +62,10 @@ class Buffer : public VideoCommon::BufferBase { return tracker.IsUsed(offset, size); } + /// TEST (EDEN_NX2_GPU_COST): the newest tick that used any 4 KiB page of the range, 0 when + /// none did since the trace started. + [[nodiscard]] u64 NewestUseTick(u64 offset, u64 size) const noexcept; + void MarkUsage(u64 offset, u64 size) noexcept; void ResetUsageTracking() noexcept { @@ -94,6 +101,11 @@ class Buffer : public VideoCommon::BufferBase { u64 last_usage_tick{}; bool is_null{}; bool sparse_compatible{}; + /// TEST (EDEN_NX2_GPU_COST): the tick of each 4 KiB page's last use, and the last ranges + /// noted with their tick (draws bind the same ranges again and again). + std::vector use_ticks; + std::array, 4> noted_uses{}; + u32 noted_cursor{}; }; class QuadArrayIndexBuffer; @@ -176,9 +188,11 @@ class BufferCacheRuntime { void PreCopyBarrier(); + /// location: the caller, which the render pass end and the copy are counted at. void CopyBuffer(VkBuffer dst_buffer, VkBuffer src_buffer, std::span copies, bool barrier, - bool can_reorder_upload = false); + bool can_reorder_upload = false, + std::source_location location = std::source_location::current()); void PostCopyBarrier(); @@ -317,6 +331,10 @@ class BufferCacheRuntime { VkFormat TexelBufferFormat(VideoCore::Surface::PixelFormat format) const; + /// TEST (EDEN_NX2_GPU_COST): counts an upload by where it ran and why. + void TraceUpload(const Buffer& buffer, std::span copies, + bool reordered); + void ReserveNullBuffer(); vk::Buffer CreateNullBuffer(); diff --git a/src/video_core/renderer_vulkan/vk_gpu_cost.cpp b/src/video_core/renderer_vulkan/vk_gpu_cost.cpp new file mode 100644 index 0000000000..aa9a8cebe3 --- /dev/null +++ b/src/video_core/renderer_vulkan/vk_gpu_cost.cpp @@ -0,0 +1,1026 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "common/dkb_frame_trace.h" +#include "common/logging.h" +#include "video_core/renderer_vulkan/vk_gpu_cost.h" +#include "video_core/renderer_vulkan/vk_scheduler.h" +#include "video_core/vulkan_common/vulkan_device.h" + +namespace Vulkan { +namespace { + +enum class Op : u8 { + SubmitStart, ///< top of pipe at the submission's start: the origin of its intervals + PriorTail, ///< until the work submitted before it is done + MainStart, ///< from the upload command buffer's end to the main one's start + SubmitEnd, ///< from the last command's work to the submission's end + Draw, + DrawIndirect, + Dispatch, + DispatchIndirect, + PassBegin, + PassBeginClear, + PassResume, ///< a begin on the framebuffer of the pass ended last + PassResumeClear, + PassEnd, + Barrier, + ImageBarrier, + LayoutBarrier, ///< an image barrier that changes a layout + WaitEvents, + SetEvent, + CopyBuffer, + CopyImage, + CopyBufferToImage, + CopyImageToBuffer, + BlitImage, + ResolveImage, + ClearColorImage, + ClearAttachments, + FillBuffer, + QueryBegin, + QueryEnd, + QueryReset, + QueryCopy, + ConditionalBegin, + ConditionalEnd, + XfbBegin, + XfbEnd, + Count, +}; +constexpr size_t NUM_OPS = static_cast(Op::Count); + +constexpr std::array OP_NAMES{ + "submission start", "prior work", "main start", "submission end", "draw", "indirect draw", + "dispatch", "indirect dispatch", "begin", "begin+clear", "resume", "resume+clear", "end", + "barrier", "image barrier", "layout change", "wait events", "set event", "copy buffer", + "copy image", "buffer to image", "image to buffer", "blit", "resolve", "clear image", + "clear attachments", "fill buffer", "query begin", "query end", "query reset", "query copy", + "conditional begin", "conditional end", "xfb begin", "xfb end", +}; + +static_assert(OP_NAMES[NUM_OPS - 1] != nullptr); + +constexpr bool IsMarker(Op op) { + return op <= Op::SubmitEnd; +} + +constexpr bool IsPassBegin(Op op) { + return op >= Op::PassBegin && op <= Op::PassResumeClear; +} + +constexpr bool IsBarrier(Op op) { + return op >= Op::Barrier && op <= Op::SetEvent; +} + +/// What ran between a render pass's end and the next begin. +enum Between : u8 { + BetweenBarrier = 1 << 0, + BetweenCopy = 1 << 1, + BetweenDispatch = 1 << 2, + BetweenQuery = 1 << 3, + BetweenClear = 1 << 4, + BetweenOther = 1 << 5, +}; +constexpr size_t NUM_BETWEEN = 6; +constexpr std::array BETWEEN_NAMES{ + "barriers", "copies", "dispatches", "queries", "clears", "other", +}; +static_assert(BETWEEN_NAMES[NUM_BETWEEN - 1] != nullptr); + +constexpr u8 BetweenBit(Op op) { + if (IsBarrier(op)) { + return BetweenBarrier; + } + switch (op) { + case Op::CopyBuffer: + case Op::CopyImage: + case Op::CopyBufferToImage: + case Op::CopyImageToBuffer: + case Op::BlitImage: + case Op::ResolveImage: + return BetweenCopy; + case Op::Dispatch: + case Op::DispatchIndirect: + return BetweenDispatch; + case Op::QueryBegin: + case Op::QueryEnd: + case Op::QueryReset: + case Op::QueryCopy: + return BetweenQuery; + case Op::ClearColorImage: + case Op::FillBuffer: + return BetweenClear; + default: + return BetweenOther; + } +} + +/// The groups of the "GPU cost" line; every timed interval is in exactly one. +enum class Category : u8 { + Draws, + Dispatches, + PassEnds, + PassBegins, + Barriers, + Uploads, + BufferCopies, + ImageUploads, + ImageDownloads, + ImageCopies, + Clears, + Queries, + Other, + Submissions, + Present, + Count, +}; +constexpr size_t NUM_CATEGORIES = static_cast(Category::Count); +constexpr std::array CATEGORY_NAMES{ + "draws", "dispatches", "pass ends (with the barrier after)", + "pass begins", "barriers", "uploads (upload command buffer)", + "buffer copies", "image uploads", "image downloads", + "image copies", "clears", "queries", + "conditional/xfb", "submissions", "present path", +}; +static_assert(CATEGORY_NAMES[NUM_CATEGORIES - 1] != nullptr); + +constexpr Category CategoryOf(RecordSite::Role role, Op op, bool upload, bool present) { + if (present) { + return Category::Present; + } + if (IsMarker(op) || role == RecordSite::Submit) { + return Category::Submissions; + } + if (role == RecordSite::PassEnd || op == Op::PassEnd) { + return Category::PassEnds; + } + if (role == RecordSite::PassBegin || IsPassBegin(op)) { + return Category::PassBegins; + } + if (IsBarrier(op)) { + return Category::Barriers; + } + switch (op) { + case Op::Draw: + case Op::DrawIndirect: + return Category::Draws; + case Op::Dispatch: + case Op::DispatchIndirect: + return Category::Dispatches; + case Op::CopyBuffer: + return upload ? Category::Uploads : Category::BufferCopies; + case Op::CopyBufferToImage: + return Category::ImageUploads; + case Op::CopyImageToBuffer: + return Category::ImageDownloads; + case Op::CopyImage: + case Op::BlitImage: + case Op::ResolveImage: + return Category::ImageCopies; + case Op::ClearColorImage: + case Op::ClearAttachments: + case Op::FillBuffer: + return Category::Clears; + case Op::QueryBegin: + case Op::QueryEnd: + case Op::QueryReset: + case Op::QueryCopy: + return Category::Queries; + default: + return Category::Other; + } +} + +constexpr u32 NO_SITE = ~0u; +constexpr u32 NO_EPISODE = ~0u; +constexpr u32 CAPACITY = 32768; +/// Kept for the submissions' start and end timestamps once the others overflow. +constexpr u32 MARKER_RESERVE = 1024; + +enum class Next : u8 { + None, ///< no begin in the sampled frame after it + Same, ///< the next begin has the framebuffer of the ended pass + Other, ///< another framebuffer +}; + +struct TimedOp { + u32 site; + u32 episode; + u32 bytes; + u16 submission; + Op op; + bool upload; +}; + +/// One render pass end and what came before the next begin. +struct Episode { + u32 site; + u32 draws; + VkFramebuffer framebuffer; + u8 between; + Next next; + bool stores; + bool crossed; ///< a submission ended before the next begin + u64 end_ns; + u64 between_ns; + u64 begin_ns; +}; + +struct SiteInfo { + std::string name; + RecordSite::Role role; + bool present; +}; + +struct Cell { + u64 ns; + u64 count; + u64 bytes; +}; + +struct QueryResult { + u64 value; + u64 available; +}; + +struct BreakTotals { + u64 count; + u64 draws; + u64 stores; + u64 same; + u64 other; + u64 none; + u64 crossed; + u64 only_barriers; + u64 nothing; + std::array between; + u64 end_ns; + u64 between_ns; + u64 begin_ns; +}; + +std::string_view BaseName(std::string_view path) { + if (const size_t slash = path.find_last_of("/\\"); slash != std::string_view::npos) { + path.remove_prefix(slash + 1); + } + return path; +} + +/// "Class::Function" from a compiler's function signature (MSVC's has the return type, the +/// calling convention and the parameters; a lambda's ends in "operator ()"). +std::string ShortFunction(std::string_view signature) { + std::string name; + int depth = 0; + for (const char c : signature) { + if (c == '(' && depth == 0) { + break; + } + if (c == '<') { + ++depth; + } else if (c == '>') { + depth = (std::max)(depth - 1, 0); + } else if (depth == 0) { + name.push_back(c); + } + } + const auto trim = [&name] { + while (!name.empty() && (name.back() == ' ' || name.back() == ':')) { + name.pop_back(); + } + }; + trim(); + if (name.ends_with("operator")) { + name.resize(name.size() - 8); + trim(); + } + if (const size_t space = name.rfind(' '); space != std::string::npos) { + name.erase(0, space + 1); + } + size_t doubled = name.find("::::"); + while (doubled != std::string::npos) { + name.erase(doubled, 2); + doubled = name.find("::::"); + } + if (const size_t last = name.rfind("::"); last != std::string::npos && last > 0) { + if (const size_t before = name.rfind("::", last - 1); before != std::string::npos) { + name.erase(0, before + 2); + } + } + return name; +} + +bool IsPresentPath(std::string_view path) { + return path.find("present/") != std::string_view::npos || + path.find("present\\") != std::string_view::npos || + path.find("vk_blit_screen") != std::string_view::npos || + path.find("renderer_vulkan.cpp") != std::string_view::npos || + path.find("vk_present_manager") != std::string_view::npos; +} + +} // Anonymous namespace + +struct GpuCost::Impl { + Impl(const Device& device_, vk::QueryPool pool_, u32 valid_bits_, double ns_per_tick_) + : device{device_}, real{device_.GetDispatchLoader()}, traced{real}, + pool{std::move(pool_)}, valid_bits{valid_bits_}, ns_per_tick{ns_per_tick_}, + host_reset{device_.IsHostQueryResetSupported()} { + stamps.resize(CAPACITY); + window_start = std::chrono::steady_clock::now(); + } + + void Stamp(VkCommandBuffer cmdbuf, Op op, u64 bytes, u32 episode) { + if (!open) { + return; + } + if (count >= (IsMarker(op) ? CAPACITY : CAPACITY - MARKER_RESERVE)) { + ++frame_overflow; + return; + } + real.vkCmdWriteTimestamp(cmdbuf, + op == Op::SubmitStart ? VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT + : VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT, + *pool, count); + stamps[count] = { + .site = current_site, + .episode = episode, + .bytes = static_cast((std::min)(bytes, u64{0xffffffff})), + .submission = submission, + .op = op, + .upload = cmdbuf == upload_handle, + }; + ++count; + } + + /// Any command that does work, after it is recorded. + void Work(VkCommandBuffer cmdbuf, Op op, u64 bytes) { + if (cmdbuf == upload_handle) { + Stamp(cmdbuf, op, bytes, NO_EPISODE); + return; + } + if ((op == Op::Draw || op == Op::DrawIndirect) && in_pass) { + ++pass_draws; + } + if (pending != NO_EPISODE && current_role == RecordSite::Own) { + Episode& episode = episodes[pending]; + episode.between = static_cast(episode.between | BetweenBit(op)); + } + Stamp(cmdbuf, op, bytes, pending); + } + + void PassEnded(VkCommandBuffer cmdbuf) { + if (open) { + pending = static_cast(episodes.size()); + episodes.push_back({ + .site = current_site, + .draws = pass_draws, + .framebuffer = pass_framebuffer, + .between = 0, + .next = Next::None, + .stores = (current_flags & RecordSite::PassHadStores) != 0, + .crossed = false, + .end_ns = 0, + .between_ns = 0, + .begin_ns = 0, + }); + } + in_pass = false; + Stamp(cmdbuf, Op::PassEnd, 0, pending); + } + + void PassBegan(VkCommandBuffer cmdbuf, const VkRenderPassBeginInfo& info) { + const u32 closing = pending; + bool same = false; + if (closing != NO_EPISODE) { + Episode& episode = episodes[closing]; + same = episode.framebuffer == info.framebuffer; + episode.next = same ? Next::Same : Next::Other; + pending = NO_EPISODE; + } + in_pass = true; + pass_framebuffer = info.framebuffer; + pass_draws = 0; + const bool clear = info.clearValueCount != 0; + const Op op = same ? (clear ? Op::PassResumeClear : Op::PassResume) + : (clear ? Op::PassBeginClear : Op::PassBegin); + Stamp(cmdbuf, op, 0, closing); + } + + u32 Intern(const RecordSite& site) { + const auto key = std::make_pair(std::make_pair(site.file, site.line), + static_cast(site.role)); + if (const auto it = site_keys.find(key); it != site_keys.end()) { + return it->second; + } + std::string name = SiteName(site.file, site.line, site.function); + const auto [named, inserted] = + site_names.try_emplace(fmt::format("{}#{}", name, static_cast(site.role)), + static_cast(sites.size())); + if (inserted) { + sites.push_back({ + .name = std::move(name), + .role = site.role, + .present = IsPresentPath(site.file), + }); + } + site_keys.emplace(key, named->second); + return named->second; + } + + void StartFrame() { + count = 0; + submission = 0; + frame_overflow = 0; + episodes.clear(); + pending = NO_EPISODE; + in_pass = false; + frame_open = true; + } + + [[nodiscard]] u64 DeltaNs(u64 begin, u64 end) const { + const u64 ticks = Common::DkbFrameTrace::TimestampDelta(begin, end, valid_bits); + return static_cast(static_cast(ticks) * ns_per_tick); + } + + void Attribute(u32 index, u64 ns) { + const TimedOp& stamp = stamps[index]; + const bool has_site = stamp.site != NO_SITE; + const RecordSite::Role role = has_site ? sites[stamp.site].role : RecordSite::Own; + const bool present = has_site && sites[stamp.site].present; + Cell& cell = cells[static_cast(role)][static_cast(stamp.op)] + [stamp.upload ? 1 : 0][present ? 1 : 0]; + cell.ns += ns; + ++cell.count; + cell.bytes += stamp.bytes; + if (has_site) { + Cell& site_cell = by_site[(u64{stamp.site} << 8) | static_cast(stamp.op)]; + site_cell.ns += ns; + ++site_cell.count; + site_cell.bytes += stamp.bytes; + } + if (stamp.episode == NO_EPISODE) { + return; + } + Episode& episode = episodes[stamp.episode]; + if (stamp.op == Op::PassEnd || role == RecordSite::PassEnd) { + episode.end_ns += ns; + } else if (IsPassBegin(stamp.op) || role == RecordSite::PassBegin) { + episode.begin_ns += ns; + } else { + episode.between_ns += ns; + } + } + + void ReadFrame(); + void Report(); + + const Device& device; + const vk::DeviceDispatch real; + vk::DeviceDispatch traced; + vk::QueryPool pool; + const u32 valid_bits; + const double ns_per_tick; + const bool host_reset; + + // The sampled frame (worker thread). + std::vector stamps; + std::vector episodes; + u32 count{}; + u16 submission{}; + u64 frame_overflow{}; + bool frame_open{}; + bool open{}; ///< in a sampled submission + VkCommandBuffer main_handle{}; + VkCommandBuffer upload_handle{}; + + // The command running. + u32 current_site = NO_SITE; + RecordSite::Role current_role = RecordSite::Own; + u8 current_flags{}; + + // The render pass state the commands left. + bool in_pass{}; + VkFramebuffer pass_framebuffer{}; + u32 pass_draws{}; + u32 pending = NO_EPISODE; ///< the episode of the last end, until the next begin + + // Source lines seen, by pointer and by name (a header's line has a pointer in each TU). + std::vector sites; + std::unordered_map site_names; + struct KeyHash { + size_t operator()(const std::pair, u32>& key) const { + return std::hash{}(key.first.first) ^ + (static_cast(key.first.second) << 8) ^ key.second; + } + }; + std::unordered_map, u32>, u32, KeyHash> site_keys; + + // The 10 s window. + std::chrono::steady_clock::time_point window_start; + std::array, 2>, NUM_OPS>, 4> cells{}; + std::unordered_map by_site; + std::unordered_map breaks; + u64 frames{}; + u64 busy_ns{}; + u64 submissions{}; + u64 stamped{}; + u64 overflow{}; + u64 bad{}; + std::vector results; +}; + +namespace { + +GpuCost::Impl* g_impl{}; + +/// A vkCmd* entry that runs the real one, then notes what it did. +template +struct Hook; + +template +struct Hook { + template + static VKAPI_ATTR void VKAPI_CALL Call(VkCommandBuffer cmdbuf, Args... args) { + GpuCost::Impl& self = *g_impl; + (self.real.*member)(cmdbuf, args...); + self.Work(cmdbuf, op, 0); + } +}; + +template +void Install(vk::DeviceDispatch& traced) { + using Pfn = std::remove_reference_t; + if (traced.*member != nullptr) { + traced.*member = &Hook::template Call; + } +} + +Op BarrierOp(u32 image_count, const VkImageMemoryBarrier* images) { + if (image_count == 0) { + return Op::Barrier; + } + for (u32 i = 0; i < image_count; ++i) { + if (images[i].oldLayout != images[i].newLayout) { + return Op::LayoutBarrier; + } + } + return Op::ImageBarrier; +} + +VKAPI_ATTR void VKAPI_CALL PipelineBarrier(VkCommandBuffer cmdbuf, VkPipelineStageFlags src, + VkPipelineStageFlags dst, VkDependencyFlags flags, + u32 memory_count, const VkMemoryBarrier* memory, + u32 buffer_count, const VkBufferMemoryBarrier* buffers, + u32 image_count, const VkImageMemoryBarrier* images) { + GpuCost::Impl& self = *g_impl; + self.real.vkCmdPipelineBarrier(cmdbuf, src, dst, flags, memory_count, memory, buffer_count, + buffers, image_count, images); + self.Work(cmdbuf, BarrierOp(image_count, images), 0); +} + +VKAPI_ATTR void VKAPI_CALL PipelineBarrier2(VkCommandBuffer cmdbuf, const VkDependencyInfo* info) { + GpuCost::Impl& self = *g_impl; + self.real.vkCmdPipelineBarrier2(cmdbuf, info); + Op op = info->imageMemoryBarrierCount == 0 ? Op::Barrier : Op::ImageBarrier; + for (u32 i = 0; i < info->imageMemoryBarrierCount; ++i) { + if (info->pImageMemoryBarriers[i].oldLayout != info->pImageMemoryBarriers[i].newLayout) { + op = Op::LayoutBarrier; + } + } + self.Work(cmdbuf, op, 0); +} + +VKAPI_ATTR void VKAPI_CALL CopyBuffer(VkCommandBuffer cmdbuf, VkBuffer src, VkBuffer dst, + u32 region_count, const VkBufferCopy* regions) { + GpuCost::Impl& self = *g_impl; + self.real.vkCmdCopyBuffer(cmdbuf, src, dst, region_count, regions); + u64 bytes = 0; + for (u32 i = 0; i < region_count; ++i) { + bytes += regions[i].size; + } + self.Work(cmdbuf, Op::CopyBuffer, bytes); +} + +VKAPI_ATTR void VKAPI_CALL BeginRenderPass(VkCommandBuffer cmdbuf, + const VkRenderPassBeginInfo* info, + VkSubpassContents contents) { + GpuCost::Impl& self = *g_impl; + self.real.vkCmdBeginRenderPass(cmdbuf, info, contents); + self.PassBegan(cmdbuf, *info); +} + +VKAPI_ATTR void VKAPI_CALL EndRenderPass(VkCommandBuffer cmdbuf) { + GpuCost::Impl& self = *g_impl; + self.real.vkCmdEndRenderPass(cmdbuf); + self.PassEnded(cmdbuf); +} + +void InstallHooks(vk::DeviceDispatch& traced) { + using D = vk::DeviceDispatch; + Install<&D::vkCmdDraw, Op::Draw>(traced); + Install<&D::vkCmdDrawIndexed, Op::Draw>(traced); + Install<&D::vkCmdDrawMeshTasksEXT, Op::Draw>(traced); + Install<&D::vkCmdDrawIndirect, Op::DrawIndirect>(traced); + Install<&D::vkCmdDrawIndexedIndirect, Op::DrawIndirect>(traced); + Install<&D::vkCmdDrawIndirectCount, Op::DrawIndirect>(traced); + Install<&D::vkCmdDrawIndexedIndirectCount, Op::DrawIndirect>(traced); + Install<&D::vkCmdDrawIndirectByteCountEXT, Op::DrawIndirect>(traced); + Install<&D::vkCmdDispatch, Op::Dispatch>(traced); + Install<&D::vkCmdDispatchIndirect, Op::DispatchIndirect>(traced); + Install<&D::vkCmdWaitEvents, Op::WaitEvents>(traced); + Install<&D::vkCmdSetEvent, Op::SetEvent>(traced); + Install<&D::vkCmdCopyImage, Op::CopyImage>(traced); + Install<&D::vkCmdCopyBufferToImage, Op::CopyBufferToImage>(traced); + Install<&D::vkCmdCopyImageToBuffer, Op::CopyImageToBuffer>(traced); + Install<&D::vkCmdBlitImage, Op::BlitImage>(traced); + Install<&D::vkCmdResolveImage, Op::ResolveImage>(traced); + Install<&D::vkCmdClearColorImage, Op::ClearColorImage>(traced); + Install<&D::vkCmdClearAttachments, Op::ClearAttachments>(traced); + Install<&D::vkCmdFillBuffer, Op::FillBuffer>(traced); + Install<&D::vkCmdBeginQuery, Op::QueryBegin>(traced); + Install<&D::vkCmdEndQuery, Op::QueryEnd>(traced); + Install<&D::vkCmdResetQueryPool, Op::QueryReset>(traced); + Install<&D::vkCmdCopyQueryPoolResults, Op::QueryCopy>(traced); + Install<&D::vkCmdBeginConditionalRenderingEXT, Op::ConditionalBegin>(traced); + Install<&D::vkCmdEndConditionalRenderingEXT, Op::ConditionalEnd>(traced); + Install<&D::vkCmdBeginTransformFeedbackEXT, Op::XfbBegin>(traced); + Install<&D::vkCmdEndTransformFeedbackEXT, Op::XfbEnd>(traced); + if (traced.vkCmdPipelineBarrier) { + traced.vkCmdPipelineBarrier = &PipelineBarrier; + } + if (traced.vkCmdPipelineBarrier2) { + traced.vkCmdPipelineBarrier2 = &PipelineBarrier2; + } + if (traced.vkCmdCopyBuffer) { + traced.vkCmdCopyBuffer = &CopyBuffer; + } + if (traced.vkCmdBeginRenderPass) { + traced.vkCmdBeginRenderPass = &BeginRenderPass; + } + if (traced.vkCmdEndRenderPass) { + traced.vkCmdEndRenderPass = &EndRenderPass; + } +} + +} // Anonymous namespace + +void GpuCost::Impl::ReadFrame() { + results.assign(count, {}); + const VkResult result = device.GetLogical().GetQueryResults( + *pool, 0, count, results.size() * sizeof(results[0]), results.data(), sizeof(results[0]), + VK_QUERY_RESULT_64_BIT | VK_QUERY_RESULT_WITH_AVAILABILITY_BIT); + if (result != VK_SUCCESS) { + ++bad; + return; + } + std::vector> upload_order(submission); + std::vector> main_order(submission); + for (u32 i = 0; i < count; ++i) { + const TimedOp& stamp = stamps[i]; + if (stamp.submission < submission) { + (stamp.upload ? upload_order : main_order)[stamp.submission].push_back(i); + } + } + for (u16 s = 0; s < submission; ++s) { + std::vector& order = upload_order[s]; + order.insert(order.end(), main_order[s].begin(), main_order[s].end()); + if (order.empty() || stamps[order.front()].op != Op::SubmitStart || + stamps[order.back()].op != Op::SubmitEnd || + std::ranges::any_of(order, [&](u32 i) { return results[i].available == 0; })) { + ++bad; + continue; + } + u64 previous = results[order.front()].value; + u64 total = 0; + for (size_t k = 1; k < order.size(); ++k) { + const u64 value = results[order[k]].value; + u64 ns = DeltaNs(previous, value); + if (ns > u64{1'000'000'000}) { + // Out of order: the interval is lost, the rest stays right. + ++bad; + ns = 0; + } + previous = value; + total += ns; + Attribute(order[k], ns); + } + busy_ns += total; + } + for (const Episode& episode : episodes) { + BreakTotals& totals = breaks[episode.site]; + ++totals.count; + totals.draws += episode.draws; + const auto one = [](bool condition) { return condition ? u64{1} : u64{0}; }; + totals.stores += one(episode.stores); + totals.same += one(episode.next == Next::Same); + totals.other += one(episode.next == Next::Other); + totals.none += one(episode.next == Next::None); + totals.crossed += one(episode.crossed); + totals.only_barriers += one(episode.between == BetweenBarrier); + totals.nothing += one(episode.between == 0); + for (size_t bit = 0; bit < NUM_BETWEEN; ++bit) { + totals.between[bit] += one(((episode.between >> bit) & 1u) != 0); + } + totals.end_ns += episode.end_ns; + totals.between_ns += episode.between_ns; + totals.begin_ns += episode.begin_ns; + } + ++frames; + submissions += submission; + stamped += count; + overflow += frame_overflow; +} + +void GpuCost::Impl::Report() { + if (frames == 0) { + return; + } + const double per_frame = 1.0 / static_cast(frames); + const auto ms = [per_frame](u64 ns) { return static_cast(ns) / 1e6 * per_frame; }; + const auto times = [per_frame](u64 n) { return static_cast(n) * per_frame; }; + + // Totals by category, each with its kinds by time. + struct Group { + Cell total{}; + std::array ops{}; + }; + std::array groups{}; + for (size_t role = 0; role < cells.size(); ++role) { + for (size_t op = 0; op < NUM_OPS; ++op) { + for (size_t upload = 0; upload < 2; ++upload) { + for (size_t present = 0; present < 2; ++present) { + const Cell& cell = cells[role][op][upload][present]; + if (cell.count == 0) { + continue; + } + Group& group = groups[static_cast( + CategoryOf(static_cast(role), static_cast(op), + upload != 0, present != 0))]; + group.total.ns += cell.ns; + group.total.count += cell.count; + group.total.bytes += cell.bytes; + group.ops[op].ns += cell.ns; + group.ops[op].count += cell.count; + } + } + } + } + std::string line; + for (size_t c = 0; c < NUM_CATEGORIES; ++c) { + const Group& group = groups[c]; + if (group.total.count == 0) { + continue; + } + line += fmt::format(" | {} {:.3f} ms {:.1f}x", CATEGORY_NAMES[c], ms(group.total.ns), + times(group.total.count)); + if (group.total.bytes != 0) { + line += fmt::format(" {:.0f} KiB", static_cast(group.total.bytes) / 1024.0 * + per_frame); + } + std::vector kinds; + for (size_t op = 0; op < NUM_OPS; ++op) { + if (group.ops[op].count != 0) { + kinds.push_back(op); + } + } + if (kinds.size() > 1) { + std::ranges::sort(kinds, std::greater<>{}, + [&group](size_t op) { return group.ops[op].ns; }); + line += " ("; + for (size_t k = 0; k < kinds.size(); ++k) { + line += fmt::format("{}{} {:.3f}/{:.1f}x", k == 0 ? "" : ", ", OP_NAMES[kinds[k]], + ms(group.ops[kinds[k]].ns), times(group.ops[kinds[k]].count)); + } + line += ")"; + } + } + const auto seconds = std::chrono::duration(std::chrono::steady_clock::now() - + window_start) + .count(); + LOG_INFO(Render_Vulkan, + "GPU cost in {:.1f} s, {} sampled frames, per frame: busy {:.3f} ms in {:.1f} " + "submissions, {:.0f} timestamps ({} over, {} bad){}", + seconds, frames, ms(busy_ns), times(submissions), times(stamped), overflow, bad, line); + + // By source line and kind. + std::unordered_map named; + for (const auto& [key, cell] : by_site) { + const SiteInfo& site = sites[static_cast(key >> 8)]; + const char* role = site.role == RecordSite::PassEnd ? "[pass end] " + : site.role == RecordSite::PassBegin ? "[next pass] " + : site.role == RecordSite::Submit ? "[submission] " + : ""; + Cell& total = named[fmt::format("{}{} at {}", role, OP_NAMES[key & 0xff], site.name)]; + total.ns += cell.ns; + total.count += cell.count; + total.bytes += cell.bytes; + } + std::vector> sorted(named.begin(), named.end()); + std::ranges::sort(sorted, std::greater<>{}, [](const auto& entry) { return entry.second.ns; }); + if (sorted.size() > 28) { + sorted.resize(28); + } + std::string site_line; + for (const auto& [name, cell] : sorted) { + site_line += fmt::format("{}{:.3f} ms {:.1f}x", site_line.empty() ? "" : " | ", + ms(cell.ns), times(cell.count)); + if (cell.bytes != 0) { + site_line += fmt::format(" {:.0f} KiB", + static_cast(cell.bytes) / 1024.0 * per_frame); + } + site_line += fmt::format(" {}", name); + } + LOG_INFO(Render_Vulkan, "GPU cost by site, per frame: {}", site_line); + + // Render pass breaks by the line that ended the pass. + std::unordered_map by_name; + for (const auto& [site, totals] : breaks) { + BreakTotals& merged = by_name[site == NO_SITE ? std::string{"unknown"} : sites[site].name]; + merged.count += totals.count; + merged.draws += totals.draws; + merged.stores += totals.stores; + merged.same += totals.same; + merged.other += totals.other; + merged.none += totals.none; + merged.crossed += totals.crossed; + merged.only_barriers += totals.only_barriers; + merged.nothing += totals.nothing; + for (size_t bit = 0; bit < NUM_BETWEEN; ++bit) { + merged.between[bit] += totals.between[bit]; + } + merged.end_ns += totals.end_ns; + merged.between_ns += totals.between_ns; + merged.begin_ns += totals.begin_ns; + } + std::vector> ends(by_name.begin(), by_name.end()); + std::ranges::sort(ends, std::greater<>{}, [](const auto& entry) { return entry.second.count; }); + if (ends.size() > 14) { + ends.resize(14); + } + for (const auto& [name, totals] : ends) { + const double n = static_cast(totals.count); + const auto percent = [n](u64 part) { return static_cast(part) * 100.0 / n; }; + std::string between; + for (size_t bit = 0; bit < NUM_BETWEEN; ++bit) { + between += + fmt::format(", {} {:.0f}%", BETWEEN_NAMES[bit], percent(totals.between[bit])); + } + LOG_INFO(Render_Vulkan, + "Render pass breaks at {}: {:.1f} a frame, {:.1f} draws in each pass, storage " + "writes in {:.0f}%; next begin: same framebuffer {:.0f}%, another {:.0f}%, none " + "in the frame {:.0f}%, after a submission {:.0f}%; between: only barriers " + "{:.0f}%, nothing {:.0f}%{}; GPU us per break: end {:.1f}, between {:.1f}, next " + "begin {:.1f}", + name, times(totals.count), static_cast(totals.draws) / n, + percent(totals.stores), percent(totals.same), percent(totals.other), + percent(totals.none), percent(totals.crossed), percent(totals.only_barriers), + percent(totals.nothing), between, static_cast(totals.end_ns) / n / 1e3, + static_cast(totals.between_ns) / n / 1e3, + static_cast(totals.begin_ns) / n / 1e3); + } + + cells = {}; + by_site.clear(); + breaks.clear(); + frames = 0; + busy_ns = 0; + submissions = 0; + stamped = 0; + overflow = 0; + bad = 0; + window_start = std::chrono::steady_clock::now(); +} + +bool GpuCost::Enabled() { + const char* const value = std::getenv("EDEN_NX2_GPU_COST"); + return value != nullptr && *value != '\0' && *value != '0'; +} + +std::unique_ptr GpuCost::Create(const Device& device, u32 timestamp_bits, + float timestamp_period, bool frames_sampled) { + if (!Enabled()) { + return nullptr; + } + if (!frames_sampled) { + LOG_WARNING(Render_Vulkan, "GPU cost (EDEN_NX2_GPU_COST) needs EDEN_DKB_FRAME_TRACE=1 and " + "device timestamps: off"); + return nullptr; + } + vk::QueryPool pool; + try { + pool = device.GetLogical().CreateQueryPool({ + .sType = VK_STRUCTURE_TYPE_QUERY_POOL_CREATE_INFO, + .pNext = nullptr, + .flags = 0, + .queryType = VK_QUERY_TYPE_TIMESTAMP, + .queryCount = CAPACITY, + .pipelineStatistics = 0, + }); + } catch (const vk::Exception& error) { + LOG_WARNING(Render_Vulkan, "GPU cost: no query pool: {}", error.what()); + return nullptr; + } + auto impl = std::make_unique(device, std::move(pool), timestamp_bits, + static_cast(timestamp_period)); + InstallHooks(impl->traced); + if (impl->host_reset) { + device.GetLogical().ResetQueryPool(*impl->pool, 0, CAPACITY); + } + g_impl = impl.get(); + LOG_INFO(Render_Vulkan, + "GPU cost (EDEN_NX2_GPU_COST): timing every Vulkan command that does work in the " + "frames EDEN_DKB_FRAME_TRACE samples, {} queries, {} reset", + CAPACITY, impl->host_reset ? "host" : "command buffer"); + return std::unique_ptr(new GpuCost(std::move(impl))); +} + +GpuCost::GpuCost(std::unique_ptr impl_) : impl{std::move(impl_)} {} + +GpuCost::~GpuCost() { + if (g_impl == impl.get()) { + g_impl = nullptr; + } +} + +std::string GpuCost::SiteName(const char* file, u32 line, const char* function) { + return fmt::format("{}:{} {}", BaseName(file), line, ShortFunction(function)); +} + +void GpuCost::BeginSubmission(vk::CommandBuffer& cmdbuf, vk::CommandBuffer& upload_cmdbuf, + bool first_of_frame) { + Impl& self = *impl; + if (first_of_frame) { + self.StartFrame(); + } + if (!self.frame_open || self.submission == 0xffff) { + return; + } + self.main_handle = *cmdbuf; + self.upload_handle = *upload_cmdbuf; + if (first_of_frame && !self.host_reset) { + self.real.vkCmdResetQueryPool(self.upload_handle, *self.pool, 0, CAPACITY); + } + self.open = true; + self.current_site = NO_SITE; + self.current_role = RecordSite::Own; + self.current_flags = 0; + self.Stamp(self.upload_handle, Op::SubmitStart, 0, NO_EPISODE); + self.Stamp(self.upload_handle, Op::PriorTail, 0, NO_EPISODE); + self.Stamp(self.main_handle, Op::MainStart, 0, NO_EPISODE); + cmdbuf = vk::CommandBuffer(self.main_handle, self.traced); + upload_cmdbuf = vk::CommandBuffer(self.upload_handle, self.traced); +} + +void GpuCost::BeginCommand(const RecordSite& site) { + Impl& self = *impl; + self.current_site = self.Intern(site); + self.current_role = site.role; + self.current_flags = site.flags; +} + +void GpuCost::EndSubmission(vk::CommandBuffer cmdbuf) { + Impl& self = *impl; + if (!self.open) { + return; + } + self.Stamp(*cmdbuf, Op::SubmitEnd, 0, NO_EPISODE); + if (self.pending != NO_EPISODE) { + self.episodes[self.pending].crossed = true; + } + self.open = false; + ++self.submission; + self.upload_handle = VK_NULL_HANDLE; + self.main_handle = VK_NULL_HANDLE; +} + +void GpuCost::FrameDone() { + Impl& self = *impl; + if (!self.frame_open) { + return; + } + self.frame_open = false; + self.open = false; + if (self.count != 0) { + self.ReadFrame(); + if (self.host_reset) { + self.device.GetLogical().ResetQueryPool(*self.pool, 0, CAPACITY); + } + } + if (std::chrono::steady_clock::now() - self.window_start >= std::chrono::seconds{10}) { + self.Report(); + } +} + +} // namespace Vulkan diff --git a/src/video_core/renderer_vulkan/vk_gpu_cost.h b/src/video_core/renderer_vulkan/vk_gpu_cost.h new file mode 100644 index 0000000000..3453e31095 --- /dev/null +++ b/src/video_core/renderer_vulkan/vk_gpu_cost.h @@ -0,0 +1,72 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later + +#pragma once + +#include +#include + +#include "common/common_types.h" +#include "video_core/vulkan_common/vulkan_wrapper.h" + +// TEST (EDEN_NX2_GPU_COST=1, in the frames EDEN_DKB_FRAME_TRACE=1 samples): where the host GPU's +// time goes, the part outside the timed draws and dispatches included. +// +// A sampled submission's command buffers record through a copy of the dispatch table whose +// vkCmd* entries count each call and, after each one that does work (not binds or dynamic +// state), write a bottom-of-pipe timestamp. Consecutive timestamps of a command buffer split the +// submission's GPU time, from a top-of-pipe one at its start to one at its end, into intervals +// that each end with one Vulkan command: the time from the end of the work before it to the end +// of that command's own work. Every interval goes to the command's kind (draw, render pass +// begin or end, barrier, copy, clear, query...) and to the source line that recorded it +// (Scheduler::Record's caller; a render pass's end to the line that ended it, its begin to the +// line that ended the pass before). +// +// Render pass breaks: after each end, what ran before the next begin (only barriers, copies, +// dispatches, queries, clears) and whether that begin resumed the same framebuffer. +// +// This only reads: the commands, their order and their synchronization are unchanged. Logged +// every 10 s over the frames sampled in it, per frame: "GPU cost" (totals by kind), "GPU cost +// by site", and one "Render pass breaks" line for each of the lines that ended the most passes. +namespace Vulkan { + +class Device; +struct RecordSite; + +class GpuCost { +public: + /// Whether EDEN_NX2_GPU_COST is set. + [[nodiscard]] static bool Enabled(); + + /// Null unless EDEN_NX2_GPU_COST is set, frames are sampled (EDEN_DKB_FRAME_TRACE's + /// timestamp pools exist) and the query pool can be created. + static std::unique_ptr Create(const Device& device, u32 timestamp_bits, + float timestamp_period, bool frames_sampled); + ~GpuCost(); + + GpuCost(const GpuCost&) = delete; + GpuCost& operator=(const GpuCost&) = delete; + + /// "file:line Class::Function" of a source location. + [[nodiscard]] static std::string SiteName(const char* file, u32 line, const char* function); + + /// Worker thread: a sampled submission starts in these command buffers. Writes its start + /// timestamps and switches both to the tracing dispatch table. + void BeginSubmission(vk::CommandBuffer& cmdbuf, vk::CommandBuffer& upload_cmdbuf, + bool first_of_frame); + /// Worker thread: the site of the command about to run. + void BeginCommand(const RecordSite& site); + /// Worker thread, in the submission's last command before its command buffers end. + void EndSubmission(vk::CommandBuffer cmdbuf); + /// Worker thread: every submission of the sampled frame is done on the GPU. + void FrameDone(); + + struct Impl; + +private: + explicit GpuCost(std::unique_ptr impl); + + std::unique_ptr impl; +}; + +} // namespace Vulkan diff --git a/src/video_core/renderer_vulkan/vk_scheduler.cpp b/src/video_core/renderer_vulkan/vk_scheduler.cpp index 4f9b4fd152..0a20d223e0 100644 --- a/src/video_core/renderer_vulkan/vk_scheduler.cpp +++ b/src/video_core/renderer_vulkan/vk_scheduler.cpp @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include @@ -31,6 +32,7 @@ #include "video_core/gpu_wait_stats.h" #include "video_core/nx2_profile.h" #include "video_core/renderer_vulkan/vk_command_pool.h" +#include "video_core/renderer_vulkan/vk_gpu_cost.h" #include "video_core/renderer_vulkan/vk_graphics_pipeline.h" #include "video_core/renderer_vulkan/vk_master_semaphore.h" #include "video_core/renderer_vulkan/vk_scheduler.h" @@ -48,18 +50,14 @@ std::atomic g_profile_finishes{}; std::atomic g_profile_finish_ns{}; std::atomic g_profile_window_start{}; std::mutex g_profile_end_mutex; -std::map g_profile_ends; +std::map, u64> g_profile_ends; -void CountRenderPassEnd(const char* file, u32 line) { +void CountRenderPassEnd(const std::source_location& location) { if (!VideoCommon::Nx2Profile::Enabled()) { return; } - std::string_view name{file}; - if (const auto slash = name.find_last_of('/'); slash != std::string_view::npos) { - name.remove_prefix(slash + 1); - } std::scoped_lock lock{g_profile_end_mutex}; - ++g_profile_ends[fmt::format("{}:{}", name, line)]; + ++g_profile_ends[{location.file_name(), location.line(), location.function_name()}]; } void ReportSubmitCounts() { @@ -81,14 +79,19 @@ void ReportSubmitCounts() { static_cast(g_profile_renderpasses.exchange(0)) / seconds, static_cast(g_profile_finishes.exchange(0)) / seconds, static_cast(g_profile_finish_ns.exchange(0)) / 1e6 / seconds); - std::vector> ends; + std::map named; { std::scoped_lock lock{g_profile_end_mutex}; for (const auto& [site, count] : g_profile_ends) { - ends.emplace_back(count, site); + const auto& [file, line, function] = site; + named[GpuCost::SiteName(file, line, function)] += count; } g_profile_ends.clear(); } + std::vector> ends; + for (const auto& [site, count] : named) { + ends.emplace_back(count, site); + } std::sort(ends.rbegin(), ends.rend()); for (size_t i = 0; i < ends.size() && i < 16; ++i) { LOG_INFO(Render_Vulkan, "Render pass ends {:.0f}/s at {}", @@ -98,14 +101,18 @@ void ReportSubmitCounts() { } void Scheduler::CommandChunk::ExecuteAll(vk::CommandBuffer cmdbuf, - vk::CommandBuffer upload_cmdbuf) { + vk::CommandBuffer upload_cmdbuf, GpuCost* cost) { auto command = first; - while (command != nullptr) { + for (size_t index = 0; command != nullptr; ++index) { auto next = command->GetNext(); + if (cost != nullptr && index < sites.size()) [[unlikely]] { + cost->BeginCommand(sites[index]); + } command->Execute(cmdbuf, upload_cmdbuf); command->~Command(); command = next; } + sites.clear(); submit = false; command_offset = 0; first = nullptr; @@ -153,6 +160,9 @@ Scheduler::Scheduler(const Device& device_, StateTracker& state_tracker_) if (VideoCommon::ReadTrace::Enabled()) { CreateReadTrace(); } + gpu_cost = GpuCost::Create(device, trace_timestamp_bits, trace_timestamp_period, + static_cast(trace_query_pool)); + record_sites = gpu_cost != nullptr; AcquireNewChunk(); AllocateWorkerCommandBuffer(); @@ -394,13 +404,13 @@ VideoCommon::ReadTrace::Timeline Scheduler::PlaceRead( u64 Scheduler::Flush(VkSemaphore signal_semaphore, VkSemaphore wait_semaphore, std::source_location location) { if (state.renderpass) { - CountRenderPassEnd(location.file_name(), location.line()); + CountRenderPassEnd(location); } if (VideoCommon::GpuSplit::check_owner) [[unlikely]] { VideoCommon::GpuSplit::CheckOwner("Scheduler::Flush"); } // When flushing, we only send data to the worker thread; no waiting is necessary. - const u64 signal_value = SubmitExecution(signal_semaphore, wait_semaphore); + const u64 signal_value = SubmitExecution(signal_semaphore, wait_semaphore, location); AllocateNewContext(); last_flush = std::chrono::steady_clock::now(); flush_pending = false; @@ -467,7 +477,7 @@ bool Scheduler::WaitSubmitted(u64 tick) { void Scheduler::Finish(VkSemaphore signal_semaphore, VkSemaphore wait_semaphore, std::source_location location) { if (state.renderpass) { - CountRenderPassEnd(location.file_name(), location.line()); + CountRenderPassEnd(location); } if (VideoCommon::GpuSplit::check_owner) [[unlikely]] { VideoCommon::GpuSplit::CheckOwner("Scheduler::Finish"); @@ -477,7 +487,7 @@ void Scheduler::Finish(VkSemaphore signal_semaphore, VkSemaphore wait_semaphore, const bool profile = VideoCommon::Nx2Profile::Enabled(); const u64 start = profile ? VideoCommon::Nx2Profile::Now() : 0; const u64 wait_start = VideoCommon::GpuWaitStats::Now(); - SubmitExecution(signal_semaphore, wait_semaphore); + SubmitExecution(signal_semaphore, wait_semaphore, location); Wait(presubmit_tick); AllocateNewContext(); VideoCommon::GpuWaitStats::AddHostWait(VideoCommon::GpuWaitStats::Now() - wait_start); @@ -553,8 +563,8 @@ void Scheduler::BeginRenderPassImpl(const Framebuffer* framebuffer, VkRenderPass values[i] = clear_values[i]; } const bool after_compute = std::exchange(compute_writes_pending, false); - Record([renderpass, framebuffer_handle, render_area, values, clear_value_count, - after_compute](vk::CommandBuffer cmdbuf) { + RecordAs([renderpass, framebuffer_handle, render_area, values, clear_value_count, + after_compute](vk::CommandBuffer cmdbuf, vk::CommandBuffer) { const VkRenderPassBeginInfo renderpass_bi{ .sType = VK_STRUCTURE_TYPE_RENDER_PASS_BEGIN_INFO, .pNext = nullptr, @@ -581,7 +591,7 @@ void Scheduler::BeginRenderPassImpl(const Framebuffer* framebuffer, VkRenderPass VK_PIPELINE_STAGE_ALL_GRAPHICS_BIT, 0, COMPUTE_TO_GRAPHICS); } cmdbuf.BeginRenderPass(renderpass_bi, VK_SUBPASS_CONTENTS_INLINE); - }); + }, last_pass_end, RecordSite::PassBegin, 0); num_renderpass_images = framebuffer->NumImages(); renderpass_images = framebuffer->Images(); renderpass_image_ranges = framebuffer->ImageRanges(); @@ -668,18 +678,18 @@ void Scheduler::RequestRenderpass(const Framebuffer* framebuffer, std::source_lo return; } if (state.renderpass) { - CountRenderPassEnd(location.file_name(), location.line()); + CountRenderPassEnd(location); } // Ends any active pass and realizes a deferred clear - EndRenderPass(); + EndRenderPass(location); BeginRenderPassImpl(framebuffer, renderpass, nullptr, 0); } void Scheduler::RequestOutsideRenderPassOperationContext(std::source_location location) { if (state.renderpass) { - CountRenderPassEnd(location.file_name(), location.line()); + CountRenderPassEnd(location); } - EndRenderPass(); + EndRenderPass(location); } bool Scheduler::UpdateGraphicsPipeline(GraphicsPipeline* pipeline) { @@ -771,7 +781,8 @@ void Scheduler::WorkerThread(std::stop_token stop_token) { // Perform the work, tracking whether the chunk was a submission // before executing. const bool has_submit = work->HasSubmit(); - work->ExecuteAll(current_cmdbuf, current_upload_cmdbuf); + work->ExecuteAll(current_cmdbuf, current_upload_cmdbuf, + trace_gpu_recording ? gpu_cost.get() : nullptr); // If the chunk was a submission, reallocate the command buffer. if (has_submit) { @@ -812,6 +823,9 @@ void Scheduler::AllocateWorkerCommandBuffer() { } if (trace_query_pool && !trace_gpu_failed) { TraceGpuSubmission(); + if (gpu_cost && trace_gpu_recording) { + gpu_cost->BeginSubmission(current_cmdbuf, current_upload_cmdbuf, trace_gpu_count == 0); + } } } @@ -1004,6 +1018,9 @@ void Scheduler::TraceGpuSubmission() { work.dispatch_attempts, shaders, work.untracked_uses); } ReportWorkTimes(); + if (gpu_cost) { + gpu_cost->FrameDone(); + } trace_gpu_tick = 0; trace_gpu_count = 0; trace_gpu_skipped = 0; @@ -1023,17 +1040,18 @@ void Scheduler::TraceGpuSubmission() { trace_gpu_recording = true; } -u64 Scheduler::SubmitExecution(VkSemaphore signal_semaphore, VkSemaphore wait_semaphore) { +u64 Scheduler::SubmitExecution(VkSemaphore signal_semaphore, VkSemaphore wait_semaphore, + std::source_location cause) { if (VideoCommon::Nx2Profile::Enabled()) { g_profile_submits.fetch_add(1, std::memory_order_relaxed); ReportSubmitCounts(); } - EndPendingOperations(); + EndPendingOperations(cause); InvalidateState(); const u64 signal_value = master_semaphore->NextTick(); - RecordWithUploadBuffer([signal_semaphore, wait_semaphore, signal_value, - this](vk::CommandBuffer cmdbuf, vk::CommandBuffer upload_cmdbuf) { + RecordAs([signal_semaphore, wait_semaphore, signal_value, + this](vk::CommandBuffer cmdbuf, vk::CommandBuffer upload_cmdbuf) { static constexpr VkMemoryBarrier WRITE_BARRIER{ .sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER, .pNext = nullptr, @@ -1043,6 +1061,9 @@ u64 Scheduler::SubmitExecution(VkSemaphore signal_semaphore, VkSemaphore wait_se upload_cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, 0, WRITE_BARRIER); upload_cmdbuf.End(); if (trace_gpu_recording) { + if (gpu_cost) { + gpu_cost->EndSubmission(cmdbuf); + } cmdbuf.WriteTimestamp(VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT, *trace_query_pool, trace_gpu_count * 2 + 1); trace_gpu_tick = signal_value; @@ -1104,7 +1125,7 @@ u64 Scheduler::SubmitExecution(VkSemaphore signal_semaphore, VkSemaphore wait_se } trace_frame_end = 0; } - }); + }, cause, RecordSite::Submit, 0); chunk->MarkSubmit(); DispatchWork(); return signal_value; @@ -1122,14 +1143,14 @@ void Scheduler::InvalidateState() { state_tracker.InvalidateCommandBufferState(); } -void Scheduler::EndPendingOperations() { +void Scheduler::EndPendingOperations(std::source_location cause) { // Pause, don't reset: a submission can land between a guest's counter reset and its report, // and resetting here would drop the samples counted so far (the report then reads too low). query_cache->CounterClose(VideoCommon::QueryType::ZPassPixelCount64); - EndRenderPass(); + EndRenderPass(cause); } -void Scheduler::EndRenderPass() +void Scheduler::EndRenderPass(std::source_location cause) { RealizeDeferredClear(); if (!state.renderpass) { @@ -1137,7 +1158,10 @@ void Scheduler::EndRenderPass() } ++recording_epoch; fragment_barrier.Reset(); + const u8 site_flags = + static_cast(render_pass_has_stores ? RecordSite::PassHadStores : 0); render_pass_has_stores = false; + last_pass_end = cause; query_cache->CounterClose(VideoCommon::QueryType::StreamingByteCount); @@ -1150,11 +1174,11 @@ void Scheduler::EndRenderPass() query_cache->CounterEnable(VideoCommon::QueryType::ZPassPixelCount64, false); query_cache->NotifySegment(false); - Record([num_images = num_renderpass_images, + RecordAs([num_images = num_renderpass_images, images = renderpass_images, ranges = renderpass_image_ranges, has_transform_feedback = device.IsExtTransformFeedbackSupported()]( - vk::CommandBuffer cmdbuf) { + vk::CommandBuffer cmdbuf, vk::CommandBuffer) { std::array barriers; for (size_t i = 0; i < num_images; ++i) { const VkImageSubresourceRange& range = ranges[i]; @@ -1205,7 +1229,7 @@ void Scheduler::EndRenderPass() VK_PIPELINE_STAGE_VERTEX_INPUT_BIT | VK_PIPELINE_STAGE_TRANSFER_BIT, 0, XFB_OUTPUT_BARRIER); } - }); + }, cause, RecordSite::PassEnd, site_flags); state.renderpass = VkRenderPass{}; num_renderpass_images = 0; diff --git a/src/video_core/renderer_vulkan/vk_scheduler.h b/src/video_core/renderer_vulkan/vk_scheduler.h index 44aaa5e5f7..4d808238a8 100644 --- a/src/video_core/renderer_vulkan/vk_scheduler.h +++ b/src/video_core/renderer_vulkan/vk_scheduler.h @@ -40,11 +40,31 @@ namespace Vulkan { class CommandPool; class Device; class Framebuffer; +class GpuCost; class GraphicsPipeline; class StateTracker; struct QueryCacheParams; +/// TEST (EDEN_NX2_GPU_COST, vk_gpu_cost.h): the source line that recorded a command. A render +/// pass's end carries the line that ended it, its begin the line that ended the pass before. +struct RecordSite { + enum Role : u8 { + Own, + PassEnd, + PassBegin, + Submit, + }; + enum Flags : u8 { + PassHadStores = 1 << 0, ///< PassEnd: a draw in the pass wrote storage buffers or images + }; + const char* file; + const char* function; + u32 line; + Role role; + u8 flags; +}; + /// The scheduler abstracts command buffer and fence management with an interface that's able to do /// OpenGL-like operations on Vulkan command buffers. class Scheduler { @@ -193,24 +213,19 @@ class Scheduler { /// Send work to a separate thread. template requires std::is_invocable_v - void RecordWithUploadBuffer(T&& command) { - if (VideoCommon::GpuSplit::check_owner) [[unlikely]] { - VideoCommon::GpuSplit::CheckOwner("Scheduler::Record"); - } - if (chunk->Record(command)) { - return; - } - DispatchWork(); - (void)chunk->Record(command); + void RecordWithUploadBuffer(T&& command, + std::source_location location = std::source_location::current()) { + RecordAs(std::forward(command), location, RecordSite::Own, 0); } template requires std::is_invocable_v - void Record(T&& c) { - this->RecordWithUploadBuffer( + void Record(T&& c, std::source_location location = std::source_location::current()) { + RecordAs( [command = std::move(c)](vk::CommandBuffer cmdbuf, vk::CommandBuffer) { command(cmdbuf); - }); + }, + location, RecordSite::Own, 0); } /// Whether the recording thread submitted `tick` within 20 ms. @@ -366,7 +381,13 @@ class Scheduler { class CommandChunk final { public: - void ExecuteAll(vk::CommandBuffer cmdbuf, vk::CommandBuffer upload_cmdbuf); + /// cost: the TEST GPU cost trace when this chunk is in a submission it samples. + void ExecuteAll(vk::CommandBuffer cmdbuf, vk::CommandBuffer upload_cmdbuf, GpuCost* cost); + + /// TEST (EDEN_NX2_GPU_COST): the site of each command, in order. + void NoteSite(const RecordSite& site) { + sites.push_back(site); + } template bool Record(T& command) { @@ -407,9 +428,25 @@ class Scheduler { size_t command_offset = 0; bool submit = false; + std::vector sites; alignas(std::max_align_t) std::array data{}; }; + template + void RecordAs(T&& command, std::source_location location, RecordSite::Role role, u8 flags) { + if (VideoCommon::GpuSplit::check_owner) [[unlikely]] { + VideoCommon::GpuSplit::CheckOwner("Scheduler::Record"); + } + if (!chunk->Record(command)) { + DispatchWork(); + (void)chunk->Record(command); + } + if (record_sites) [[unlikely]] { + chunk->NoteSite({location.file_name(), location.function_name(), location.line(), role, + flags}); + } + } + struct State { VkRenderPass renderpass{}; VkFramebuffer framebuffer{}; @@ -451,13 +488,16 @@ class Scheduler { [[nodiscard]] VideoCommon::ReadTrace::Timeline PlaceRead( const VideoCommon::ReadTrace::Read& read) const; - u64 SubmitExecution(VkSemaphore signal_semaphore, VkSemaphore wait_semaphore); + /// cause: the line that asked for the submission, which ends a render pass still open. + u64 SubmitExecution(VkSemaphore signal_semaphore, VkSemaphore wait_semaphore, + std::source_location cause); void AllocateNewContext(); - void EndPendingOperations(); + void EndPendingOperations(std::source_location cause); - void EndRenderPass(); + /// cause: the line that needs the render pass ended (TEST EDEN_NX2_GPU_COST's site of it). + void EndRenderPass(std::source_location cause = std::source_location::current()); void AcquireNewChunk(); @@ -514,6 +554,12 @@ class Scheduler { std::unique_ptr read_trace; VideoCommon::ReadTrace::Counts trace_counts{}; + /// TEST: EDEN_NX2_GPU_COST. record_sites: every command notes its RecordSite. + std::unique_ptr gpu_cost; + bool record_sites{}; + /// The line that ended the last render pass: the site of the next pass's begin. + std::source_location last_pass_end{}; + VideoCommon::QueryCacheBase* query_cache = nullptr; vk::CommandBuffer current_cmdbuf; From bd737eba16c5281431ab01c5c2382a5cf6cd38a5 Mon Sep 17 00:00:00 2001 From: jpkto <255363096+jpkto@users.noreply.github.com> Date: Sun, 8 Nov 2026 02:14:44 +0100 Subject: [PATCH 032/105] video_core: value QMD releases on the split's front, publish sooner after waits In test47f most suspended acquires wait for a compute dispatch: of about 20,000 resumes per 10 s, about 15,000 were latched by a QMD reduction or maximum. The front could not tell what a launch releases, so it suspended the channel until the back ran it. The front now runs the QMD scheduling of every channel as the back will (FrontQmdView, video_core/front_qmd.h): the same NativeQmdScheduler code, on a table of its own, fed the same signals in ring order (PCAS methods and inline QMD launches, sent directly or by macros). When a launch runs there, the front notes what its releases write: - plain and four-word releases: the payload; - maxima: the maximum with the word when the front knows the word (the last release sent there, or memory when nothing queued writes it), else the payload as a lower bound; - reductions: the result when the word is known, else unknown. The semaphore view takes a lower bound for greater-or-equal acquires of it or less (passed by the queued release), and waits for its position otherwise. This is exact only if the front reads the descriptors the back will read. The back reads them later, from guest memory without the GPU caches. They hold the same where no map still queued can change, and where nothing queued writes the QMD except through the caches, on pages GPU work never wrote. An inline QMD launch reads the words the back writes right before. Elsewhere the front's table may no longer match the back's: it stops valuing, counts the launches as writing anywhere, and takes the back's table over at the next QMD method after the back has run everything sent. To tell the writes apart, GpuSplit::Writes gets Caches: entries that write the caches anywhere and guest memory only through them, where GPU work wrote. WaitForWrites still waits for them. The new MemoryWritesTarget does not. Changes: - Caches: 3D draws (and the deferred draw after a macro), conditional rendering, syncpoint increments, back macros, fence work, list-end submits and CPU flush requests. - Noted, with their writes placed: puller releases, compute reports (16 bytes) and inline QMDs (256 bytes). - Noted, writing nothing: channel binds, the dirty checks of macro payloads, command signals, invalidations, methods an engine only keeps in its method sink, and compute register and macro code writes. - Unchanged (anywhere): uploads, copies, maps and the rest. Write slots go from 1,024 to 16,384, so a QMD rarely shares one with the constant buffer words of queued draws. QMD table entries no longer keep a copy of the descriptor (it was read again before each use), which keeps the table copy small. Churn. In 47e every acquire drained the split, and a drain resets the publish interval to 8 words, so the back restarted at once. In 47f a resume or a positioned wait kept it at 256. The back then idled until the front had parsed 256 more words: 3,377 idle periods a second at 90 us, against 9,520 at 30 us. Now the end of a positioned wait, a resumed channel and each command from the CPU publish after 8 words again, doubling to 256. A met suspended acquire also no longer polls twice per resume (the scheduler asked, then the channel's Step asked again). Tests: the QMD scheduler with its own table, the split's write classes and lower bounds, and the front valuing inline, deferred, chained and reduced launches against a back run on the same thread. A harness ran FrontQmdView against a model of the back lagging behind. It covered 200 random runs with patches, unplaced writes and releases into QMDs. In 589,000 valued signals and 233,000 checked releases, the note always matched what the back wrote. Co-Authored-By: Claude Opus 5.5 --- src/tests/video_core/maxwell_fast_method.cpp | 220 +++++++++++++++++++ src/tests/video_core/native_qmd.cpp | 42 ++++ src/video_core/CMakeLists.txt | 1 + src/video_core/control/scheduler.cpp | 15 +- src/video_core/dma_pusher.cpp | 46 ++-- src/video_core/engines/kepler_compute.cpp | 183 +++++++++++++-- src/video_core/engines/kepler_compute.h | 3 + src/video_core/engines/maxwell_3d.cpp | 20 +- src/video_core/engines/puller.cpp | 87 +++++--- src/video_core/engines/puller.h | 3 + src/video_core/front_qmd.h | 180 +++++++++++++++ src/video_core/gpu_split.cpp | 29 ++- src/video_core/gpu_split.h | 74 +++++-- src/video_core/gpu_thread.cpp | 23 +- src/video_core/memory_manager.h | 7 + src/video_core/native_qmd.h | 30 ++- 16 files changed, 845 insertions(+), 118 deletions(-) create mode 100644 src/video_core/front_qmd.h diff --git a/src/tests/video_core/maxwell_fast_method.cpp b/src/tests/video_core/maxwell_fast_method.cpp index 9aba1837c1..561f6a4f13 100644 --- a/src/tests/video_core/maxwell_fast_method.cpp +++ b/src/tests/video_core/maxwell_fast_method.cpp @@ -23,10 +23,14 @@ #include "core/memory.h" #include "tests/core/system_test_context.h" #include "video_core/dirty_flags.h" +#include "video_core/engines/kepler_compute.h" #include "video_core/engines/maxwell_3d.h" #include "video_core/gpu.h" #include "video_core/gpu_split.h" +#include "video_core/macro.h" #include "video_core/memory_manager.h" +#include "video_core/native_qmd.h" +#include "video_core/nx2.h" #include "video_core/renderer_null/null_rasterizer.h" namespace VideoCommon { @@ -78,6 +82,17 @@ struct GpuSplitTestAccess { return GpuSplit::WriteSlot(a >> GpuSplit::WriteGranuleBits) == GpuSplit::WriteSlot(b >> GpuSplit::WriteGranuleBits); } + + /// Runs what was sent and not run yet on the calling thread, as the back does. + static void RunBack(GpuSplit& split) { + GpuSplit::on_back_thread = true; + u64 position = split.consumed.load(); + while (position < split.write_position) { + position = Execute(split, position, split.write_position); + } + split.consumed.store(position); + GpuSplit::on_back_thread = false; + } }; } // namespace VideoCommon @@ -1423,3 +1438,208 @@ TEST_CASE("Split macro DMA reads wait only for the queued work that can write th CHECK(engine.regs.reg_array[state] == 0x88888888); CHECK(engine.regs.reg_array[copy] == 0x88888888); } + +namespace { + +// Two pages of guest memory at gpu_address (device_address) for the tests below. +struct MappedPages { + static constexpr VAddr cpu_address = 0x4000; + static constexpr DAddr device_address = 0x10000; + static constexpr GPUVAddr gpu_address = 0x100000; + + explicit MappedPages(Fixture& f) : fixture{f}, process{f.system.Kernel()} { + auto& table = process.GetPageTable().GetImpl(); + table.Resize(20, 12); + backing = f.physical.GetPointerFromRaw(0x10000); + for (u64 page = 0; page < 2; ++page) { + const VAddr address = cpu_address + page * 0x1000; + table.entries.GetAndFault(address >> 12) + .Store(false, Common::PageType::Memory, 1, + reinterpret_cast(backing + page * 0x1000) - address); + } + process.GetMemory().SetCurrentPageTable(process); + asid = f.device_memory.RegisterProcess(&process.GetMemory()); + f.device_memory.BindInterface(&f.rasterizer); + f.device_memory.Map(device_address, cpu_address, 0x2000, asid); + f.memory.BindRasterizer(&f.rasterizer); + f.memory.Map(gpu_address, device_address, 0x2000, Tegra::PTEKind::PITCH, false); + std::memset(backing, 0, 0x2000); + } + ~MappedPages() { + fixture.device_memory.Unmap(device_address, 0x2000); + fixture.device_memory.UnregisterProcess(asid); + fixture.system.Kernel().UnregisterKernelObject(&process); + } + MappedPages(const MappedPages&) = delete; + MappedPages& operator=(const MappedPages&) = delete; + + Fixture& fixture; + Kernel::KProcess process; + u8* backing{}; + Core::Asid asid{}; +}; + +} // Anonymous namespace + +TEST_CASE("Split tells guest memory writes from what only the GPU caches hold", + "[video_core][gpu_split_dependencies]") { + using Access = VideoCommon::GpuSplitTestAccess; + using Writes = VideoCommon::GpuSplit::Writes; + Fixture f; + f.system.Initialize(); + MappedPages pages{f}; + constexpr GPUVAddr word = MappedPages::gpu_address + 0x40; + constexpr GPUVAddr other = MappedPages::gpu_address + 0x140; + REQUIRE_FALSE(Access::ShareWriteSlot(MappedPages::device_address + 0x40, + MappedPages::device_address + 0x140)); + VideoCommon::GpuSplit split; + Access::Initialize(split, f.system); + split.Defer([] {}, Writes::Noted); + split.NoteWrite(f.memory, word, 4); + const u64 noted = Access::End(split); + // Shader stores: a read of the caches waits for them, a read of guest memory only for the + // writes noted there. + split.Defer([] {}, Writes::Caches); + CHECK(Access::WritesTarget(split, f.memory, other, 4) == Access::End(split)); + CHECK(split.MemoryWritesTarget(f.memory, word, 4) == noted); + CHECK(split.MemoryWritesTarget(f.memory, other, 4) == 0); + CHECK(split.LastUnplacedWrite() == 0); + // Work nobody noted the writes of writes guest memory anywhere. + split.Defer([] {}); + CHECK(split.LastUnplacedWrite() == Access::End(split)); + CHECK(split.MemoryWritesTarget(f.memory, other, 4) == Access::End(split)); + // A maximum's lower bound. + split.NoteSemaphoreWrite(MappedPages::device_address + 0x40, 5, true, true); + const auto queued = split.QueuedSemaphoreWriteAt(MappedPages::device_address + 0x40); + REQUIRE(queued.has_value()); + CHECK(queued->known); + CHECK(queued->at_least); + CHECK(queued->payload == 5); +} + +TEST_CASE("Split front values the releases of the QMD launches it sends as the back runs them", + "[video_core][gpu_split_dependencies]") { + using Access = VideoCommon::GpuSplitTestAccess; + using Descriptor = Tegra::NativeQmdScheduler::Descriptor; + using Writes = VideoCommon::GpuSplit::Writes; + constexpr u32 Enable = 1U << 23; + constexpr u32 OneWord = 1U << 30; + constexpr u32 Maximum = 1U << 19; + constexpr u32 Reduction = 1U << 26; // ADD + constexpr GPUVAddr qmds = MappedPages::gpu_address; + constexpr GPUVAddr words = MappedPages::gpu_address + 0x1000; + constexpr DAddr word_devices = MappedPages::device_address + 0x1000; + Fixture f; + f.system.Initialize(); + Tegra::Nx2ClearGpuWrittenPages(); + MappedPages pages{f}; + Tegra::Engines::KeplerCompute compute{f.memory}; + Tegra::MacroEngine macros{true}; + std::array positions{}; + compute.SetNativeMacroMode(¯os, positions); + compute.BindRasterizer(&f.rasterizer); + VideoCommon::GpuSplit split; + Access::Initialize(split, f.system); + const auto send = [&](u32 method, u32 argument) { + compute.FrontCallMethod(f.system, split, method, argument, true, nullptr); + }; + const auto make_qmd = [](u32 dependencies, u32 word_offset, u32 flags, u32 payload) { + Descriptor qmd{}; + qmd[18] = 0x30; + qmd[4] = dependencies << 14; + qmd[24] = static_cast(words) + word_offset; + qmd[25] = flags; + qmd[26] = payload; + return qmd; + }; + // An inline QMD, in memory already (the null rasterizer's inline writes write nothing): its + // words but the last, then, after the back ran them when `caught_up`, the last. + const auto inline_qmd = [&](GPUVAddr address, const Descriptor& qmd, bool caught_up) { + std::memcpy(pages.backing + (address - MappedPages::gpu_address), qmd.data(), sizeof(qmd)); + send(0xc6, static_cast((address >> 8) >> 32)); + send(0xc7, static_cast(address >> 8)); + for (u32 i = 0; i < 63; ++i) { + send(0xc8 + i, qmd[i]); + } + if (caught_up) { + Access::RunBack(split); + } + send(0x107, qmd[63]); + }; + const auto word = [&](u32 offset) { return f.memory.Read(words + offset); }; + const auto queued = [&](u32 offset) { + return split.QueuedSemaphoreWriteAt(word_devices + offset); + }; + + // A QMD that runs at once: its release is the front's once its table is the back's, which it + // takes over when the back has run everything sent. + inline_qmd(qmds, make_qmd(0, 0, Enable | OneWord, 5), true); + auto release = queued(0); + REQUIRE(release.has_value()); + CHECK(release->known); + CHECK_FALSE(release->at_least); + CHECK(release->payload == 5); + Access::RunBack(split); + + // One that waits for a dependency runs at the decrement, a maximum with the word in memory. + f.memory.Write(words + 0x10, 9); + inline_qmd(qmds + 0x100, make_qmd(1, 0x10, Enable | OneWord | Maximum, 7), false); + CHECK_FALSE(queued(0x10).has_value()); + send(0xad, static_cast((qmds + 0x100) >> 8)); + Access::RunBack(split); + send(0xb0, 7); + release = queued(0x10); + REQUIRE(release.has_value()); + CHECK(release->known); + CHECK_FALSE(release->at_least); + CHECK(release->payload == 9); + Access::RunBack(split); + CHECK(word(0x10) == 9); + + // Over a value the front can't tell, a maximum holds its payload at least. + split.Defer([] {}, Writes::Noted); + split.NoteSemaphoreWrite(word_devices + 0x10, 0, false); + inline_qmd(qmds + 0x200, make_qmd(0, 0x10, Enable | OneWord | Maximum, 12), false); + release = queued(0x10); + REQUIRE(release.has_value()); + CHECK(release->known); + CHECK(release->at_least); + CHECK(release->payload == 12); + Access::RunBack(split); + CHECK(word(0x10) == 12); + + // A reduction over the release before it, still queued. + inline_qmd(qmds + 0x300, make_qmd(0, 0x20, Enable | OneWord | Maximum, 4), false); + inline_qmd(qmds + 0x400, make_qmd(0, 0x20, Enable | OneWord | Reduction, 3), false); + release = queued(0x20); + REQUIRE(release.has_value()); + CHECK(release->known); + CHECK_FALSE(release->at_least); + CHECK(release->payload == 7); + Access::RunBack(split); + CHECK(word(0x20) == 7); + + // Work that may write guest memory anywhere, still queued: the front can't tell what the + // back reads, its table is no longer the back's. + split.Defer([] {}); + inline_qmd(qmds + 0x500, make_qmd(0, 0x30, Enable | OneWord, 1), false); + CHECK_FALSE(queued(0x30).has_value()); + Access::RunBack(split); + // Taken over again once the back ran everything. + inline_qmd(qmds + 0x600, make_qmd(0, 0x30, Enable | OneWord, 2), true); + release = queued(0x30); + REQUIRE(release.has_value()); + CHECK(release->payload == 2); + Access::RunBack(split); + + // A write to a waiting QMD still queued when its decrement is sent: not valued. + inline_qmd(qmds + 0x700, make_qmd(1, 0x40, Enable | OneWord | Maximum, 3), false); + Access::RunBack(split); + split.Defer([] {}, Writes::Noted); + split.NoteWrite(f.memory, qmds + 0x700 + 26 * sizeof(u32), sizeof(u32)); + send(0xad, static_cast((qmds + 0x700) >> 8)); + send(0xb0, 7); + CHECK_FALSE(queued(0x40).has_value()); + Access::RunBack(split); + CHECK(word(0x40) == 3); +} diff --git a/src/tests/video_core/native_qmd.cpp b/src/tests/video_core/native_qmd.cpp index 560ae4b13f..2d00eb8267 100644 --- a/src/tests/video_core/native_qmd.cpp +++ b/src/tests/video_core/native_qmd.cpp @@ -149,3 +149,45 @@ TEST_CASE("Semaphore reductions follow the documented ops", "[video_core][native REQUIRE(ReduceSemaphore(6, false, 0b1100, 0b1010) == 0b1110); REQUIRE(ReduceSemaphore(7, false, 0b1100, 0b1010) == 0b0110); } + +TEST_CASE("A QMD scheduler with a table of its own schedules apart from the shared one", + "[video_core][native_qmd]") { + constexpr Address address = 0x40000; + auto qmd = MakeQmd(); + qmd[4] = 1U << 14; // One dependency + SetRelease(qmd, 24, 0x5000, Enable | OneWord, 1); + const std::map memory{{address, qmd}}; + const auto signal = [&](NativeQmdScheduler& scheduler, Word action) { + std::vector released; + const auto error = scheduler.Signal( + address, action, + [&](Address qmd_address, Descriptor& descriptor) { + const auto it = memory.find(qmd_address); + if (it == memory.end()) { + return false; + } + descriptor = it->second; + return true; + }, + [&](Address destination, Word payload, Word flags) { + released.push_back({destination, payload, flags}); + return true; + }); + REQUIRE(error.empty()); + return released; + }; + const std::vector run{{0x5000, 1, Enable | OneWord}}; + // Loaded on the shared table, it waits for its dependency. + NativeQmdScheduler shared; + REQUIRE(signal(shared, 3).empty()); + // A copy of the table runs it at the decrement as the shared one does, and apart from it. + NativeQmdScheduler own{NativeQmdScheduler::OwnTable{}}; + own.CopyStates(shared); + REQUIRE(signal(own, 7) == run); + REQUIRE(signal(own, 7).empty()); // Completed: a decrement is kept for its next load + REQUIRE(signal(shared, 7) == run); + // A table that never saw the load keeps the decrement for it. + NativeQmdScheduler other{NativeQmdScheduler::OwnTable{}}; + REQUIRE(signal(other, 7).empty()); + REQUIRE(signal(other, 3) == run); +} diff --git a/src/video_core/CMakeLists.txt b/src/video_core/CMakeLists.txt index a4e9c6b30f..79394084ea 100644 --- a/src/video_core/CMakeLists.txt +++ b/src/video_core/CMakeLists.txt @@ -99,6 +99,7 @@ add_library(video_core STATIC macro_memo.h macro_turing.h fence_manager.h + front_qmd.h gpu.cpp gpu.h gpu_timestamp.h diff --git a/src/video_core/control/scheduler.cpp b/src/video_core/control/scheduler.cpp index 03031422f2..255f36c68e 100644 --- a/src/video_core/control/scheduler.cpp +++ b/src/video_core/control/scheduler.cpp @@ -22,10 +22,12 @@ void Scheduler::BindChannel(GPU& gpu, s32 bind_id) { // With the GPU thread split the rasterizer's channel switches with the methods around it, on // the back thread (deferred outside the lock: the back takes it to bind). if (auto& split = VideoCommon::GpuSplit::Get(); split.Active()) { - split.Defer([this, gpu_ptr = &gpu, bind_id] { - std::scoped_lock lk{scheduling_guard}; - gpu_ptr->BindChannel(bind_id); - }); + split.Defer( + [this, gpu_ptr = &gpu, bind_id] { + std::scoped_lock lk{scheduling_guard}; + gpu_ptr->BindChannel(bind_id); + }, + VideoCommon::GpuSplit::Writes::Noted); return; } std::scoped_lock lk{scheduling_guard}; @@ -81,6 +83,11 @@ bool Scheduler::ResumePending(GPU& gpu) { if (!can_resume) { continue; } + // The back ran the release: it may be out of work by the time the channel's next + // commands would otherwise go to it. + if (auto& split = VideoCommon::GpuSplit::Get(); split.Active()) { + split.PublishSoon(); + } BindChannel(gpu, state->bind_id); dma_pusher.DispatchCalls(); progress = true; diff --git a/src/video_core/dma_pusher.cpp b/src/video_core/dma_pusher.cpp index 377c9d4328..10aad31fef 100644 --- a/src/video_core/dma_pusher.cpp +++ b/src/video_core/dma_pusher.cpp @@ -96,10 +96,13 @@ void DmaPusher::DispatchCalls() { } } if (auto& split = VideoCommon::GpuSplit::Get(); split.Active()) { - split.Defer([gpu = &system.GPU()] { - gpu->FlushCommands(); - gpu->OnCommandListEnd(); - }); + // Submits, and releases the fences done (downloads where the GPU wrote). + split.Defer( + [gpu = &system.GPU()] { + gpu->FlushCommands(); + gpu->OnCommandListEnd(); + }, + VideoCommon::GpuSplit::Writes::Caches); split.Publish(); return; } @@ -186,10 +189,12 @@ bool DmaPusher::Step() { auto* const engine_state = subchannels[dma_state.subchannel]; if (auto& split = VideoCommon::GpuSplit::Get(); split.Active()) { // The caches behind IsMemoryDirty belong to the split's back thread. - split.Defer([engine_state, memory = &memory_manager, address = GPUVAddr{dma_state.dma_get}, - bytes = words * sizeof(u32)] { - engine_state->current_dirty = memory->IsMemoryDirty(address, bytes); - }); + split.Defer( + [engine_state, memory = &memory_manager, + address = GPUVAddr{dma_state.dma_get}, bytes = words * sizeof(u32)] { + engine_state->current_dirty = memory->IsMemoryDirty(address, bytes); + }, + VideoCommon::GpuSplit::Writes::Noted); } else { engine_state->current_dirty = memory_manager.IsMemoryDirty(dma_state.dma_get, words * sizeof(u32)); } @@ -526,8 +531,12 @@ void DmaPusher::CallMethod(u32 argument) { } } } + // A method the engine only keeps for later (its method sink) writes nothing yet. + using Writes = VideoCommon::GpuSplit::Writes; split.Method(subchannel, dma_state.method, argument, dma_state.is_last_call, - WantsSegment() ? &segment : nullptr); + WantsSegment() ? &segment : nullptr, + subchannel->execution_mask[dma_state.method] ? Writes::Anywhere + : Writes::Noted); if (engine_type == Engines::EngineTypes::MaxwellDMA) { NoteDmaSemaphore(dma_state.method, argument); } @@ -685,14 +694,17 @@ void DmaPusher::CheckUnwaitedWriters(GPUVAddr address, size_t size) { // The list did not wait for compute or copy work sent after the last sync. At the list's // place, the back has run that work: did it make the list's pages GPU-written? VideoCommon::Nx2DrainCheck::NoteListUnwaited(); - split.Defer([memory = &memory_manager, address, size] { - bool written = false; - for (GPUVAddr page = address & ~GPUVAddr{0xfff}; page < address + size; page += 0x1000) { - const auto device = memory->GpuToCpuAddress(page); - written = written || (device && Tegra::Nx2IsGpuWritten(*device)); - } - VideoCommon::Nx2DrainCheck::NoteListChecked(written); - }); + split.Defer( + [memory = &memory_manager, address, size] { + bool written = false; + for (GPUVAddr page = address & ~GPUVAddr{0xfff}; page < address + size; + page += 0x1000) { + const auto device = memory->GpuToCpuAddress(page); + written = written || (device && Tegra::Nx2IsGpuWritten(*device)); + } + VideoCommon::Nx2DrainCheck::NoteListChecked(written); + }, + VideoCommon::GpuSplit::Writes::Noted); } bool DmaPusher::ReadGpuWrittenCommands(GPUVAddr address, u32 count) { diff --git a/src/video_core/engines/kepler_compute.cpp b/src/video_core/engines/kepler_compute.cpp index 020045db29..bea30071f6 100644 --- a/src/video_core/engines/kepler_compute.cpp +++ b/src/video_core/engines/kepler_compute.cpp @@ -23,6 +23,7 @@ #include "core/core.h" #include "video_core/gpu_split.h" #include "video_core/fire_emblem_upscale.h" +#include "video_core/front_qmd.h" #include "video_core/nx2_profile.h" #include "video_core/nx2.h" #include "video_core/nx2_drain_check.h" @@ -1418,9 +1419,93 @@ Texture::TSCEntry KeplerCompute::GetTSCEntry(u32 tsc_index) const { return tsc_entry; } +namespace { + +// The QMD scheduling of every channel, run on the front as the back will run it. +VideoCommon::FrontQmdView& GetFrontQmd() { + static VideoCommon::FrontQmdView view; + return view; +} + +// An engine's GPU address space as its back reads QMDs and their release words (FrontQmdView). +struct FrontQmdMemory { + const MemoryManager& memory; + + [[nodiscard]] bool IsMapped(GPUVAddr address, u64 size) const { + return memory.IsFullyMappedRange(address, size); + } + void Read(GPUVAddr address, void* data, u64 size) const { + memory.ReadBlockUnsafe(address, data, size); + } + [[nodiscard]] u32 ReadWord(GPUVAddr address) const { + return memory.Read(address); + } + [[nodiscard]] bool InSyncpointShim(GPUVAddr address, u64 size) const { + return memory.OverlapsSyncpointShim(address, size); + } + [[nodiscard]] std::optional Device(GPUVAddr address) const { + return memory.GpuToCpuAddress(address); + } + [[nodiscard]] bool GpuWritten(u64 device, u64 size) const { + return Tegra::Nx2MayBeGpuWritten(device, size); + } + [[nodiscard]] u64 WritesTarget(const VideoCommon::GpuSplit& split, GPUVAddr address, + u64 size) const { + return split.MemoryWritesTarget(memory, address, size); + } + void NoteWrite(VideoCommon::GpuSplit& split, GPUVAddr address, u64 size) const { + split.NoteWrite(memory, address, size); + } +}; + +// The QMD methods: PCAS (0xae-0xb0) and the inline QMD's last word (0x107). +[[nodiscard]] bool IsQmdMethod(u32 method) { + return method == 0xae || method == 0xaf || method == 0xb0 || method == 0x107; +} + +// Whether a PCAS method signals the QMD scheduler, as ProcessNativeMethod filters them. +[[nodiscard]] bool SignalsQmd(u32 method, u32 argument) { + if (method == 0xaf) { + return (argument & 2) != 0; + } + return method == 0xb0 && (argument & 15) != 0 && (argument & 15) != 1; +} + +// What the back writes to guest memory running `method` (GpuSplit::Writes), as +// ProcessNativeMethod runs it: nothing for register writes and the macro code and bindings (the +// front's), what FrontNoteWrites notes for reports, inline QMDs and QMD launches, anywhere for +// inline uploads (and with EDEN_NATIVE_DRAW_DIAGNOSTICS, the methods it traces). +[[nodiscard]] VideoCommon::GpuSplit::Writes FrontWrites(const std::bitset<0xe00>& plain_methods, + u32 method) { + using Writes = VideoCommon::GpuSplit::Writes; + if (method >= plain_methods.size()) { + return Writes::Anywhere; + } + if (plain_methods[method]) { + return Writes::Noted; + } + switch (method) { + case 0x45: + case 0x46: + case 0x47: + case 0x48: + case 0xae: + case 0xaf: + case 0xb0: + case 0x107: + case 0x6c3: + return Writes::Noted; + default: + return Writes::Anywhere; + } +} + +} // Anonymous namespace + // Split GPU thread with macros on its front half (gpu_split.h, EDEN_NX2_SPLIT=2): the front // runs compute macros against its own copy of the native registers (which hold the last value -// written to each, on the back too) and loads and binds the compute macro code itself. +// written to each, on the back too) and loads and binds the compute macro code itself. It also +// runs the QMD scheduling ahead of the back (FrontQmdView), to tell what the launches release. struct KeplerCompute::FrontState { std::array registers{}; std::optional executing_macro; @@ -1444,6 +1529,8 @@ KeplerCompute::FrontState& KeplerCompute::Front(VideoCommon::GpuSplit& split) { front->bind_pointer = native_bind_pointer; front->macro = native_macro; front_macros = true; + // A new session's or a restarted split's: the QMDs the back scheduled meanwhile. + GetFrontQmd().Lose(); } return *front; } @@ -1457,6 +1544,8 @@ void KeplerCompute::ReleaseFront() { native_macro = front->macro; front.reset(); front_macros = false; + // The engines schedule their QMDs on their own again. + GetFrontQmd().Lose(); LOG_INFO(HW_GPU, "GPU thread split stopped: compute macros back on the engine"); } @@ -1528,9 +1617,40 @@ void KeplerCompute::FrontCallMethod(Core::System& system, VideoCommon::GpuSplit& return; } FrontApply(fs, method, argument); - split.Method(this, method, argument, is_last_call, segment); - if (method == 0x6c3) { - FrontNoteReport(split, fs, argument, false); + if (IsQmdMethod(method)) { + GetFrontQmd().Sync(split, native_qmd); + } + split.Method(this, method, argument, is_last_call, segment, + FrontWrites(native_plain_methods, method)); + FrontNoteWrites(split, fs, method, argument, false); +} + +// What the back writes to guest memory running `method` with `argument`, just sent as +// FrontWrites tells: where reports and inline QMDs write, and what QMD launches release. +void KeplerCompute::FrontNoteWrites(VideoCommon::GpuSplit& split, const FrontState& fs, u32 method, + u32 argument, bool from_macro) { + switch (method) { + case 0x6c3: { + // ProcessNativeReportSemaphore: a payload with its timestamp at most. + const GPUVAddr address = + (GPUVAddr{fs.registers[0x6c0] & 0x1ffffff} << 32) | fs.registers[0x6c1]; + split.NoteWrite(memory_manager, address, 16); + FrontNoteReport(split, fs, argument, from_macro); + break; + } + case 0x107: + FrontInlineQmd(split, fs, argument); + break; + case 0xae: + case 0xaf: + case 0xb0: + if (SignalsQmd(method, argument)) { + GetFrontQmd().Signal(split, FrontQmdMemory{memory_manager}, + GPUVAddr{fs.registers[0xad]} << 8, argument, nullptr); + } + break; + default: + break; } } @@ -1560,7 +1680,8 @@ void KeplerCompute::FrontNoteReport(VideoCommon::GpuSplit& split, const FrontSta std::optional base; if (!from_macro) { if (const auto queued = split.QueuedSemaphoreWriteAt(*device)) { - if (queued->known && queued->position >= split.LastWriterPosition()) { + if (queued->known && !queued->at_least && + queued->position >= split.LastWriterPosition()) { base = queued->payload; } } else if (!split.WritersPending() && !Nx2MayBeGpuWritten(*device, sizeof(u32))) { @@ -1578,6 +1699,28 @@ void KeplerCompute::FrontNoteReport(VideoCommon::GpuSplit& split, const FrontSta } } +// LOAD_INLINE_QMD_DATA's last word (ProcessNativeMethod): the back writes the inline QMD to its +// address and launches it, unless the QMD waits for a PCAS to schedule it. +void KeplerCompute::FrontInlineQmd(VideoCommon::GpuSplit& split, const FrontState& fs, + u32 last_word) { + const GPUVAddr address = ((GPUVAddr{fs.registers[0xc6]} << 32) | fs.registers[0xc7]) << 8; + if (!memory_manager.IsFullyMappedRange(address, 256)) { + // The back writes nothing either, unless a map still queued changes that. + if (!split.Ran(split.LastUnplacedWrite())) { + GetFrontQmd().Lose(); + split.NoteAnyWrite(); + } + return; + } + split.NoteWrite(memory_manager, address, 256); + if ((fs.registers[0xa7] & (1U << 24)) != 0 || (fs.registers[0xcc] & (1U << 12)) == 0) { + NativeQmdScheduler::Descriptor qmd; + std::memcpy(qmd.data(), fs.registers.data() + 0xc8, sizeof(qmd)); + qmd[63] = last_word; + GetFrontQmd().Signal(split, FrontQmdMemory{memory_manager}, address, 3, &qmd); + } +} + void KeplerCompute::FrontCallMultiMethod(Core::System& system, VideoCommon::GpuSplit& split, u32 method, const u32* base_start, u32 amount, u32 methods_pending, const GPUVAddr* segment) { @@ -1588,16 +1731,21 @@ void KeplerCompute::FrontCallMultiMethod(Core::System& system, VideoCommon::GpuS } return; } - if (method < fs.registers.size()) { - for (u32 i = 0; i < amount; ++i) { - FrontApply(fs, method, base_start[i]); - } + if (method >= fs.registers.size()) { + split.MultiMethod(this, method, base_start, amount, methods_pending, segment); + return; } - split.MultiMethod(this, method, base_start, amount, methods_pending, segment); - if (method == 0x6c3) { - for (u32 i = 0; i < amount; ++i) { - FrontNoteReport(split, fs, base_start[i], false); - } + for (u32 i = 0; i < amount; ++i) { + FrontApply(fs, method, base_start[i]); + } + if (IsQmdMethod(method)) { + GetFrontQmd().Sync(split, native_qmd); + } + split.MultiMethod(this, method, base_start, amount, methods_pending, segment, + FrontWrites(native_plain_methods, method)); + // The back runs them one by one; what they write is the back's once it ran them all. + for (u32 i = 0; i < amount; ++i) { + FrontNoteWrites(split, fs, method, base_start[i], false); } } @@ -1619,10 +1767,11 @@ void KeplerCompute::FrontRunMacro(Core::System& system, VideoCommon::GpuSplit& s return false; } FrontApply(fs, address, value); - split.MacroWrite(this, address, value); - if (address == 0x6c3) { - FrontNoteReport(split, fs, value, true); + if (IsQmdMethod(address)) { + GetFrontQmd().Sync(split, native_qmd); } + split.MacroWrite(this, address, value, FrontWrites(native_plain_methods, address)); + FrontNoteWrites(split, fs, address, value, true); return true; }; #ifdef ARCHITECTURE_x86_64 diff --git a/src/video_core/engines/kepler_compute.h b/src/video_core/engines/kepler_compute.h index 6ceeec90f6..a9588c18b7 100644 --- a/src/video_core/engines/kepler_compute.h +++ b/src/video_core/engines/kepler_compute.h @@ -274,8 +274,11 @@ class KeplerCompute final : public EngineInterface { FrontState& Front(VideoCommon::GpuSplit& split); void ReleaseFront(); void FrontApply(FrontState& state, u32 method, u32 argument); + void FrontNoteWrites(VideoCommon::GpuSplit& split, const FrontState& state, u32 method, + u32 argument, bool from_macro); void FrontNoteReport(VideoCommon::GpuSplit& split, const FrontState& state, u32 control, bool from_macro); + void FrontInlineQmd(VideoCommon::GpuSplit& split, const FrontState& state, u32 last_word); MacroEngine& FrontMacroEngine(FrontState& state); std::span FrontMacroPositions(FrontState& state); void FrontRunMacro(Core::System& system, VideoCommon::GpuSplit& split, FrontState& state); diff --git a/src/video_core/engines/maxwell_3d.cpp b/src/video_core/engines/maxwell_3d.cpp index 44b84b9a2d..c0b876ebe8 100644 --- a/src/video_core/engines/maxwell_3d.cpp +++ b/src/video_core/engines/maxwell_3d.cpp @@ -1603,7 +1603,9 @@ void Maxwell3D::FrontRunMacro(Core::System& system, VideoCommon::GpuSplit& split fs.parameters.clear(); if (fs.draw_pending) { fs.draw_pending = false; - split.Defer([this, system_ptr = &system] { FinishFrontMacro(*system_ptr); }); + // A deferred draw. + split.Defer([this, system_ptr = &system] { FinishFrontMacro(*system_ptr); }, + VideoCommon::GpuSplit::Writes::Caches); } if (FrontVerifyEnabled() && (++fs.calls & 4095) == 0) [[unlikely]] { FrontVerify(split, fs); @@ -1728,7 +1730,8 @@ bool Maxwell3D::FrontRunOnBack(Core::System& system, VideoCommon::GpuSplit& spli escapes->fetch_add(1, std::memory_order_release); } }, - fs.parameters); + // Its writes (BackWritable) touch registers and the caches, not guest memory. + fs.parameters, VideoCommon::GpuSplit::Writes::Caches); const u64 position = split.Position(); fs.back_last = position; const bool tracked = fs.control == Regs::ShadowRamControl::Track || @@ -1933,9 +1936,9 @@ void Maxwell3D::FrontNoteWrites(VideoCommon::GpuSplit& split, const FrontState& }; if (fs.draw_pending) { // A draw, or one an instance loop deferred to this entry (ProcessDirtyRegisters): where - // its shaders store, and what the caches download for it, only the back knows. - split.NoteAnyWrite(); - return; + // its shaders store, and what the caches download for it, only the back knows. Guest + // memory it writes through the caches only. + split.NoteCacheWrite(); } if (method >= cb_data && method < cb_data + 16) { // FrontWrite and FrontCallMultiMethod advanced the offset past these words already. @@ -1978,13 +1981,14 @@ void Maxwell3D::FrontNoteWrites(VideoCommon::GpuSplit& split, const FrontState& (mode == Regs::RenderEnable::Mode::Conditional || mode == Regs::RenderEnable::Mode::IfEqual || mode == Regs::RenderEnable::Mode::IfNotEqual)) { - split.NoteAnyWrite(); + split.NoteCacheWrite(); } break; } case MAXWELL3D_REG_INDEX(sync_info): - // A syncpoint increment: macros read syncpoints through memory too (MemoryManager::Read). - split.NoteAnyWrite(); + // A syncpoint increment: macros read syncpoints through memory too (MemoryManager::Read + // of the syncpoint shim), guest memory holds nothing new. + split.NoteCacheWrite(); break; default: break; diff --git a/src/video_core/engines/puller.cpp b/src/video_core/engines/puller.cpp index c0097eb6d5..bd792675c2 100644 --- a/src/video_core/engines/puller.cpp +++ b/src/video_core/engines/puller.cpp @@ -50,15 +50,21 @@ u32 ReadSemaphoreWord(DmaPusher& dma_pusher, GPUVAddr address) { // With the GPU thread split (gpu_split.h) the rasterizer belongs to the back thread: work that // only feeds it runs there in order; reads of what it wrote first wait for it (DrainBack). +// `writes`: what it writes of guest memory, for the front's reads (GpuSplit::Writes). template -void OnBack(const Func& func) { +void OnBack(const Func& func, + VideoCommon::GpuSplit::Writes writes = VideoCommon::GpuSplit::Writes::Anywhere) { if (auto& split = VideoCommon::GpuSplit::Get(); split.Active()) { - split.Defer(func); + split.Defer(func, writes); } else { func(); } } +// Fence work (ReleaseFences, fences and syncpoints signaled): downloads land in guest memory +// where the GPU wrote, nothing else. +constexpr auto FenceWrites = VideoCommon::GpuSplit::Writes::Caches; + void DrainBack() { VideoCommon::GpuSplit::Get().Drain(VideoCommon::DrainSite::Acquire); } @@ -80,12 +86,14 @@ void ReleaseFences(DmaPusher& dma_pusher) { VideoCommon::DrainSite::FenceRelease); } -// A release was just sent to the back: tell the front what it writes (CheckWithoutDrain). +// A release was just sent to the back (as Writes::Noted): tell the front what it writes +// (CheckWithoutDrain). void NoteRelease(DmaPusher& dma_pusher, GPUVAddr address, u32 payload, bool four_words) { auto& split = VideoCommon::GpuSplit::Get(); if (!split.Active()) { return; } + split.NoteWrite(dma_pusher.memory_manager, address, four_words ? 16 : sizeof(u32)); const auto device = dma_pusher.memory_manager.GpuToCpuAddress(address); if (!device) { return; @@ -151,6 +159,13 @@ ViewAnswer AskView(DmaPusher& dma_pusher, const Puller::PendingAcquire& acquire, if (!queued->known) { return {Kind::Unknown, queued->position}; } + if (queued->at_least) { + // A maximum (QMD releases): the word holds the payload or more once the back ran it, + // which meets a greater-or-equal acquire of the payload or less. Else memory tells. + return acquire.WaitsToReach() && acquire.Matches(queued->payload) + ? ViewAnswer{Kind::MetByQueued} + : ViewAnswer{Kind::Unknown, queued->position}; + } if (!acquire.Matches(queued->payload)) { return {Kind::NotMet}; } @@ -169,9 +184,11 @@ ViewAnswer AskView(DmaPusher& dma_pusher, const Puller::PendingAcquire& acquire, // it once the back ran the flush (PendingAcquire::flush_position). void RequestFlush(DmaPusher& dma_pusher, Puller::PendingAcquire& acquire, u64 device) { auto& split = VideoCommon::GpuSplit::Get(); - split.Defer([rasterizer = dma_pusher.rasterizer, device] { - rasterizer->FlushRegion(device, sizeof(u32), VideoCommon::CacheType::BufferCache); - }); + split.Defer( + [rasterizer = dma_pusher.rasterizer, device] { + rasterizer->FlushRegion(device, sizeof(u32), VideoCommon::CacheType::BufferCache); + }, + VideoCommon::GpuSplit::Writes::Caches); acquire.flush_position = split.Position(); acquire.flushed = true; } @@ -254,7 +271,7 @@ void MaybeReleaseFences(DmaPusher& dma_pusher, Puller::PendingAcquire& acquire) return; } acquire.fences_released = true; - OnBack([rasterizer = dma_pusher.rasterizer] { rasterizer->ReleaseFences(); }); + OnBack([rasterizer = dma_pusher.rasterizer] { rasterizer->ReleaseFences(); }, FenceWrites); } // Logs once an acquire that waits for more than 2 s. `current`: the word as read after a drain, @@ -300,12 +317,13 @@ bool Puller::PendingAcquire::WaitsToReach() const { } bool Puller::CanResumeAcquire(DmaPusher& dma_pusher) { - if (!pending_acquire) { + if (!pending_acquire || pending_acquire->met) { return true; } if (const auto device = dma_pusher.memory_manager.GpuToCpuAddress(pending_acquire->address); device && ViewApplies()) { - return PollWithoutDrain(dma_pusher, *device); + pending_acquire->met = PollWithoutDrain(dma_pusher, *device); + return pending_acquire->met; } const bool check = VideoCommon::Nx2DrainCheck::g_enabled; const bool met_before = @@ -469,13 +487,14 @@ void Puller::ProcessFenceActionMethod(DmaPusher& dma_pusher) { case Puller::FenceOperation::Acquire: // UNIMPLEMENTED_MSG("Channel Scheduling pending."); // WaitFence(regs.fence_action.syncpoint_id, regs.fence_value); - OnBack([rasterizer = dma_pusher.rasterizer] { rasterizer->ReleaseFences(); }); + OnBack([rasterizer = dma_pusher.rasterizer] { rasterizer->ReleaseFences(); }, FenceWrites); VideoCommon::GpuSplit::Get().NoteSync(); break; case Puller::FenceOperation::Increment: - OnBack([rasterizer = dma_pusher.rasterizer, id = static_cast(regs.fence_action.syncpoint_id.Value())] { - rasterizer->SignalSyncPoint(id); - }); + OnBack([rasterizer = dma_pusher.rasterizer, + id = static_cast( + regs.fence_action.syncpoint_id.Value())] { rasterizer->SignalSyncPoint(id); }, + FenceWrites); break; default: UNIMPLEMENTED_MSG("Unimplemented operation {}", regs.fence_action.op.Value()); @@ -507,9 +526,12 @@ void Puller::ProcessSemaphoreTriggerMethod(DmaPusher& dma_pusher) { const auto properties = g_nx2_ampere && (regs.semaphore_trigger & (1U << 24)) != 0 ? VideoCommon::QueryPropertiesFlags::IsAFence : VideoCommon::QueryPropertiesFlags::HasTimeout; - OnBack([rasterizer = dma_pusher.rasterizer, sequence_address, properties, payload] { - rasterizer->Query(sequence_address, VideoCommon::QueryType::Payload, properties, payload, 0); - }); + OnBack( + [rasterizer = dma_pusher.rasterizer, sequence_address, properties, payload] { + rasterizer->Query(sequence_address, VideoCommon::QueryType::Payload, properties, + payload, 0); + }, + VideoCommon::GpuSplit::Writes::Noted); NoteRelease(dma_pusher, sequence_address, payload, properties == VideoCommon::QueryPropertiesFlags::HasTimeout); } else if (g_nx2_ampere) { @@ -643,10 +665,12 @@ bool Puller::AcquireWithoutDrain(DmaPusher& dma_pusher, PendingAcquire& acquire) void Puller::ProcessSemaphoreRelease(DmaPusher& dma_pusher) { const GPUVAddr sequence_address{regs.semaphore_address.SemaphoreAddress()}; const u32 payload = regs.semaphore_release; - OnBack([rasterizer = dma_pusher.rasterizer, sequence_address, payload] { - rasterizer->Query(sequence_address, VideoCommon::QueryType::Payload, - VideoCommon::QueryPropertiesFlags::IsAFence, payload, 0); - }); + OnBack( + [rasterizer = dma_pusher.rasterizer, sequence_address, payload] { + rasterizer->Query(sequence_address, VideoCommon::QueryType::Payload, + VideoCommon::QueryPropertiesFlags::IsAFence, payload, 0); + }, + VideoCommon::GpuSplit::Writes::Noted); NoteRelease(dma_pusher, sequence_address, payload, false); } @@ -673,10 +697,12 @@ void Puller::ProcessSemaphoreExecute(DmaPusher& dma_pusher, u32 argument) { const u32 payload = regs.reg_array[0x19]; const u32 operation = argument & 7; if (operation == 1) { // RELEASE - OnBack([rasterizer = dma_pusher.rasterizer, address, payload] { - rasterizer->Query(address, VideoCommon::QueryType::Payload, - VideoCommon::QueryPropertiesFlags::IsAFence, payload, 0); - }); + OnBack( + [rasterizer = dma_pusher.rasterizer, address, payload] { + rasterizer->Query(address, VideoCommon::QueryType::Payload, + VideoCommon::QueryPropertiesFlags::IsAFence, payload, 0); + }, + VideoCommon::GpuSplit::Writes::Noted); NoteRelease(dma_pusher, address, payload, false); return; } @@ -696,9 +722,9 @@ void Puller::ProcessSemaphoreExecute(DmaPusher& dma_pusher, u32 argument) { // reads after the acquire waits for the work sent before it, as after a drain. auto& split = VideoCommon::GpuSplit::Get(); split.NoteSync(); - OnBack([rasterizer = dma_pusher.rasterizer] { rasterizer->ReleaseFences(); }); + OnBack([rasterizer = dma_pusher.rasterizer] { rasterizer->ReleaseFences(); }, FenceWrites); if (const auto queued = split.QueuedSemaphoreWriteAt(*device)) { - if (queued->known) { + if (queued->known && !queued->at_least) { word = queued->payload; } } else { @@ -741,13 +767,15 @@ void Puller::CallPullerMethod(DmaPusher& dma_pusher, const MethodCall& method_ca case BufferMethods::WrcacheFlush: break; case BufferMethods::RefCnt: - OnBack([rasterizer = dma_pusher.rasterizer] { rasterizer->SignalReference(); }); + OnBack([rasterizer = dma_pusher.rasterizer] { rasterizer->SignalReference(); }, + FenceWrites); break; case BufferMethods::SyncpointOperation: ProcessFenceActionMethod(dma_pusher); break; case BufferMethods::WaitForIdle: - OnBack([rasterizer = dma_pusher.rasterizer] { rasterizer->WaitForIdle(); }); + OnBack([rasterizer = dma_pusher.rasterizer] { rasterizer->WaitForIdle(); }, + VideoCommon::GpuSplit::Writes::Caches); VideoCommon::GpuSplit::Get().NoteSync(); break; case BufferMethods::SemaphoreOperation: { @@ -764,7 +792,8 @@ void Puller::CallPullerMethod(DmaPusher& dma_pusher, const MethodCall& method_ca } case BufferMethods::MemOpB: { // Implement this better. - OnBack([rasterizer = dma_pusher.rasterizer] { rasterizer->InvalidateGPUCache(); }); + OnBack([rasterizer = dma_pusher.rasterizer] { rasterizer->InvalidateGPUCache(); }, + VideoCommon::GpuSplit::Writes::Caches); break; } case BufferMethods::MemOpC: diff --git a/src/video_core/engines/puller.h b/src/video_core/engines/puller.h index 1f9c070b6f..9cf81bc7d3 100644 --- a/src/video_core/engines/puller.h +++ b/src/video_core/engines/puller.h @@ -90,6 +90,9 @@ class Puller final { /// the drain this replaces. u64 sent_at{}; bool fences_released{}; + /// CanResumeAcquire found it met: the channel resumes when it runs next, without + /// asking again (Control::Scheduler asks before it runs a suspended channel). + bool met{}; bool flushed{}; ///< TEST (EDEN_NX2_ACQUIRE_STATS): a flush was sent for it [[nodiscard]] bool Matches(u32 current) const; diff --git a/src/video_core/front_qmd.h b/src/video_core/front_qmd.h new file mode 100644 index 0000000000..d455cc9674 --- /dev/null +++ b/src/video_core/front_qmd.h @@ -0,0 +1,180 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later + +#pragma once + +#include +#include +#include + +#include "common/common_types.h" +#include "video_core/gpu_split.h" +#include "video_core/native_qmd.h" + +namespace VideoCommon { + +/// The QMD scheduling of every channel (Tegra::NativeQmdScheduler), run on the split GPU thread's +/// front as its back will run it, on a table of its own: what the launches release is known when +/// they are sent, for the front's view of the semaphore words (GpuSplit::NoteSemaphoreWrite). +/// +/// The front reads the descriptors when it sends a signal, the back when it runs it, both from +/// guest memory without the GPU caches. They read the same where no map still queued can change +/// and nothing queued writes there but through the caches (GpuSplit::MemoryWritesTarget) onto +/// pages GPU work never wrote (Nx2MayBeGpuWritten). Where the front can't tell, its table may +/// differ from the back's until it takes the back's over again (Sync), and the launches count as +/// writing guest memory anywhere. +/// +/// `Memory`, the channel's GPU address space as the back reads it, has: +/// - bool IsMapped(GPUVAddr, u64), void Read(GPUVAddr, void*, u64), u32 ReadWord(GPUVAddr) (as +/// MemoryManager::Read), bool InSyncpointShim(GPUVAddr, u64); +/// - std::optional Device(GPUVAddr), bool GpuWritten(u64 device, u64 size); +/// - u64 WritesTarget(const GpuSplit&, GPUVAddr, u64), void NoteWrite(GpuSplit&, GPUVAddr, u64). +class FrontQmdView { +public: + using Descriptor = Tegra::NativeQmdScheduler::Descriptor; + + /// Before a QMD method is sent: takes over the table `back` shares with the other engines once + /// the back ran everything sent. + void Sync(const GpuSplit& split, const Tegra::NativeQmdScheduler& back) { + if (!synced && split.Ran(split.Position())) [[unlikely]] { + scheduler.CopyStates(back); + synced = true; + } + } + + /// The engines schedule QMDs the front does not see (the split stopped, a front starts). + void Lose() { + synced = false; + } + + [[nodiscard]] bool Synced() const { + return synced; + } + + /// The scheduling of PCAS action `argument` on the QMD at `address` that the back runs at the + /// entry just sent (KeplerCompute::ProcessNativeMethod); `inline_qmd`: the QMD it writes there + /// right before. Notes what the launches it runs release. + template + void Signal(GpuSplit& split, const Memory& memory, GPUVAddr address, u32 argument, + const Descriptor* inline_qmd) { + split.NoteCacheWrite(); // The dispatches' shader stores + if (!synced) { + split.NoteAnyWrite(); + return; + } + releases.clear(); + // A map still queued may change what the back reads, mapped or not. + bool exact = split.Ran(split.LastUnplacedWrite()); + const auto read = [&](u64 qmd_address, Descriptor& descriptor) { + if (!memory.IsMapped(qmd_address, sizeof(descriptor))) { + return false; + } + if (inline_qmd != nullptr && qmd_address == address) { + descriptor = *inline_qmd; + } else { + memory.Read(qmd_address, descriptor.data(), sizeof(descriptor)); + const auto device = memory.Device(qmd_address); + exact = exact && device.has_value() && + !memory.GpuWritten(*device, sizeof(descriptor)) && + split.Ran(memory.WritesTarget(split, qmd_address, sizeof(descriptor))); + } + // A release this signal ran before writes into the QMD. + for (const Release& earlier : releases) { + exact = exact && (earlier.address + 16 <= qmd_address || + earlier.address >= qmd_address + sizeof(descriptor)); + } + return true; + }; + const auto release = [&](u64 destination, u32 payload, u32 flags) { + const bool four_words = (flags >> 30) == 0; + if (!memory.IsMapped(destination, four_words ? 16 : sizeof(u32))) { + return false; + } + releases.push_back({destination, payload, flags}); + return true; + }; + scheduler.Signal(address, argument, read, release, [](const Descriptor&) { return true; }); + if (!exact) { + synced = false; + split.NoteAnyWrite(); + return; + } + for (const Release& noted : releases) { + NoteRelease(split, memory, noted.address, noted.payload, noted.flags); + } + } + +private: + struct Release { + GPUVAddr address; + u32 payload; + u32 flags; + }; + + /// A release the back runs with the entry just sent, as ProcessNativeMethod's: what the word + /// holds once the back ran it. + template + void NoteRelease(GpuSplit& split, const Memory& memory, GPUVAddr address, u32 payload, + u32 flags) { + const bool four_words = (flags >> 30) == 0; + const bool reduction = (flags & (1U << 26)) != 0; + const bool maximum = !reduction && !four_words && (flags & (1U << 19)) != 0; + const auto device = memory.Device(address); + // The word a reduction or a maximum combines the payload with, before this write. + std::optional base; + if (device && (reduction || maximum)) { + base = WordAt(split, memory, address, *device); + } + memory.NoteWrite(split, address, four_words ? 16 : sizeof(u32)); + if (!device) { + return; + } + if (reduction) { + const bool is_signed = ((flags >> 24) & 1) != 0; + split.NoteSemaphoreWrite( + *device, + base ? Tegra::ReduceSemaphore((flags >> 20) & 7, is_signed, *base, payload) : 0, + base.has_value()); + } else if (four_words) { + // The payload as 64 bits, a timestamp after it. + split.NoteSemaphoreWrite(*device, payload, true); + split.NoteSemaphoreWrite(*device + 4, 0, true); + } else if (maximum) { + // Whatever the word held, it holds the payload at least. + split.NoteSemaphoreWrite(*device, base ? (std::max)(*base, payload) : payload, true, + !base.has_value()); + } else { + split.NoteSemaphoreWrite(*device, payload, true); + } + } + + /// The word the back finds at `address` (`device`) when it runs the entry just sent, where + /// the front knows it: the last release sent there while the back may not have run it, else + /// memory, once nothing queued may write there. + template + [[nodiscard]] std::optional WordAt(const GpuSplit& split, const Memory& memory, + GPUVAddr address, u64 device) const { + // The back reads the syncpoint shim from the syncpoints. + if (memory.InSyncpointShim(address, sizeof(u32))) { + return std::nullopt; + } + if (const auto queued = split.QueuedSemaphoreWriteAt(device)) { + if (queued->known && !queued->at_least && + queued->position >= split.LastUnplacedWrite()) { + return queued->payload; + } + return std::nullopt; + } + if (memory.GpuWritten(device, sizeof(u32)) || + !split.Ran(memory.WritesTarget(split, address, sizeof(u32)))) { + return std::nullopt; + } + return memory.ReadWord(address); + } + + Tegra::NativeQmdScheduler scheduler{Tegra::NativeQmdScheduler::OwnTable{}}; + std::vector releases; + bool synced{}; +}; + +} // namespace VideoCommon diff --git a/src/video_core/gpu_split.cpp b/src/video_core/gpu_split.cpp index f0800c9945..f8dbec71cf 100644 --- a/src/video_core/gpu_split.cpp +++ b/src/video_core/gpu_split.cpp @@ -145,6 +145,7 @@ void GpuSplit::Stop() { semaphore_writes.clear(); semaphore_writes_limit = 1024; last_any_write = 0; + last_unplaced_write = 0; last_noted_write = 0; write_positions.fill(0); published_local = 0; @@ -217,7 +218,7 @@ void GpuSplit::Drain(DrainSite site) { for (u32 spins = 0;; ++spins) { consumed_cache = consumed.load(std::memory_order_acquire); if (consumed_cache == write_position) { - publish_every = 8; + publish_every = PublishAfterWait; Nx2Profile::AddDrain(site, start); return; } @@ -237,7 +238,7 @@ bool GpuSplit::WaitForSyncedWriters(DrainSite site) { return WaitFor(std::min({last_writer_position, last_sync_position, write_position}), site); } -void GpuSplit::NoteSemaphoreWrite(u64 device_address, u32 payload, bool known) { +void GpuSplit::NoteSemaphoreWrite(u64 device_address, u32 payload, bool known, bool at_least) { if (semaphore_writes.size() >= semaphore_writes_limit) { // The words of releases the back ran hold their values in memory. const u64 done = consumed.load(std::memory_order_acquire); @@ -246,7 +247,7 @@ void GpuSplit::NoteSemaphoreWrite(u64 device_address, u32 payload, bool known) { semaphore_writes_limit = std::max(1024, semaphore_writes.size() * 2); } semaphore_writes.insert_or_assign(device_address & ~u64{3}, - SemaphoreWrite{write_position, payload, known}); + SemaphoreWrite{write_position, payload, known, at_least}); } std::optional GpuSplit::QueuedSemaphoreWriteAt( @@ -256,7 +257,8 @@ std::optional GpuSplit::QueuedSemaphoreWriteAt( it->second.position <= consumed.load(std::memory_order_acquire)) { return std::nullopt; } - return QueuedSemaphoreWrite{it->second.payload, it->second.known, it->second.position}; + return QueuedSemaphoreWrite{it->second.payload, it->second.known, it->second.at_least, + it->second.position}; } bool GpuSplit::WritersPending() const { @@ -312,6 +314,19 @@ u64 GpuSplit::WritesTarget(const Tegra::MemoryManager& memory, GPUVAddr address, return target; } +u64 GpuSplit::MemoryWritesTarget(const Tegra::MemoryManager& memory, GPUVAddr address, + u64 size) const { + u64 target = last_unplaced_write; + if (size > MaxNotedBytes || address + size < address) { + target = std::max(target, last_noted_write); + } else { + ForEachDeviceRange(memory, address, size, [this, &target](DAddr device, u64 bytes) { + target = std::max(target, DeviceWritesTarget(device, bytes)); + }); + } + return std::min(target, write_position); +} + u64 GpuSplit::DeviceWritesTarget(DAddr address, u64 size) const { u64 target = 0; const u64 last = (address + size - 1) >> WriteGranuleBits; @@ -339,6 +354,8 @@ bool GpuSplit::WaitFor(u64 target, DrainSite site) { consumed_cache = consumed.load(std::memory_order_acquire); if (consumed_cache >= target) { wait_target.store(~u64{0}, std::memory_order_relaxed); + // The back may run out of what was sent before long. + publish_every = PublishAfterWait; Nx2Profile::AddDrain(site, start); return true; } @@ -397,9 +414,7 @@ void GpuSplit::MultiMethod(Tegra::Engines::EngineInterface* engine, u32 method, amount -= piece; methods_pending -= piece; } - if (writes == Writes::Anywhere) { - NoteAnyWrite(); - } + NoteWrites(writes); } void GpuSplit::DeferFunction(std::function&& func) { diff --git a/src/video_core/gpu_split.h b/src/video_core/gpu_split.h index 3938944a5b..dbe71825e5 100644 --- a/src/video_core/gpu_split.h +++ b/src/video_core/gpu_split.h @@ -130,10 +130,14 @@ class GpuSplit { void Stop(); /// What the sender of an entry tells about the guest memory the back writes running it, for - /// the front's reads of guest memory (WaitForWrites). + /// the front's reads of guest memory (WaitForWrites, MemoryWritesTarget). enum class Writes : u8 { Anywhere, ///< Unknown: a read anywhere waits for the entry - Noted, ///< Nothing, or what the sender notes right after sending it (NoteWrite) + /// The GPU caches anywhere (shader stores, copies into buffers), guest memory only through + /// them, where GPU work wrote (Nx2MayBeGpuWritten): a read of the caches waits for it, a + /// read of other guest memory not (MemoryWritesTarget) + Caches, + Noted, ///< Nothing, or what the sender notes right after sending it (NoteWrite) }; /// An engine method write, executed as DmaPusher would (method sink, ConsumeSink, call). @@ -150,9 +154,7 @@ class GpuSplit { entry[1] = *segment; } Commit(words); - if (writes == Writes::Anywhere) { - NoteAnyWrite(); - } + NoteWrites(writes); } void MultiMethod(Tegra::Engines::EngineInterface* engine, u32 method, const u32* words, u32 amount, u32 methods_pending, const GPUVAddr* segment, @@ -165,9 +167,7 @@ class GpuSplit { entry[0] = static_cast(Kind::Method) | IsLast | Direct | (u64{method & 0xffff} << 8) | (u64{argument} << 32); Commit(1); - if (writes == Writes::Anywhere) { - NoteAnyWrite(); - } + NoteWrites(writes); } /// Runs `func` on the back after everything sent before it. Must be trivially copyable. @@ -180,9 +180,7 @@ class GpuSplit { entry[1] = reinterpret_cast(&CallThunk); std::memcpy(entry + 2, &func, sizeof(Func)); Commit(2 + payload); - if (writes == Writes::Anywhere) { - NoteAnyWrite(); - } + NoteWrites(writes); } /// Defer, calling `func` with a copy of `words` (a std::span). template @@ -201,9 +199,7 @@ class GpuSplit { std::memcpy(entry + 3 + function_words, words.data(), words.size_bytes()); } Commit(2 + payload); - if (writes == Writes::Anywhere) { - NoteAnyWrite(); - } + NoteWrites(writes); } void DeferFunction(std::function&& func); /// Runs `func` on the back and waits for it: for work on back-owned state, such as a buffer @@ -226,6 +222,11 @@ class GpuSplit { /// Makes what was sent visible to the back. void Publish(); + /// The front waited for the back, which may run out of work soon: publish the next words + /// after a few, as after a drain, not after PublishEvery. + void PublishSoon() { + publish_every = PublishAfterWait; + } /// Waits until the back executed everything sent so far. void Drain(DrainSite site = DrainSite::Other); @@ -268,12 +269,14 @@ class GpuSplit { /// still queued. `site`: whose wait it is, for EDEN_NX2_GPU_PROFILE. bool WaitForSyncedWriters(DrainSite site = DrainSite::ListWriters); /// A semaphore release was just sent: once the back ran it, the word at `device_address` holds - /// `payload`, or a value the front can't tell (`known` false: a reduction, a counter report). - void NoteSemaphoreWrite(u64 device_address, u32 payload, bool known); + /// `payload`, or a value the front can't tell (`known` false: a reduction, a counter report), + /// or `payload` at least (`at_least`: a maximum with a value the front can't tell). + void NoteSemaphoreWrite(u64 device_address, u32 payload, bool known, bool at_least = false); struct QueuedSemaphoreWrite { u32 payload; bool known; - u64 position; ///< Once the back ran this (Ran), the word holds the write + bool at_least; ///< Known only as a lower bound (unsigned) + u64 position; ///< Once the back ran this (Ran), the word holds the write }; /// The last write sent to the semaphore word at `device_address`, if the back may not have run /// it yet. @@ -296,7 +299,30 @@ class GpuSplit { /// The entry just sent may write guest memory anywhere. void NoteAnyWrite() { last_any_write = write_position; + last_unplaced_write = write_position; + } + /// The entry just sent may write the GPU caches anywhere (Writes::Caches). + void NoteCacheWrite() { + last_any_write = write_position; + } + void NoteWrites(Writes writes) { + if (writes == Writes::Anywhere) { + NoteAnyWrite(); + } else if (writes == Writes::Caches) { + NoteCacheWrite(); + } + } + /// The position after the last entry sent that may write guest memory where the front does + /// not know (NoteAnyWrite), outside what the caches hold. + [[nodiscard]] u64 LastUnplacedWrite() const { + return (std::min)(last_unplaced_write, write_position); } + /// Once the back ran everything before the position returned, no entry sent so far writes + /// guest memory in [address, address + size) of `memory` but through the GPU caches + /// (Writes::Caches): what the back reads there without them, as the QMD scheduler does, is in + /// memory then. Placed as WaitForWrites places writes. + [[nodiscard]] u64 MemoryWritesTarget(const Tegra::MemoryManager& memory, GPUVAddr address, + u64 size) const; /// Waits until the back executed every entry sent so far that can write guest memory in /// [address, address + size) of `memory`, not the others. Returns whether one was queued. /// Writes are placed by the device memory the GPU addresses map to when noted and read: GPU @@ -321,13 +347,15 @@ class GpuSplit { static constexpr size_t RingMask = RingWords - 1; static constexpr u32 MaxMultiWords = 8192; static constexpr size_t PublishEvery = 256; + static constexpr size_t PublishAfterWait = 8; /// NoteWrite places writes by 256-byte granule of device memory, up to MaxNotedBytes long /// (longer ones count as anywhere), in write_positions slots hashed by granule: granules that /// share a slot share its position, so a read may wait for a write elsewhere, never miss one. + /// 16384 slots: the constant buffer words of a few hundred queued draws share few with a QMD. static constexpr u32 WriteGranuleBits = 8; static constexpr u64 MaxNotedBytes = 0x4000; - static constexpr u32 WriteSlotBits = 10; + static constexpr u32 WriteSlotBits = 14; static size_t WriteSlot(u64 granule) { return static_cast((granule * 0x9E3779B97F4A7C15ULL) >> (64 - WriteSlotBits)); @@ -410,18 +438,22 @@ class GpuSplit { u64 position; ///< write_position once it was sent u32 payload; bool known; + bool at_least; }; /// The front's NoteSemaphoreWrite, by word address. std::unordered_map semaphore_writes; size_t semaphore_writes_limit = 1024; ///< Forget what the back ran past this many - /// After the last entry that may write guest memory anywhere (NoteAnyWrite), the last one - /// NoteWrite placed, and the last placed write in each slot's granules. + /// After the last entry that may write guest memory or the GPU caches anywhere (NoteAnyWrite, + /// NoteCacheWrite), the last one that may write guest memory itself anywhere (NoteAnyWrite), + /// the last one NoteWrite placed, and the last placed write in each slot's granules. u64 last_any_write{}; + u64 last_unplaced_write{}; u64 last_noted_write{}; std::array write_positions{}; u64 published_local{}; u64 consumed_cache{}; - /// Words between publishes: small right after a drain, while the back has nothing to do. + /// Words between publishes: small right after the front waited for the back (Drain, + /// PublishSoon), while the back may have nothing to do. size_t publish_every{PublishEvery}; Tegra::Engines::EngineInterface* front_engine{}; diff --git a/src/video_core/gpu_thread.cpp b/src/video_core/gpu_thread.cpp index 2ad7942e5e..711c62a5f6 100644 --- a/src/video_core/gpu_thread.cpp +++ b/src/video_core/gpu_thread.cpp @@ -118,6 +118,8 @@ void ThreadManager::StartThread(VideoCore::RendererBase& renderer, Core::Fronten } last_progress = std::chrono::steady_clock::now(); if (split.Active()) { + // Between commands the back may have run out of work. + split.PublishSoon(); // Only command lists are parsed here; everything else runs on the back thread // in order, and so does the signal that the command ran. auto* const rasterizer_ptr = renderer.ReadRasterizer(); @@ -136,19 +138,22 @@ void ThreadManager::StartThread(VideoCore::RendererBase& renderer, Core::Fronten split.DeferFunction(std::move(run->func)); } } else if (const auto* flush = std::get_if(&next.data)) { - split.Defer([rasterizer_ptr, addr = flush->addr, size = flush->size] { - rasterizer_ptr->FlushRegion(addr, size); - }); + // Writes guest memory with what the caches hold of it. + split.Defer([rasterizer_ptr, addr = flush->addr, + size = flush->size] { rasterizer_ptr->FlushRegion(addr, size); }, + VideoCommon::GpuSplit::Writes::Caches); } else if (const auto* invalidate = std::get_if(&next.data)) { - split.Defer([rasterizer_ptr, addr = invalidate->addr, size = invalidate->size] { - rasterizer_ptr->OnCacheInvalidation(addr, size); - }); + split.Defer( + [rasterizer_ptr, addr = invalidate->addr, size = invalidate->size] { + rasterizer_ptr->OnCacheInvalidation(addr, size); + }, + VideoCommon::GpuSplit::Writes::Noted); } else { ASSERT(false); } - split.Defer([state_ptr = &state, fence = next.fence, block = next.block] { - SignalCommand(*state_ptr, fence, block); - }); + split.Defer([state_ptr = &state, fence = next.fence, + block = next.block] { SignalCommand(*state_ptr, fence, block); }, + VideoCommon::GpuSplit::Writes::Noted); split.Publish(); continue; } diff --git a/src/video_core/memory_manager.h b/src/video_core/memory_manager.h index 194a190151..b74f9045ec 100644 --- a/src/video_core/memory_manager.h +++ b/src/video_core/memory_manager.h @@ -48,6 +48,13 @@ class MemoryManager final { // Switch 2: GPU address of the read-only syncpoint shim (u32 per syncpoint); 0 = none. void SetNativeSyncpointShim(GPUVAddr address); static constexpr u32 NumShimSyncpoints = 192; // Tegra::Host1x::SyncpointManager + /// Whether [address, address + size) overlaps the syncpoint shim, whose words Read serves + /// from the guest syncpoints instead of memory. + [[nodiscard]] bool OverlapsSyncpointShim(GPUVAddr address, u64 size) const { + const GPUVAddr shim = native_syncpoint_shim.load(std::memory_order_acquire); + return shim != 0 && address < shim + NumShimSyncpoints * sizeof(u32) && + address + size > shim; + } /// Switch 2 lays out block-linear surfaces with the console's GOB swizzle. void EnableNativeTiling() { diff --git a/src/video_core/native_qmd.h b/src/video_core/native_qmd.h index f32afb363b..bbf4266b44 100644 --- a/src/video_core/native_qmd.h +++ b/src/video_core/native_qmd.h @@ -37,6 +37,20 @@ class NativeQmdScheduler { using Address = std::uint64_t; using Descriptor = std::array; struct RejectShaderDispatch { bool operator()(const Descriptor&) const { return false; } }; + /// A scheduler with a table of its own, not the one the channels share: the split GPU thread's + /// front runs the scheduling ahead of its back to know what the launches release + /// (KeplerCompute). + struct OwnTable {}; + NativeQmdScheduler() = default; + explicit NativeQmdScheduler(OwnTable) : table{&own_states} {} + NativeQmdScheduler(const NativeQmdScheduler&) = delete; + NativeQmdScheduler& operator=(const NativeQmdScheduler&) = delete; + /// Takes the QMD states of `other` over, while nothing runs on either. + void CopyStates(const NativeQmdScheduler& other) { + if (table != other.table) { + *table = *other.table; + } + } // Optional diagnostics: called with "load"/"count"/"defer"/"run" events. std::function trace; // Optional diagnostics: called with the QMD, slot and raw words of a release it skips. @@ -57,6 +71,9 @@ class NativeQmdScheduler { template std::string_view Signal(Address address, Word argument, Read read, Release release, Dispatch dispatch) { struct Pending { Address address; Word action; Word count; }; + auto& states = *table; + // What `read` returns is used before the next read only: a table entry holds no copy. + Descriptor descriptor{}; std::string_view error; std::array pending{}; std::size_t count = 1; @@ -73,8 +90,8 @@ class NativeQmdScheduler { auto& state = states[request.address]; if (request.action == 3 || request.action == 2) { if (request.action == 3 || !state.loaded) { - if (!read(request.address, state.descriptor)) return "unmapped QMD"; - const auto& qmd = state.descriptor; + if (!read(request.address, descriptor)) return "unmapped QMD"; + const auto& qmd = descriptor; if ((qmd[18] & 0xff) != 0x30 || (qmd[4] & 0x100)) return "unsupported QMD version or queue"; if constexpr (std::is_same_v) { if (qmd[12] && (qmd[13] & 0xffff) && (qmd[14] & 0xffff)) @@ -106,8 +123,8 @@ class NativeQmdScheduler { if (state.dependencies < 0) return "QMD dependency underflow"; // The SDK patches release payloads while a scheduled QMD is // waiting for its remaining dependencies. Fetch them at execution. - if (!read(request.address, state.descriptor)) return "unmapped QMD"; - const auto qmd = state.descriptor; + if (!read(request.address, descriptor)) return "unmapped QMD"; + const auto qmd = descriptor; if ((qmd[18] & 0xff) != 0x30 || (qmd[4] & 0x100)) return "unsupported QMD version or queue"; // A program Eden can't run is skipped, but the QMD still completes: its releases // are what other queues and the CPU wait on (the console never fails a launch). @@ -144,7 +161,6 @@ class NativeQmdScheduler { static constexpr std::array ReleaseSlots{24, 28, 52}; struct State { - Descriptor descriptor{}; std::int32_t dependencies{}; std::int32_t deferred{}; bool loaded{}; @@ -154,6 +170,8 @@ class NativeQmdScheduler { // One table for every channel: a QMD's dependency counter is state of the QMD itself, and // title 1 schedules a QMD on one channel while another decrements it (a decrement that // arrived first was kept by the other channel's scheduler and the QMD never launched). - static inline std::unordered_map states; + static inline std::unordered_map shared_states; + std::unordered_map own_states; + std::unordered_map* table{&shared_states}; }; } // namespace Tegra From 2c59c7bb581f6c8a1c7e0531d61cc51d0b059bbd Mon Sep 17 00:00:00 2001 From: jpkto <255363096+jpkto@users.noreply.github.com> Date: Sun, 8 Nov 2026 02:27:20 +0100 Subject: [PATCH 033/105] TEST: check the QMD view's releases, count it and the late publishes EDEN_NX2_DRAIN_CHECK now also checks the QMD view. The back records what it releases for QMD launches. After the QMD methods the front sent (a signal, an inline launch, a run of them, or a macro's write of one), it compares that list with what the front noted, in order. Same words, and values equal, or at least as large for a lower bound. The "Drain checks" line ends with: QMD view: methods valued N with R releases, released otherwise than noted W; not valued U W must stay 0. The first 16 that differ are logged with both lists. The acquire checks already cover the decisions the new notes make. EDEN_NX2_ACQUIRE_STATS adds: - "passed by a queued QMD launch's release": passes by a queued release at words the QMD view noted releases to; - QMD launches the front valued, among the releasers of words that suspended acquires waited at ("none" is now other QMD releases and the CPU); - a "QMD view" line: signals valued, and why the others were not (table not synced yet, work that may write anywhere still queued, a QMD on a GPU-written page, a write into the QMD still queued, a release of the same signal into it); releases noted exact, at least or unknown; how often the front took the back's table over, and how long that took. The resume counts are no longer doubled: the clean commit polls a suspended acquire once per resume. EDEN_NX2_GPU_PROFILE adds to the profile line: - "publishes to an idle back late by T ms (N/s)": time the back waited for work while the front held words it could have run. It counts from when the back ran out, or from when the first of those words was sent, if that was later. This is the churn cost the clean commit's earlier publishes cut. - "resnapshot words": words copied again when a channel resumes, with a "resnapshot" zone for the time the copies take. DeferFunction takes a write class, so the checks' entries write nothing for the view. Checked with a harness (FrontQmdView against a model of the back, 200 random runs): all 519,764 check lists matched, as did the clean commit's 233,000 notes. A second program prints the new report lines. Rebased onto upstream: GpuSplit takes the publish profiling from Nx2Profile::Enabled() when the split starts, instead of reading EDEN_NX2_GPU_PROFILE at static initialization. Since #714 the profile switch is read again when the GPU is made, so a switch set in nx2_switches.txt now profiles the publishes too. Co-Authored-By: Claude Opus 5.5 --- src/video_core/dma_pusher.cpp | 3 + src/video_core/engines/kepler_compute.cpp | 38 ++++- src/video_core/engines/kepler_compute.h | 1 + src/video_core/front_qmd.h | 77 ++++++++-- src/video_core/gpu_split.cpp | 44 +++++- src/video_core/gpu_split.h | 16 +- src/video_core/nx2_drain_check.h | 177 ++++++++++++++++++++-- src/video_core/nx2_profile.h | 18 ++- 8 files changed, 344 insertions(+), 30 deletions(-) diff --git a/src/video_core/dma_pusher.cpp b/src/video_core/dma_pusher.cpp index 10aad31fef..d7e2af0659 100644 --- a/src/video_core/dma_pusher.cpp +++ b/src/video_core/dma_pusher.cpp @@ -644,6 +644,7 @@ void DmaPusher::NoteDmaSemaphore(u32 method, u32 argument) { } void DmaPusher::RefreshSnapshot(CommandList& list, u64 now) { + VideoCommon::Nx2Profile::Scope profile_scope{VideoCommon::Nx2Profile::Zone::Resnapshot}; // TEST list.snapshot_suspensions = now; for (size_t i = 0; i < list.command_lists.size() && i < list.submitted_offsets.size(); ++i) { const CommandListHeader& entry = list.command_lists[i]; @@ -653,6 +654,8 @@ void DmaPusher::RefreshSnapshot(CommandList& list, u64 now) { } memory_manager.ReadBlockUnsafe(entry.addr, list.submitted_words.data() + offset, entry.size * sizeof(u32)); + VideoCommon::Nx2Profile::Add(VideoCommon::Nx2Profile::Counter::ResnapshotWords, + entry.size); // TEST } list.deferred_entries.clear(); } diff --git a/src/video_core/engines/kepler_compute.cpp b/src/video_core/engines/kepler_compute.cpp index bea30071f6..bbfb3bc1f3 100644 --- a/src/video_core/engines/kepler_compute.cpp +++ b/src/video_core/engines/kepler_compute.cpp @@ -370,11 +370,21 @@ bool KeplerCompute::ProcessNativeMethod(Core::System& system, u32 method, u32 ar [&](u64 address, u32 payload, u32 flags) { const bool four_words = (flags >> 30) == 0; if (!memory_manager.IsFullyMappedRange(address, four_words ? 16 : sizeof(payload))) return false; + // TEST (EDEN_NX2_DRAIN_CHECK): what the front's QMD view compares its notes with. + const auto checked = [this, address](u32 offset, u32 value) { + if (VideoCommon::Nx2DrainCheck::g_enabled) [[unlikely]] { + if (const auto device = memory_manager.GpuToCpuAddress(address)) { + VideoCommon::Nx2DrainCheck::NoteQmdRelease( + *device + offset, value, VideoCommon::GpuSplit::OnBackThread()); + } + } + }; if (flags & (1U << 26)) { - rasterizer->SignalFence([this, address, payload, flags] { + rasterizer->SignalFence([this, address, payload, flags, checked] { const u32 value = ReduceSemaphore((flags >> 20) & 7, (flags >> 24) & 1, memory_manager.Read(address), payload); memory_manager.Write(address, value); + checked(0, value); if (const auto device = memory_manager.GpuToCpuAddress(address)) { Tegra::Nx2SemaphoreWaiters::Get().OnWrite( *device, value, Tegra::Nx2SemaphoreWaiters::Source::QmdReduction); @@ -393,13 +403,16 @@ bool KeplerCompute::ProcessNativeMethod(Core::System& system, u32 method, u32 ar rasterizer->Query(address, VideoCommon::QueryType::Payload, VideoCommon::QueryPropertiesFlags::IsAFence | VideoCommon::QueryPropertiesFlags::HasTimeout, payload, 0); + checked(0, payload); + checked(4, 0); } else if (flags & (1U << 19)) { // NVN2 tracks QMD completion with a max release (bit 19) into a word other // releases also advance: a plain write would move it backwards and leave // GEQ acquires waiting forever. - rasterizer->SignalFence([this, address, payload] { + rasterizer->SignalFence([this, address, payload, checked] { const u32 value = (std::max)(memory_manager.Read(address), payload); memory_manager.Write(address, value); + checked(0, value); if (const auto device = memory_manager.GpuToCpuAddress(address)) { Tegra::Nx2SemaphoreWaiters::Get().OnWrite( *device, value, Tegra::Nx2SemaphoreWaiters::Source::QmdMax); @@ -415,6 +428,7 @@ bool KeplerCompute::ProcessNativeMethod(Core::System& system, u32 method, u32 ar } rasterizer->Query(address, VideoCommon::QueryType::Payload, VideoCommon::QueryPropertiesFlags::IsAFence, payload, 0); + checked(0, payload); } return true; }, [&](const auto& descriptor) { return DispatchNativeCompute(descriptor); }); @@ -1623,6 +1637,24 @@ void KeplerCompute::FrontCallMethod(Core::System& system, VideoCommon::GpuSplit& split.Method(this, method, argument, is_last_call, segment, FrontWrites(native_plain_methods, method)); FrontNoteWrites(split, fs, method, argument, false); + FrontQmdCheck(split); +} + +// TEST (EDEN_NX2_DRAIN_CHECK): has the back compare what it releases for the QMD methods just +// sent with what the front's QMD view noted for them. +void KeplerCompute::FrontQmdCheck(VideoCommon::GpuSplit& split) { + if (!VideoCommon::Nx2DrainCheck::g_enabled) [[likely]] { + return; + } + auto checks = GetFrontQmd().TakeChecks(); + if (!checks) { + return; + } + split.DeferFunction( + [predicted = std::move(checks->releases), valued = checks->valued] { + VideoCommon::Nx2DrainCheck::CheckQmdReleases(predicted, valued); + }, + VideoCommon::GpuSplit::Writes::Noted); } // What the back writes to guest memory running `method` with `argument`, just sent as @@ -1747,6 +1779,7 @@ void KeplerCompute::FrontCallMultiMethod(Core::System& system, VideoCommon::GpuS for (u32 i = 0; i < amount; ++i) { FrontNoteWrites(split, fs, method, base_start[i], false); } + FrontQmdCheck(split); } void KeplerCompute::FrontRunMacro(Core::System& system, VideoCommon::GpuSplit& split, @@ -1772,6 +1805,7 @@ void KeplerCompute::FrontRunMacro(Core::System& system, VideoCommon::GpuSplit& s } split.MacroWrite(this, address, value, FrontWrites(native_plain_methods, address)); FrontNoteWrites(split, fs, address, value, true); + FrontQmdCheck(split); return true; }; #ifdef ARCHITECTURE_x86_64 diff --git a/src/video_core/engines/kepler_compute.h b/src/video_core/engines/kepler_compute.h index a9588c18b7..3287b81541 100644 --- a/src/video_core/engines/kepler_compute.h +++ b/src/video_core/engines/kepler_compute.h @@ -279,6 +279,7 @@ class KeplerCompute final : public EngineInterface { void FrontNoteReport(VideoCommon::GpuSplit& split, const FrontState& state, u32 control, bool from_macro); void FrontInlineQmd(VideoCommon::GpuSplit& split, const FrontState& state, u32 last_word); + void FrontQmdCheck(VideoCommon::GpuSplit& split); ///< TEST MacroEngine& FrontMacroEngine(FrontState& state); std::span FrontMacroPositions(FrontState& state); void FrontRunMacro(Core::System& system, VideoCommon::GpuSplit& split, FrontState& state); diff --git a/src/video_core/front_qmd.h b/src/video_core/front_qmd.h index d455cc9674..c45b506386 100644 --- a/src/video_core/front_qmd.h +++ b/src/video_core/front_qmd.h @@ -4,12 +4,14 @@ #pragma once #include +#include #include #include #include "common/common_types.h" #include "video_core/gpu_split.h" #include "video_core/native_qmd.h" +#include "video_core/nx2_drain_check.h" namespace VideoCommon { @@ -37,11 +39,30 @@ class FrontQmdView { /// the back ran everything sent. void Sync(const GpuSplit& split, const Tegra::NativeQmdScheduler& back) { if (!synced && split.Ran(split.Position())) [[unlikely]] { + const auto start = std::chrono::steady_clock::now(); // TEST scheduler.CopyStates(back); synced = true; + Nx2AcquireStats::NoteQmdSync(std::chrono::steady_clock::now() - start); } } + /// TEST (EDEN_NX2_DRAIN_CHECK): the releases noted since the last call, and whether every + /// signal since was valued; nothing if no signal was sent since. + struct Checks { + std::vector releases; + bool valued; + }; + [[nodiscard]] std::optional TakeChecks() { + if (!checks_pending) { + return std::nullopt; + } + checks_pending = false; + Checks taken{std::move(checks), !checks_unvalued}; + checks.clear(); + checks_unvalued = false; + return taken; + } + /// The engines schedule QMDs the front does not see (the split stopped, a front starts). void Lose() { synced = false; @@ -58,13 +79,21 @@ class FrontQmdView { void Signal(GpuSplit& split, const Memory& memory, GPUVAddr address, u32 argument, const Descriptor* inline_qmd) { split.NoteCacheWrite(); // The dispatches' shader stores + using Lost = Nx2AcquireStats::QmdLost; + checks_pending = Nx2DrainCheck::g_enabled; // TEST if (!synced) { split.NoteAnyWrite(); + checks_unvalued = true; // TEST + Nx2AcquireStats::NoteQmdSignal(Lost::NotSynced); return; } releases.clear(); // A map still queued may change what the back reads, mapped or not. bool exact = split.Ran(split.LastUnplacedWrite()); + std::optional lost; // TEST: why not + if (!exact) { + lost = Lost::Unplaced; + } const auto read = [&](u64 qmd_address, Descriptor& descriptor) { if (!memory.IsMapped(qmd_address, sizeof(descriptor))) { return false; @@ -74,12 +103,21 @@ class FrontQmdView { } else { memory.Read(qmd_address, descriptor.data(), sizeof(descriptor)); const auto device = memory.Device(qmd_address); - exact = exact && device.has_value() && - !memory.GpuWritten(*device, sizeof(descriptor)) && - split.Ran(memory.WritesTarget(split, qmd_address, sizeof(descriptor))); + const bool written = + !device.has_value() || memory.GpuWritten(*device, sizeof(descriptor)); + const bool pending = + !split.Ran(memory.WritesTarget(split, qmd_address, sizeof(descriptor))); + if (exact && (written || pending)) { + lost = written ? Lost::GpuWritten : Lost::Pending; + } + exact = exact && !written && !pending; } // A release this signal ran before writes into the QMD. for (const Release& earlier : releases) { + if (exact && earlier.address + 16 > qmd_address && + earlier.address < qmd_address + sizeof(descriptor)) { + lost = Lost::ReleasedInto; + } exact = exact && (earlier.address + 16 <= qmd_address || earlier.address >= qmd_address + sizeof(descriptor)); } @@ -94,9 +132,11 @@ class FrontQmdView { return true; }; scheduler.Signal(address, argument, read, release, [](const Descriptor&) { return true; }); + Nx2AcquireStats::NoteQmdSignal(lost); if (!exact) { synced = false; split.NoteAnyWrite(); + checks_unvalued = true; // TEST return; } for (const Release& noted : releases) { @@ -129,22 +169,37 @@ class FrontQmdView { if (!device) { return; } + // TEST: what was noted (EDEN_NX2_DRAIN_CHECK, EDEN_NX2_ACQUIRE_STATS). + const auto check = [this](u64 word, u32 value, bool known, bool at_least) { + using Note = Nx2AcquireStats::QmdNote; + Nx2AcquireStats::NoteQmdNote(!known ? Note::Unknown + : at_least ? Note::AtLeast + : Note::Exact); + if (checks_pending) { + checks.push_back({word, value, known, at_least}); + } + }; + Nx2AcquireStats::NoteRelease(*device, Nx2AcquireStats::Releaser::Qmd); if (reduction) { const bool is_signed = ((flags >> 24) & 1) != 0; - split.NoteSemaphoreWrite( - *device, - base ? Tegra::ReduceSemaphore((flags >> 20) & 7, is_signed, *base, payload) : 0, - base.has_value()); + const u32 value = + base ? Tegra::ReduceSemaphore((flags >> 20) & 7, is_signed, *base, payload) : 0; + split.NoteSemaphoreWrite(*device, value, base.has_value()); + check(*device, value, base.has_value(), false); } else if (four_words) { // The payload as 64 bits, a timestamp after it. split.NoteSemaphoreWrite(*device, payload, true); split.NoteSemaphoreWrite(*device + 4, 0, true); + check(*device, payload, true, false); + check(*device + 4, 0, true, false); } else if (maximum) { // Whatever the word held, it holds the payload at least. - split.NoteSemaphoreWrite(*device, base ? (std::max)(*base, payload) : payload, true, - !base.has_value()); + const u32 value = base ? (std::max)(*base, payload) : payload; + split.NoteSemaphoreWrite(*device, value, true, !base.has_value()); + check(*device, value, true, !base.has_value()); } else { split.NoteSemaphoreWrite(*device, payload, true); + check(*device, payload, true, false); } } @@ -175,6 +230,10 @@ class FrontQmdView { Tegra::NativeQmdScheduler scheduler{Tegra::NativeQmdScheduler::OwnTable{}}; std::vector releases; bool synced{}; + /// TEST (EDEN_NX2_DRAIN_CHECK): see TakeChecks. + std::vector checks; + bool checks_pending{}; + bool checks_unvalued{}; }; } // namespace VideoCommon diff --git a/src/video_core/gpu_split.cpp b/src/video_core/gpu_split.cpp index f8dbec71cf..cda0e266bc 100644 --- a/src/video_core/gpu_split.cpp +++ b/src/video_core/gpu_split.cpp @@ -116,6 +116,7 @@ void GpuSplit::MaybeStart(Core::System& system_) { } system = &system_; check_owner = std::getenv("EDEN_NX2_SPLIT_CHECK") != nullptr; + profile_publishes = Nx2Profile::Enabled(); // TEST ring = std::make_unique(RingWords); back_thread = std::jthread([this](std::stop_token stop_token) { BackLoop(stop_token); }); active = true; @@ -194,10 +195,31 @@ void GpuSplit::WaitForSpace(size_t words) { } } +void GpuSplit::NoteFirstUnpublished() { + first_unpublished_ticks = Nx2Profile::Ticks(); +} + +void GpuSplit::NoteLatePublish() { + const u64 idle_since = back_idle_since.load(std::memory_order_relaxed); + if (idle_since == 0 || first_unpublished_ticks == 0) { + return; + } + // The back could have run the words since it ran out of work or since the first was sent. + const u64 since = std::max(idle_since, first_unpublished_ticks); + const u64 now = Nx2Profile::Ticks(); + if (now > since) { + Nx2Profile::Add(Nx2Profile::Counter::LatePublishTicks, now - since); + Nx2Profile::Add(Nx2Profile::Counter::LatePublishes, 1); + } +} + void GpuSplit::Publish() { if (published_local == write_position) { return; } + if (profile_publishes) [[unlikely]] { + NoteLatePublish(); // TEST + } published_local = write_position; publish_every = std::min(publish_every * 2, PublishEvery); published.store(write_position, std::memory_order_seq_cst); @@ -417,12 +439,14 @@ void GpuSplit::MultiMethod(Tegra::Engines::EngineInterface* engine, u32 method, NoteWrites(writes); } -void GpuSplit::DeferFunction(std::function&& func) { +void GpuSplit::DeferFunction(std::function&& func, Writes writes) { auto* const heap = new std::function(std::move(func)); - Defer([heap] { - (*heap)(); - delete heap; - }); + Defer( + [heap] { + (*heap)(); + delete heap; + }, + writes); } size_t GpuSplit::Execute(const u64* entry, size_t words) { @@ -546,6 +570,16 @@ void GpuSplit::BackLoop(std::stop_token stop_token) { if (available == read_position) { Nx2Profile::Scope scope{Nx2Profile::Zone::SplitWait}; const GpuWaitStats::IdleScope idle{GpuWaitStats::g_back_idle_ns}; + // TEST (profile_publishes): out of work from now until this block ends. + struct IdleMark { + std::atomic& since; + ~IdleMark() { + since.store(0, std::memory_order_relaxed); + } + } const idle_mark{back_idle_since}; + if (profile_publishes) { + back_idle_since.store(Nx2Profile::Ticks(), std::memory_order_relaxed); + } // The front usually sends more within microseconds: spin briefly before sleeping. for (u32 spins = 0; spins < 8192 && available == read_position && !hook_requested.load(std::memory_order_relaxed) && diff --git a/src/video_core/gpu_split.h b/src/video_core/gpu_split.h index dbe71825e5..0c85729da7 100644 --- a/src/video_core/gpu_split.h +++ b/src/video_core/gpu_split.h @@ -6,6 +6,7 @@ #include #include #include +#include #include #include #include @@ -201,7 +202,7 @@ class GpuSplit { Commit(2 + payload); NoteWrites(writes); } - void DeferFunction(std::function&& func); + void DeferFunction(std::function&& func, Writes writes = Writes::Anywhere); /// Runs `func` on the back and waits for it: for work on back-owned state, such as a buffer /// cache download that records into the Vulkan scheduler. A drained back still submits from /// its idle hook, so the front never touches that state itself. @@ -389,11 +390,21 @@ class GpuSplit { } u64* ReserveSlow(size_t words); void Commit(size_t words) { + if (profile_publishes && write_position == published_local) [[unlikely]] { + NoteFirstUnpublished(); // TEST + } write_position += words; if (write_position - published_local >= publish_every) { Publish(); } } + /// TEST (EDEN_NX2_GPU_PROFILE): how long the back waited for work while the front held words + /// it could have run (Publish). + /// Nx2Profile::Enabled() as the split started (MaybeStart, after VideoCore::CreateGPU read + /// the switch). + bool profile_publishes{}; + void NoteFirstUnpublished(); + void NoteLatePublish(); void WaitForSpace(size_t words); void SelectEngine(Tegra::Engines::EngineInterface* engine) { if (engine != front_engine) [[unlikely]] { @@ -452,6 +463,7 @@ class GpuSplit { std::array write_positions{}; u64 published_local{}; u64 consumed_cache{}; + u64 first_unpublished_ticks{}; ///< TEST: when the first word not published yet was sent /// Words between publishes: small right after the front waited for the back (Drain, /// PublishSoon), while the back may have nothing to do. size_t publish_every{PublishEvery}; @@ -481,6 +493,8 @@ class GpuSplit { /// The front sleeps in Backoff (#161: instead of yielding). alignas(128) std::atomic front_waiting{}; std::atomic front_wake{}; + /// TEST (profile_publishes): when the back last ran out of work, 0 while it runs. + alignas(128) std::atomic back_idle_since{}; std::jthread back_thread; }; diff --git a/src/video_core/nx2_drain_check.h b/src/video_core/nx2_drain_check.h index 7906c5bdd9..6555ac040c 100644 --- a/src/video_core/nx2_drain_check.h +++ b/src/video_core/nx2_drain_check.h @@ -8,9 +8,12 @@ #include #include #include +#include +#include #include #include #include +#include #include @@ -29,7 +32,9 @@ // - acquires and polls the semaphore view decides (Puller AskView), verified: the front drains // and reads the word as before anyway, and counts the decisions the drain contradicts. A pass // the drained word does not meet must stay 0; such a pass is corrected (the channel waits), and -// a wait the drained word meets ends (counted: a drain would have passed or resumed it). +// a wait the drained word meets ends (counted: a drain would have passed or resumed it); +// - the releases the front noted for the QMD launches it valued (FrontQmdView), against the ones +// the back then ran, compared on the back right after the QMD methods. Wrong ones must stay 0. namespace VideoCommon::Nx2DrainCheck { // A plain flag: the front checks it at every command list and acquire. @@ -54,6 +59,13 @@ struct State { /// the drained word met. std::array, 2> view_waits{}; std::array, 2> view_waits_met{}; + /// QMD methods the front valued and the releases compared for them, those that differed, + /// and methods it could not value (their releases are not compared). + std::atomic qmd_methods{}; + std::atomic qmd_releases{}; + std::atomic qmd_wrong{}; + std::atomic qmd_unvalued{}; + std::atomic qmd_logged{}; std::mutex released_mutex; std::unordered_set released; ///< Device addresses of the semaphores the GPU released std::chrono::steady_clock::time_point last_report{}; ///< The front's @@ -113,6 +125,71 @@ inline void NoteViewWait(bool poll, bool met_after_drain) { } } +/// A release of a QMD launch: the word and what it holds once the back ran it, or a value the +/// front could not tell (`known` false), or that value at least (`at_least`). +struct QmdRelease { + u64 device; + u32 value; + bool known; + bool at_least; + + bool operator==(const QmdRelease&) const = default; +}; + +/// The releases the back ran for QMD launches since the last check (the back's thread only). +inline std::vector& BackQmdReleases() { + static std::vector released; + return released; +} + +/// The split's back runs a release of a QMD launch (KeplerCompute::ProcessNativeMethod). +inline void NoteQmdRelease(u64 device_address, u32 value, bool on_back) { + if (on_back) { + BackQmdReleases().push_back({device_address, value, true, false}); + } +} + +/// The back, right after QMD methods the front sent: what it released for them against what the +/// front noted (`predicted`, in order), or nothing to compare where the front could not value +/// them (`valued` false). +inline void CheckQmdReleases(std::span predicted, bool valued) { + State& state = Get(); + auto& released = BackQmdReleases(); + if (!valued) { + state.qmd_unvalued.fetch_add(1, std::memory_order_relaxed); + released.clear(); + return; + } + state.qmd_methods.fetch_add(1, std::memory_order_relaxed); + state.qmd_releases.fetch_add(released.size(), std::memory_order_relaxed); + bool same = released.size() == predicted.size(); + for (size_t i = 0; same && i < released.size(); ++i) { + const QmdRelease& note = predicted[i]; + same = released[i].device == note.device && + (!note.known || (note.at_least ? released[i].value >= note.value + : released[i].value == note.value)); + } + if (!same) { + state.qmd_wrong.fetch_add(1, std::memory_order_relaxed); + if (state.qmd_logged.fetch_add(1, std::memory_order_relaxed) < 16) { + std::string text; + for (const QmdRelease& release : released) { + text += fmt::format(" {:#x}={:#x}", release.device, release.value); + } + text += " | noted"; + for (const QmdRelease& note : predicted) { + text += fmt::format(" {:#x}{}{:#x}", note.device, + !note.known ? "=?" + : note.at_least ? ">=" + : "=", + note.value); + } + LOG_ERROR(HW_GPU, "QMD view check: the back released{}", text); + } + } + released.clear(); +} + /// The front fetches a list without waiting for writers still queued after the last sync. inline void NoteListUnwaited() { Get().lists_unwaited.fetch_add(1, std::memory_order_relaxed); @@ -181,14 +258,20 @@ inline void MaybeReport() { const std::string acquire_waits = waits(false); const std::string poll_passes = passes(true); const std::string poll_waits = waits(true); + const u64 qmd_methods = take(state.qmd_methods); + const u64 qmd_releases = take(state.qmd_releases); + const u64 qmd_wrong = take(state.qmd_wrong); + const u64 qmd_unvalued = take(state.qmd_unvalued); LOG_INFO(HW_GPU, "Drain checks in {:.1f} s: command lists that did not wait for writers sent after the " "last sync {}, GPU-written by those writers {} | drained acquires at addresses the " "GPU never released {}, at ones it released {} | drained polls of suspended acquires " "at addresses never released {}, at released ones {} | semaphore view: acquires " - "passed {}, suspended {}; polls resumed {}, kept waiting {}", + "passed {}, suspended {}; polls resumed {}, kept waiting {} | QMD view: methods " + "valued {} with {} releases, released otherwise than noted {}; not valued {}", seconds, unwaited, written, acquires_never, acquires_released, polls_never, - polls_released, acquire_passes, acquire_waits, poll_passes, poll_waits); + polls_released, acquire_passes, acquire_waits, poll_passes, poll_waits, qmd_methods, + qmd_releases, qmd_wrong, qmd_unvalued); } } // namespace VideoCommon::Nx2DrainCheck @@ -224,6 +307,25 @@ enum class Releaser : u32 { Report3D, ///< 3D report semaphores ReportCompute, ///< Compute report semaphores Copy, ///< Copy engine semaphores + Qmd, ///< QMD launches the front valued (FrontQmdView) + Count, +}; + +/// Why the front's QMD view could not value a signal (FrontQmdView::Signal). +enum class QmdLost : u32 { + NotSynced, ///< Its table is not the back's yet (the back has not caught up since) + Unplaced, ///< Work that may write guest memory anywhere (or map) was still queued + GpuWritten, ///< A QMD it read is on a page GPU work wrote + Pending, ///< A write into a QMD it read was still queued + ReleasedInto, ///< A release of the same signal writes into a QMD it read + Count, +}; + +/// What the QMD view noted for a release. +enum class QmdNote : u32 { + Exact, + AtLeast, ///< A maximum with a word it can't tell + Unknown, ///< A reduction of a word it can't tell Count, }; @@ -252,6 +354,12 @@ struct State { u64 wait_ns{}; u64 wait_max_ns{}; u64 waits_over_1ms{}; + u64 passed_by_qmd{}; ///< Of the passes by a queued release, at words QMD launches release + u64 qmd_valued{}; + std::array(QmdLost::Count)> qmd_lost{}; + std::array(QmdNote::Count)> qmd_notes{}; + u64 qmd_syncs{}; + std::chrono::steady_clock::duration qmd_sync_time{}; std::unordered_map noted; ///< Releaser bits by word std::chrono::steady_clock::time_point last_report{}; }; @@ -283,6 +391,35 @@ inline void NoteRelease(u64 device_address, Releaser releaser) { } } +/// The QMD view valued a signal (`lost` empty) or not. +inline void NoteQmdSignal(std::optional lost) { + if (!g_enabled) { + return; + } + State& state = Get(); + if (lost) { + ++state.qmd_lost[static_cast(*lost)]; + } else { + ++state.qmd_valued; + } +} + +/// The QMD view noted a release so. +inline void NoteQmdNote(QmdNote note) { + if (g_enabled) { + ++Get().qmd_notes[static_cast(note)]; + } +} + +/// The QMD view took the back's table over, in `time`. +inline void NoteQmdSync(std::chrono::steady_clock::duration time) { + if (g_enabled) { + State& state = Get(); + ++state.qmd_syncs; + state.qmd_sync_time += time; + } +} + /// An acquire of `operation` (Puller GpuSemaphoreOperation) at `device_address`, decided so. /// Suspended: whether its word is on a GPU-written page, and the buffer cache held it newer. inline void NoteAcquire(u32 operation, Decision decision, u64 device_address, bool waited, @@ -292,6 +429,12 @@ inline void NoteAcquire(u32 operation, Decision decision, u64 device_address, bo if (waited) { ++state.positioned_waits; } + if (decision == Decision::Queued) { + const auto it = state.noted.find(device_address & ~u64{3}); + if (it != state.noted.end() && ((it->second >> static_cast(Releaser::Qmd)) & 1)) { + ++state.passed_by_qmd; + } + } if (decision != Decision::Suspended) { return; } @@ -365,18 +508,30 @@ inline void MaybeReport() { const u64 resumed = resumes[0] + resumes[1] + resumes[2] + resumes[3]; LOG_INFO(HW_GPU, "Acquires in {:.1f} s (EDEN_NX2_ACQUIRE_STATS): equal {}, greater or equal {}, mask " - "{}; positioned waits {} | suspended at words on GPU-written pages {} (held newer by " - "the buffer cache {}), with releases sent by the puller {}, 3D reports {}, compute " - "reports {}, the copy engine {}, none (QMD releases, the CPU) {} | resumed {}: " + "{}; positioned waits {}; passed by a queued QMD launch's release {} | suspended at " + "words on GPU-written pages {} (held newer by the buffer cache {}), with releases " + "sent by the puller {}, 3D reports {}, compute reports {}, the copy engine {}, QMD " + "launches the front valued {}, none (other QMD releases, the CPU) {} | resumed {}: " "latched {} (payload releases {}, query results {}, QMD reductions {}, QMD maxima {}, " "other {}), by memory {}, by a queued release {}, by the drain check {}, after a " "flush {} | waited {:.1f} ms, at most {:.3f} ms, over 1 ms {}", seconds, by_operation(0), by_operation(1), by_operation(2), state.positioned_waits, - state.suspended_gpu_written, state.suspended_flush, noted[0], noted[1], noted[2], - noted[3], state.suspended_never_noted, resumed, resumes[0], latches[1], latches[2], - latches[3], latches[4], latches[0], resumes[1], resumes[2], resumes[3], - state.resumes_after_flush, static_cast(state.wait_ns) / 1e6, - static_cast(state.wait_max_ns) / 1e6, state.waits_over_1ms); + state.passed_by_qmd, state.suspended_gpu_written, state.suspended_flush, noted[0], + noted[1], noted[2], noted[3], noted[4], state.suspended_never_noted, resumed, + resumes[0], latches[1], latches[2], latches[3], latches[4], latches[0], resumes[1], + resumes[2], resumes[3], state.resumes_after_flush, + static_cast(state.wait_ns) / 1e6, static_cast(state.wait_max_ns) / 1e6, + state.waits_over_1ms); + const auto& lost = state.qmd_lost; + const auto& notes = state.qmd_notes; + LOG_INFO(HW_GPU, + "QMD view in {:.1f} s (EDEN_NX2_ACQUIRE_STATS): signals valued {}; not valued: table " + "not synced yet {}, work that may write anywhere queued {}, QMD on a GPU-written page " + "{}, a write into the QMD queued {}, a release of the signal into it {} | releases " + "noted exact {}, at least {}, unknown {} | tables taken over {} in {:.3f} ms", + seconds, state.qmd_valued, lost[0], lost[1], lost[2], lost[3], lost[4], notes[0], + notes[1], notes[2], state.qmd_syncs, + std::chrono::duration(state.qmd_sync_time).count()); auto noted_words = std::move(state.noted); state = State{}; state.noted = std::move(noted_words); diff --git a/src/video_core/nx2_profile.h b/src/video_core/nx2_profile.h index 58ed087c79..8f6cd2e3bd 100644 --- a/src/video_core/nx2_profile.h +++ b/src/video_core/nx2_profile.h @@ -63,6 +63,7 @@ enum class Zone : u32 { ListSubmitWait, ListGpuWait, AcquirePoll, ///< TEST: a suspended acquire's poll (CanResumeAcquire in ResumePending) + Resnapshot, ///< TEST: command lists copied again at a resume (DmaPusher::RefreshSnapshot) Count, }; @@ -74,7 +75,7 @@ inline constexpr std::array(Zone::Count)> zone_ "descriptor acquire", "bind stage buffers", "push descriptors", "render targets", "configure draw", "dynamic state", "draw record", "texture upload", "fence", "fence skipped", "method", "split drain", "split wait", - "writer wait", "list submit wait", "list GPU wait", "acquire poll", + "writer wait", "list submit wait", "list GPU wait", "acquire poll", "resnapshot", }; // A plain flag: Enabled() runs several times per method write (no guard to check). @@ -110,7 +111,7 @@ inline u64 Ticks() { struct ThreadData { std::array(Zone::Count)> exclusive_ns{}; std::array(Zone::Count)> count{}; - std::array counters{}; ///< See Counter + std::array counters{}; ///< See Counter std::array(DrainSite::Count)> drain_ns{}; std::array(DrainSite::Count)> drain_count{}; u64 child_ns{}; ///< Time of the zones nested in the current one @@ -164,6 +165,11 @@ enum class Counter : u32 { SameValueWrites, ///< Of MethodWrites: the register already held the value AcquireChecks, ///< Semaphore acquires evaluated AcquireWaits, ///< Of AcquireChecks: not met yet (the channel waits) + /// TEST: the split's front publishes words while its back waits for work (GpuSplit::Publish): + /// the back's wait since it ran out or since the first of them was sent (Ticks), and count + LatePublishTicks, + LatePublishes, + ResnapshotWords, ///< TEST: command list words copied again at resumes }; /// Method calls on this thread by method (3D registers on the back, macros on the front), and @@ -283,6 +289,14 @@ inline void Report(const char* thread_name = "GPU thread") { line += fmt::format(" | acquires {:.0f} ({:.0f} waited)", static_cast(data.counters[6]) * 1e9 / window, static_cast(data.counters[7]) * 1e9 / window); + if (data.counters[9] != 0 || data.counters[10] != 0) { + // TEST: the back idle while words it could run waited on the front. + line += fmt::format(" | publishes to an idle back late by {:.1f} ms ({:.0f}/s) | " + "resnapshot words {:.0f}/s", + static_cast(data.counters[8]) * 1000.0 / window_ticks, + static_cast(data.counters[9]) * 1e9 / window, + static_cast(data.counters[10]) * 1e9 / window); + } std::string drains; for (size_t i = 0; i < data.drain_count.size(); ++i) { if (data.drain_count[i] != 0) { From 4edd776ba45fda3c4463e41b871f883866101213 Mon Sep 17 00:00:00 2001 From: jpkto <255363096+jpkto@users.noreply.github.com> Date: Sun, 8 Nov 2026 03:33:18 +0100 Subject: [PATCH 034/105] TEST: classify uploads only with EDEN_NX2_GPU_COST=2 The test48b GPU cost run fell from 59 to 29 fps on the split GPU thread's back. The upload classification caused it, not the timestamps: Buffer::MarkUsage noted a tick for every 4 KiB page of each new range at each tick, and the largest buffer is 224 MB. "bind stage buffers" took 4.8 us a call instead of 0.43. EDEN_NX2_GPU_COST=1 now times the GPU only. =2 also keeps the page ticks and logs the "Uploads" line, as =1 did before. Co-Authored-By: Claude Opus 5.5 --- src/video_core/renderer_vulkan/vk_buffer_cache.cpp | 4 ++-- src/video_core/renderer_vulkan/vk_gpu_cost.cpp | 5 +++++ src/video_core/renderer_vulkan/vk_gpu_cost.h | 5 +++++ 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/src/video_core/renderer_vulkan/vk_buffer_cache.cpp b/src/video_core/renderer_vulkan/vk_buffer_cache.cpp index 70d69e797c..bb8417a3dd 100644 --- a/src/video_core/renderer_vulkan/vk_buffer_cache.cpp +++ b/src/video_core/renderer_vulkan/vk_buffer_cache.cpp @@ -34,8 +34,8 @@ namespace Vulkan { namespace { -/// TEST (EDEN_NX2_GPU_COST): Buffer::MarkUsage notes page ticks, CanReorderUpload classifies. -const bool g_trace_uploads = GpuCost::Enabled(); +/// TEST (EDEN_NX2_GPU_COST=2): Buffer::MarkUsage notes page ticks, CanReorderUpload classifies. +const bool g_trace_uploads = GpuCost::TracesUploads(); void HostReadBarrier(vk::CommandBuffer cmdbuf) { const VkMemoryBarrier barrier{ diff --git a/src/video_core/renderer_vulkan/vk_gpu_cost.cpp b/src/video_core/renderer_vulkan/vk_gpu_cost.cpp index aa9a8cebe3..95d72f7a4a 100644 --- a/src/video_core/renderer_vulkan/vk_gpu_cost.cpp +++ b/src/video_core/renderer_vulkan/vk_gpu_cost.cpp @@ -908,6 +908,11 @@ bool GpuCost::Enabled() { return value != nullptr && *value != '\0' && *value != '0'; } +bool GpuCost::TracesUploads() { + const char* const value = std::getenv("EDEN_NX2_GPU_COST"); + return value != nullptr && std::atoi(value) >= 2; +} + std::unique_ptr GpuCost::Create(const Device& device, u32 timestamp_bits, float timestamp_period, bool frames_sampled) { if (!Enabled()) { diff --git a/src/video_core/renderer_vulkan/vk_gpu_cost.h b/src/video_core/renderer_vulkan/vk_gpu_cost.h index 3453e31095..e2effc1c1a 100644 --- a/src/video_core/renderer_vulkan/vk_gpu_cost.h +++ b/src/video_core/renderer_vulkan/vk_gpu_cost.h @@ -28,6 +28,7 @@ // This only reads: the commands, their order and their synchronization are unchanged. Logged // every 10 s over the frames sampled in it, per frame: "GPU cost" (totals by kind), "GPU cost // by site", and one "Render pass breaks" line for each of the lines that ended the most passes. +// EDEN_NX2_GPU_COST=2 also logs "Uploads": where each upload's pages were last used. namespace Vulkan { class Device; @@ -38,6 +39,10 @@ class GpuCost { /// Whether EDEN_NX2_GPU_COST is set. [[nodiscard]] static bool Enabled(); + /// EDEN_NX2_GPU_COST=2: also classify the uploads that run in order (vk_buffer_cache.cpp). + /// Its page ticks cost the GPU thread several microseconds per bind of a large buffer. + [[nodiscard]] static bool TracesUploads(); + /// Null unless EDEN_NX2_GPU_COST is set, frames are sampled (EDEN_DKB_FRAME_TRACE's /// timestamp pools exist) and the query pool can be created. static std::unique_ptr Create(const Device& device, u32 timestamp_bits, From d612710df302e146f9dd4d8ee2be78ab8a6f36e5 Mon Sep 17 00:00:00 2001 From: jpkto <255363096+jpkto@users.noreply.github.com> Date: Sun, 8 Nov 2026 03:40:35 +0100 Subject: [PATCH 035/105] vulkan: one command and one pair of barriers for a run of in-order uploads An upload that can't go to the upload command buffer is recorded in order: a barrier, the copy, a barrier (BufferCacheRuntime::CopyBuffer). Ocarina of Time makes about 600 of them a frame, nearly all inline writes into GPU-written pages (InlineMemoryImplementation, ~200 bytes each). The GPU cost trace counted 614 copies and 1,212 barriers a frame there. They come outside render passes: only 2.2 a frame end one. So they come in runs, for instance between a wait for idle and the next pass. Scheduler::RecordUpload keeps such uploads from staging memory (the stream buffer) pending in an UploadBatch. They are recorded as one command: the same barrier before, all their copies, and the barrier after one, which here also reaches transfers. The batch is recorded before any other command, before the chunk goes to the worker, and before a submission takes its tick. These record the batch first instead of joining it: - an upload that writes bytes a pending one writes; - one that reads a buffer a pending one writes, or writes one it reads; - one that would take the batch past 64 copies. That bounds the overlap check each upload makes on the recording thread. Why nothing loses its order: - The batch holds only uploads recorded with no other command between them. So the barrier before it covers all earlier work for each copy, as the barrier before each upload did. - One by one, the barrier after an upload ordered its copy ahead of later graphics and compute work. The barrier before the next upload ordered it ahead of every later transfer, in later submissions too. The barrier after the batch does both for all its copies, so it names the transfer stage as well. - Nothing orders two copies of one batch against each other. They don't need it: no two write the same bytes, and none writes a buffer another reads. - Each upload still lands in the tick it was recorded in, so staging regions and usage ticks stay right. - Other copies are unchanged: downloads, buffer-to-buffer copies, uploads from dedicated staging buffers, and copies without barriers. It also closes a gap for these uploads. The last upload of a run was ordered ahead of no later transfer, such as a reordered upload of the same bytes in the next submission's upload command buffer once TickFrame has cleared the usage bits. The barrier after the batch orders it. The recording itself (UploadBatch::Record) is what the scheduler runs. A lavapipe harness runs it under synchronization validation: - 400 small uploads, some adjacent, some rewriting bytes, over two buffers, between earlier and later transfer work. Batched and one by one give the same bytes, which match a CPU model, and no hazard is reported. So do 200 uploads that never overlap, split into batches of 64. - A transfer with no barrier in a later submission, writing over the uploads: no hazard after a batch. It was already ordered after every upload but the last when they were recorded one by one. - Controls reported as hazards: - a batch with two copies writing the same bytes (WRITE_AFTER_WRITE); - that later transfer over the last upload recorded one by one; - that later transfer after a batch closed with the old barrier. - The layer (1.3.275) takes ALL_GRAPHICS for every stage of a graphics queue, transfers included, which the specification's ALL_GRAPHICS does not include. So these cases name the graphics stages of the old barriers one by one. Co-Authored-By: Claude Opus 5.5 --- src/video_core/CMakeLists.txt | 1 + src/video_core/renderer_vulkan/upload_batch.h | 110 ++++++++++++++++++ .../renderer_vulkan/vk_buffer_cache.cpp | 8 ++ .../renderer_vulkan/vk_scheduler.cpp | 42 ++++++- src/video_core/renderer_vulkan/vk_scheduler.h | 19 +++ 5 files changed, 179 insertions(+), 1 deletion(-) create mode 100644 src/video_core/renderer_vulkan/upload_batch.h diff --git a/src/video_core/CMakeLists.txt b/src/video_core/CMakeLists.txt index 79394084ea..84c624d4f6 100644 --- a/src/video_core/CMakeLists.txt +++ b/src/video_core/CMakeLists.txt @@ -228,6 +228,7 @@ add_library(video_core STATIC renderer_vulkan/vk_rasterizer.cpp renderer_vulkan/vk_rasterizer.h renderer_vulkan/submission_policy.h + renderer_vulkan/upload_batch.h renderer_vulkan/vk_render_pass_cache.cpp renderer_vulkan/vk_render_pass_cache.h renderer_vulkan/vk_resource_pool.cpp diff --git a/src/video_core/renderer_vulkan/upload_batch.h b/src/video_core/renderer_vulkan/upload_batch.h new file mode 100644 index 0000000000..7c360c6b6a --- /dev/null +++ b/src/video_core/renderer_vulkan/upload_batch.h @@ -0,0 +1,110 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later + +#pragma once + +#include +#include + +#include "common/common_types.h" +#include "video_core/vulkan_common/vulkan_wrapper.h" + +namespace Vulkan { + +/// Uploads in order (Scheduler::RecordUpload) waiting to be recorded as one command: their +/// copies run between one barrier before and one after. Copies of one batch may run in any +/// order and at once, so a batch never holds two copies that write the same bytes, or one that +/// writes a buffer another reads. +class UploadBatch { +public: + /// Copies with the same source and destination, consecutive in `copies`. + struct Group { + VkBuffer src; + VkBuffer dst; + u32 first; + u32 count; + }; + + /// More copies than this go in a later batch. It bounds CanAdd's checks on the recording + /// thread, at the cost of two barriers per this many uploads. + static constexpr size_t MaxCopies = 64; + + [[nodiscard]] bool Empty() const noexcept { + return copies.empty(); + } + + /// Whether these copies can join the batch: it has room, they read no buffer a copy in it + /// writes and write none it reads, and they write no byte of dst that a copy in it writes. + /// Otherwise the batch is recorded first. + [[nodiscard]] bool CanAdd(VkBuffer src, VkBuffer dst, + std::span new_copies) const { + if (copies.size() + new_copies.size() > MaxCopies) { + return false; + } + for (const Group& group : groups) { + if (src == group.dst || dst == group.src) { + return false; + } + if (group.dst != dst) { + continue; + } + for (u32 index = group.first; index < group.first + group.count; ++index) { + const VkBufferCopy& pending = copies[index]; + for (const VkBufferCopy& copy : new_copies) { + if (copy.dstOffset < pending.dstOffset + pending.size && + pending.dstOffset < copy.dstOffset + copy.size) { + return false; + } + } + } + } + return true; + } + + /// Records a batch: the barrier before an in-order buffer copy (BufferCacheRuntime:: + /// CopyBuffer's), the copies, and the barrier after one, made to reach transfers as well. + /// Recorded one by one, the barriers between two uploads ordered the copies before them + /// ahead of the copies after them, which CanAdd makes needless, and ahead of all later work: + /// graphics and compute through the barrier after a copy, transfers (in later submissions + /// too) through the barrier before the next one. The barrier after the batch does both. + static void Record(vk::CommandBuffer cmdbuf, std::span batch_copies, + std::span batch_groups) { + static constexpr VkMemoryBarrier READ_BARRIER{ + .sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER, + .pNext = nullptr, + .srcAccessMask = VK_ACCESS_MEMORY_WRITE_BIT, + .dstAccessMask = VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT, + }; + static constexpr VkMemoryBarrier WRITE_BARRIER{ + .sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER, + .pNext = nullptr, + .srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT, + .dstAccessMask = VK_ACCESS_MEMORY_READ_BIT | VK_ACCESS_MEMORY_WRITE_BIT, + }; + cmdbuf.PipelineBarrier(vk::PIPELINE_STAGE_GRAPHICS_COMPUTE_TRANSFER, + VK_PIPELINE_STAGE_TRANSFER_BIT, 0, READ_BARRIER); + for (const Group& group : batch_groups) { + cmdbuf.CopyBuffer(group.src, group.dst, + vk::Span(batch_copies.data() + group.first, + group.count)); + } + cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_TRANSFER_BIT, + vk::PIPELINE_STAGE_GRAPHICS_COMPUTE_TRANSFER, 0, WRITE_BARRIER); + } + + void Add(VkBuffer src, VkBuffer dst, std::span new_copies) { + const u32 first = static_cast(copies.size()); + const u32 count = static_cast(new_copies.size()); + copies.insert(copies.end(), new_copies.begin(), new_copies.end()); + if (!groups.empty() && groups.back().src == src && groups.back().dst == dst) { + groups.back().count += count; + } else { + groups.push_back({src, dst, first, count}); + } + } + + std::vector copies; + std::vector groups; +}; + +} // namespace Vulkan diff --git a/src/video_core/renderer_vulkan/vk_buffer_cache.cpp b/src/video_core/renderer_vulkan/vk_buffer_cache.cpp index bb8417a3dd..d513334c33 100644 --- a/src/video_core/renderer_vulkan/vk_buffer_cache.cpp +++ b/src/video_core/renderer_vulkan/vk_buffer_cache.cpp @@ -712,6 +712,14 @@ void BufferCacheRuntime::CopyBuffer(VkBuffer dst_buffer, VkBuffer src_buffer, } scheduler.RequestOutsideRenderPassOperationContext(location); + if (barrier && src_buffer == staging_pool.StreamBuf() && !vk_copies.empty()) { + // An upload in order, with these barriers. Runs of them (Ocarina of Time's inline writes + // into GPU-written pages: ~600 a frame) go in one command between one pair of them. + scheduler.RecordUpload(src_buffer, dst_buffer, + std::span(vk_copies.data(), vk_copies.size()), + location); + return; + } scheduler.Record( [src_buffer, dst_buffer, vk_copies, barrier](vk::CommandBuffer cmdbuf) { if (barrier) { diff --git a/src/video_core/renderer_vulkan/vk_scheduler.cpp b/src/video_core/renderer_vulkan/vk_scheduler.cpp index 0a20d223e0..3eb113f203 100644 --- a/src/video_core/renderer_vulkan/vk_scheduler.cpp +++ b/src/video_core/renderer_vulkan/vk_scheduler.cpp @@ -38,6 +38,7 @@ #include "video_core/renderer_vulkan/vk_scheduler.h" #include "video_core/renderer_vulkan/vk_state_tracker.h" #include "video_core/renderer_vulkan/vk_texture_cache.h" +#include "video_core/renderer_vulkan/upload_batch.h" #include "video_core/vulkan_common/vulkan_device.h" #include "video_core/vulkan_common/vulkan_wrapper.h" @@ -122,7 +123,8 @@ void Scheduler::CommandChunk::ExecuteAll(vk::CommandBuffer cmdbuf, Scheduler::Scheduler(const Device& device_, StateTracker& state_tracker_) : device{device_}, state_tracker{state_tracker_}, master_semaphore{std::make_unique(device)}, - command_pool{std::make_unique(*master_semaphore, device)} { + command_pool{std::make_unique(*master_semaphore, device)}, + upload_batch{std::make_unique()} { if (Common::DkbFrameTrace::Enabled()) { const auto physical = device.GetPhysical(); @@ -523,6 +525,10 @@ void Scheduler::DispatchWork() { if (VideoCommon::GpuSplit::check_owner) [[unlikely]] { VideoCommon::GpuSplit::CheckOwner("Scheduler::DispatchWork"); } + if (uploads_pending) { + // What the worker gets is everything recorded so far. + FlushUploads(); + } if (chunk && !chunk->Empty()) { // Waking the worker is a system call (~2% of the GPU thread at one dispatch per 8 // draws): skip it while the worker is still busy, it pops the queue before waiting. @@ -539,6 +545,36 @@ void Scheduler::DispatchWork() { } } +void Scheduler::RecordUpload(VkBuffer src, VkBuffer dst, std::span copies, + std::source_location location) { + if (VideoCommon::GpuSplit::check_owner) [[unlikely]] { + VideoCommon::GpuSplit::CheckOwner("Scheduler::RecordUpload"); + } + if (copies.empty()) { + return; + } + if (uploads_pending && !upload_batch->CanAdd(src, dst, copies)) { + FlushUploads(); + } + if (!uploads_pending) { + upload_batch_site = location; + } + upload_batch->Add(src, dst, copies); + uploads_pending = true; +} + +void Scheduler::FlushUploads() { + // Copies, so the batch's vectors keep their capacity for the next one. + auto command = [copies = upload_batch->copies, + groups = upload_batch->groups](vk::CommandBuffer cmdbuf, vk::CommandBuffer) { + UploadBatch::Record(cmdbuf, copies, groups); + }; + upload_batch->copies.clear(); + upload_batch->groups.clear(); + uploads_pending = false; + RecordAs(std::move(command), upload_batch_site, RecordSite::Own, 0); +} + void Scheduler::BeginRenderPassImpl(const Framebuffer* framebuffer, VkRenderPass renderpass, const VkClearValue* clear_values, u32 clear_value_count) { if (VideoCommon::Nx2Profile::Enabled()) { @@ -1042,6 +1078,10 @@ void Scheduler::TraceGpuSubmission() { u64 Scheduler::SubmitExecution(VkSemaphore signal_semaphore, VkSemaphore wait_semaphore, std::source_location cause) { + if (uploads_pending) { + // In the tick they were recorded at. + FlushUploads(); + } if (VideoCommon::Nx2Profile::Enabled()) { g_profile_submits.fetch_add(1, std::memory_order_relaxed); ReportSubmitCounts(); diff --git a/src/video_core/renderer_vulkan/vk_scheduler.h b/src/video_core/renderer_vulkan/vk_scheduler.h index 4d808238a8..8bf1acb194 100644 --- a/src/video_core/renderer_vulkan/vk_scheduler.h +++ b/src/video_core/renderer_vulkan/vk_scheduler.h @@ -43,6 +43,7 @@ class Framebuffer; class GpuCost; class GraphicsPipeline; class StateTracker; +class UploadBatch; struct QueryCacheParams; @@ -210,6 +211,13 @@ class Scheduler { on_submit = std::move(func); } + /// An upload in order: copies into dst after the work recorded before them and before the + /// work recorded after them (the barriers of an in-order buffer copy). Uploads with no other + /// command between them share one command and one pair of barriers, unless one would write + /// bytes another writes, or a buffer another reads. Call outside a render pass. + void RecordUpload(VkBuffer src, VkBuffer dst, std::span copies, + std::source_location location = std::source_location::current()); + /// Send work to a separate thread. template requires std::is_invocable_v @@ -437,6 +445,10 @@ class Scheduler { if (VideoCommon::GpuSplit::check_owner) [[unlikely]] { VideoCommon::GpuSplit::CheckOwner("Scheduler::Record"); } + if (uploads_pending) { + // Uploads recorded before this command go before it. + FlushUploads(); + } if (!chunk->Record(command)) { DispatchWork(); (void)chunk->Record(command); @@ -557,6 +569,13 @@ class Scheduler { /// TEST: EDEN_NX2_GPU_COST. record_sites: every command notes its RecordSite. std::unique_ptr gpu_cost; bool record_sites{}; + + /// RecordUpload's copies not recorded yet; uploads_pending while it holds any. + std::unique_ptr upload_batch; + std::source_location upload_batch_site{}; + bool uploads_pending{}; + /// Records the pending uploads as one command. + void FlushUploads(); /// The line that ended the last render pass: the site of the next pass's begin. std::source_location last_pass_end{}; From 1457bd01fb4cb670380025c780aaff4958af5d60 Mon Sep 17 00:00:00 2001 From: jpkto <255363096+jpkto@users.noreply.github.com> Date: Sun, 8 Nov 2026 03:43:18 +0100 Subject: [PATCH 036/105] vulkan: keep a deferred clear's render pass for the draws after it A deferred clear becomes the next render pass. It is begun with a variant of the framebuffer's render pass that clears instead of loading (Framebuffer::RenderPassVariant). The first draw runs in it. The next draw's RequestRenderpass compared the render pass handles and found the framebuffer's own pass, not the variant. So it ended the pass and began another on the same framebuffer: the attachments were stored only to be loaded back. The GPU cost trace counts 10.9 such ends a frame at GraphicsPipeline::ConfigureDraw in Ocarina of Time, nothing recorded between them and 79% resuming the same framebuffer. RequestRenderpass now keeps the pass when it is such a variant on the same framebuffer and render area. Why it's the same: - The variant is made from the framebuffer's own RenderPassKey with only its clear masks set. Without discard masks it differs only in load ops: same attachments, formats, samples, store ops, layouts and resolve attachments. - Ending it and beginning the framebuffer's pass stored every attachment and loaded it back. Staying in the pass keeps the same contents, and the pass ends with the same store ops. - Within one pass, rasterization order puts each draw's attachment accesses after the earlier draws'. The fragment barrier and feedback loop checks, which end a pass when a draw reads what an earlier one wrote, run before RequestRenderpass as before. - Pipelines stay compatible: render pass compatibility ignores load and store ops, and the first draw already ran in the variant. - Queries and conditional rendering continue in the pass instead of closing and reopening around a break. - Variants that discard (MSAA resolve on tilers) store differently and keep the old behavior. Tested on lavapipe under synchronization validation: - a framebuffer made with its own render pass, a clear variant, and two attachment writes; - the old end-and-reload sequence and the kept pass give the same pixels; - validation reports nothing for either. Co-Authored-By: Claude Opus 5.5 --- src/video_core/renderer_vulkan/vk_scheduler.cpp | 13 +++++++++++-- src/video_core/renderer_vulkan/vk_scheduler.h | 3 +++ 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/src/video_core/renderer_vulkan/vk_scheduler.cpp b/src/video_core/renderer_vulkan/vk_scheduler.cpp index 3eb113f203..5b7cd32f8e 100644 --- a/src/video_core/renderer_vulkan/vk_scheduler.cpp +++ b/src/video_core/renderer_vulkan/vk_scheduler.cpp @@ -584,6 +584,7 @@ void Scheduler::BeginRenderPassImpl(const Framebuffer* framebuffer, VkRenderPass const VkFramebuffer framebuffer_handle = framebuffer->Handle(); const VkExtent2D render_area = framebuffer->RenderArea(); state.renderpass = renderpass; + state.clear_variant = false; state.framebuffer = framebuffer_handle; state.render_area = render_area; @@ -661,6 +662,9 @@ void Scheduler::RealizeDeferredClear() { dc.color_clear_mask, dc.depth_stencil, color_discard_mask, depth_stencil_discard); EndRenderPass(); BeginRenderPassImpl(dc.framebuffer, renderpass, clear_values.data(), count); + // Without discards, the variant only clears instead of loading: its store ops, layouts and + // attachments are the framebuffer's own. + state.clear_variant = color_discard_mask == 0 && !depth_stencil_discard; } bool Scheduler::DeferColorClear(const Framebuffer* framebuffer, u32 rt_slot, @@ -708,8 +712,12 @@ void Scheduler::RequestRenderpass(const Framebuffer* framebuffer, std::source_lo const VkRenderPass renderpass = framebuffer->RenderPass(); const VkFramebuffer framebuffer_handle = framebuffer->Handle(); const VkExtent2D render_area = framebuffer->RenderArea(); - if (renderpass == state.renderpass && framebuffer_handle == state.framebuffer && - render_area.width == state.render_area.width && + // A deferred clear's pass serves the draws after its first one too: ending it there and + // beginning the framebuffer's own pass stored the attachments only to load them back (about + // 9 render passes a frame in Ocarina of Time). + if (state.renderpass != VK_NULL_HANDLE && + (renderpass == state.renderpass || state.clear_variant) && + framebuffer_handle == state.framebuffer && render_area.width == state.render_area.width && render_area.height == state.render_area.height) { return; } @@ -1272,6 +1280,7 @@ void Scheduler::EndRenderPass(std::source_location cause) }, cause, RecordSite::PassEnd, site_flags); state.renderpass = VkRenderPass{}; + state.clear_variant = false; num_renderpass_images = 0; } diff --git a/src/video_core/renderer_vulkan/vk_scheduler.h b/src/video_core/renderer_vulkan/vk_scheduler.h index 8bf1acb194..caaf61f3b7 100644 --- a/src/video_core/renderer_vulkan/vk_scheduler.h +++ b/src/video_core/renderer_vulkan/vk_scheduler.h @@ -463,6 +463,9 @@ class Scheduler { VkRenderPass renderpass{}; VkFramebuffer framebuffer{}; VkExtent2D render_area = {0, 0}; + /// The pass is a deferred clear's variant of the framebuffer's render pass that differs + /// from it only in load ops, so later draws on that framebuffer can stay in it. + bool clear_variant = false; GraphicsPipeline* graphics_pipeline = nullptr; bool is_rescaling = false; bool rescaling_defined = false; From b62f49104d26c6f7f9bc666a467732c005c2a05e Mon Sep 17 00:00:00 2001 From: jpkto <255363096+jpkto@users.noreply.github.com> Date: Sun, 8 Nov 2026 03:52:50 +0100 Subject: [PATCH 037/105] vulkan: record a dispatch's barrier in the dispatch's own command Every compute dispatch recorded its read barrier as a command of its own, right before the dispatch's: Ocarina of Time's 147 dispatches a frame made 147 more commands for the back thread to record and the worker to run. The barrier now opens the dispatch's command. The Vulkan commands and their order stay the same: - The only command recorded between the two was TraceComputePipeline's, and only when profiling. It counts on the CPU and touches no command buffer. - The dispatch's command records the barrier before anything else, so it still runs when the pipeline isn't bound or DLSS replaced the dispatch. - When DLSS may replace the dispatch, its evaluation is recorded between the barrier and the dispatch. That case keeps the separate barrier. - NoteComputeWrites is a flag for the next render pass, and no pass begins between it and the dispatch. Co-Authored-By: Claude Opus 5.5 --- .../renderer_vulkan/vk_rasterizer.cpp | 46 +++++++++++-------- 1 file changed, 26 insertions(+), 20 deletions(-) diff --git a/src/video_core/renderer_vulkan/vk_rasterizer.cpp b/src/video_core/renderer_vulkan/vk_rasterizer.cpp index ac669b3169..467161c2b8 100644 --- a/src/video_core/renderer_vulkan/vk_rasterizer.cpp +++ b/src/video_core/renderer_vulkan/vk_rasterizer.cpp @@ -262,6 +262,20 @@ bool IsPrimitiveRestartSupported(const Device& device, VkPrimitiveTopology topol (topology == VK_PRIMITIVE_TOPOLOGY_PATCH_LIST && device.IsPatchListPrimitiveRestartSupported())); } + +/// Before a dispatch: earlier writes visible to it, its indirect arguments included. +void RecordDispatchBarrier(vk::CommandBuffer cmdbuf) { + static constexpr VkMemoryBarrier READ_BARRIER{ + .sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER, + .pNext = nullptr, + .srcAccessMask = VK_ACCESS_MEMORY_WRITE_BIT, + .dstAccessMask = VK_ACCESS_MEMORY_READ_BIT, + }; + cmdbuf.PipelineBarrier(vk::PIPELINE_STAGE_GRAPHICS_COMPUTE_TRANSFER, + VK_PIPELINE_STAGE_DRAW_INDIRECT_BIT | + VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, + 0, READ_BARRIER); +} } // Anonymous namespace void RasterizerVulkan::ReadMeshEmulationProbe() { @@ -1805,23 +1819,6 @@ void RasterizerVulkan::DispatchCompute() { } const auto& qmd{kepler_compute->launch_description}; - // Earlier writes must be visible to the dispatch, its indirect arguments included, and what it - // writes to the next render pass. - const auto record_read_barrier = [this] { - static constexpr VkMemoryBarrier READ_BARRIER{ - .sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER, - .pNext = nullptr, - .srcAccessMask = VK_ACCESS_MEMORY_WRITE_BIT, - .dstAccessMask = VK_ACCESS_MEMORY_READ_BIT, - }; - scheduler.Record([](vk::CommandBuffer cmdbuf) { - cmdbuf.PipelineBarrier(vk::PIPELINE_STAGE_GRAPHICS_COMPUTE_TRANSFER, - VK_PIPELINE_STAGE_DRAW_INDIRECT_BIT | - VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, - 0, READ_BARRIER); - }); - scheduler.NoteComputeWrites(); - }; if (frame_dump.CapturingBuffers()) { std::optional> indirect_grid; if (indirect_address) { @@ -1841,10 +1838,11 @@ void RasterizerVulkan::DispatchCompute() { const auto [buffer, offset] = buffer_cache.ObtainBuffer(*indirect_address, 12, sync_info, post_op); scheduler.RequestOutsideRenderPassOperationContext(); - record_read_barrier(); + scheduler.NoteComputeWrites(); scheduler.TraceComputePipeline(pipeline->ShaderHash()); scheduler.Record([this, pipeline, indirect_buffer = buffer->Handle(), indirect_offset = offset](vk::CommandBuffer cmdbuf) { + RecordDispatchBarrier(cmdbuf); if (!pipeline->IsBound()) { return; } @@ -1862,7 +1860,11 @@ void RasterizerVulkan::DispatchCompute() { return; } scheduler.RequestOutsideRenderPassOperationContext(); - record_read_barrier(); + scheduler.NoteComputeWrites(); + if (can_replace) { + // DLSS's evaluation may be recorded next, before the dispatch: the barrier goes first. + scheduler.Record([](vk::CommandBuffer cmdbuf) { RecordDispatchBarrier(cmdbuf); }); + } #ifdef EGLANTINE_STREAMLINE const auto* const replacement = can_replace && !byte_addressed_shadow && TryOcarinaDlss(*pipeline, true) @@ -1874,11 +1876,15 @@ void RasterizerVulkan::DispatchCompute() { #else scheduler.TraceComputePipeline(pipeline->ShaderHash()); #endif - scheduler.Record([this, pipeline, dim, byte_addressed_shadow, bind = std::move(deferred_bind) + scheduler.Record([this, pipeline, dim, byte_addressed_shadow, bind = std::move(deferred_bind), + barrier_recorded = can_replace #ifdef EGLANTINE_STREAMLINE , replacement #endif ](vk::CommandBuffer cmdbuf) { + if (!barrier_recorded) { + RecordDispatchBarrier(cmdbuf); + } #ifdef EGLANTINE_STREAMLINE if (replacement && !replacement->EvaluationFailed()) { return; From 8b5be87f7978d8539a4e39b101cf334c65d4799c Mon Sep 17 00:00:00 2001 From: jpkto <255363096+jpkto@users.noreply.github.com> Date: Sun, 8 Nov 2026 04:22:01 +0100 Subject: [PATCH 038/105] vulkan: order later transfers after a copy that asked for barriers A buffer copy recorded in order ends with a barrier from transfers to graphics and compute. Later transfers are not in it. A reordered upload in a later submission goes into that submission's upload command buffer with no barrier of its own before it. So when it writes bytes the earlier copy wrote, nothing orders the two writes, and the older one can land last. In-order copies don't count as uses when CanReorderUpload decides. Once TickFrame clears the usage marks, the same bytes can get a reordered upload in the next submission. Uploads from the stream buffer already end with a barrier that reaches transfers (they go out in batches). This adds transfers to the barrier after the remaining copies: buffer-to-buffer copies, uploads from dedicated staging buffers and downloads. It can only add a little waiting. Found while checking the upload batching against the synchronization validation layer. Co-Authored-By: Claude Opus 5.5 --- src/video_core/renderer_vulkan/vk_buffer_cache.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/video_core/renderer_vulkan/vk_buffer_cache.cpp b/src/video_core/renderer_vulkan/vk_buffer_cache.cpp index d513334c33..5b9ede9e37 100644 --- a/src/video_core/renderer_vulkan/vk_buffer_cache.cpp +++ b/src/video_core/renderer_vulkan/vk_buffer_cache.cpp @@ -728,8 +728,11 @@ void BufferCacheRuntime::CopyBuffer(VkBuffer dst_buffer, VkBuffer src_buffer, } cmdbuf.CopyBuffer(src_buffer, dst_buffer, vk_copies); if (barrier) { + // Later transfers too: a reordered upload of the same bytes in a later + // submission has no barrier of its own before it. cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_TRANSFER_BIT, - vk::PIPELINE_STAGE_GRAPHICS_COMPUTE, 0, WRITE_BARRIER); + vk::PIPELINE_STAGE_GRAPHICS_COMPUTE_TRANSFER, 0, + WRITE_BARRIER); } }, location); From 62db242b344f548035b16838fb7a7faa79a6d370 Mon Sep 17 00:00:00 2001 From: jpkto <255363096+jpkto@users.noreply.github.com> Date: Sun, 8 Nov 2026 04:24:18 +0100 Subject: [PATCH 039/105] video_core: publish to the split's back once it waits, copy fewer list words at resumes In test49 the back waited for work 131 ms a second, near its limit of 913 ms busy, and for 49 ms of that (6,805 times) the front held words it could have run. The front publishes every 256 words (8 after one of its waits), so the back ran dry in the middle of a long parse and waited until the front reached the next threshold. The back now stores its position in back_idle_at when it runs out of work, before it spins. At each commit the front compares it with the position it last published: while the two are equal the back waits for exactly the words the front holds, and the front publishes them once it has 8, as after its own waits. The back writes the word once per wait, and the front's load stays in its cache otherwise, so the cost per word is a load and a compare. The minimum of 8 keeps a back that runs faster than the front from taking one word per publish. When an acquire resumes, the front copies every queued command list again (RefreshSnapshot): 97 million words a second in test49. The current list's entries before the one the channel suspended in ran already and are never parsed again, and the suspended entry reads its remaining words from memory (CanUseSubmittedCommands). These are no longer copied. The lists behind it are copied as before. Tested with the split harness (real GpuSplit and back thread), in a copy with two new checks: an idle back gets 9 words without a publish while 6 still wait, and a back held in an entry gets nothing early. The doubling check now holds the back in an entry, since an idle back would take the words sooner. 20 runs on 2 CPUs, 10 pinned to one, 10 with EDEN_NX2_GPU_PROFILE: all passed. Co-Authored-By: Claude Opus 5.5 --- src/video_core/dma_pusher.cpp | 11 ++++++++--- src/video_core/dma_pusher.h | 5 +++-- src/video_core/gpu_split.cpp | 3 +++ src/video_core/gpu_split.h | 11 ++++++++++- 4 files changed, 24 insertions(+), 6 deletions(-) diff --git a/src/video_core/dma_pusher.cpp b/src/video_core/dma_pusher.cpp index d7e2af0659..17392a0f9a 100644 --- a/src/video_core/dma_pusher.cpp +++ b/src/video_core/dma_pusher.cpp @@ -125,12 +125,16 @@ bool DmaPusher::Step() { // as this frame's commands (methods on unbound subchannels, garbage QMDs, then a channel // waiting forever on a release it never got). The acquire is released now, so the commands // are written and the memory not yet reused: copy them here. EDEN_NX2_RESNAPSHOT=0: off. + // The current list's entries before this one ran already, and an entry the acquire suspended + // inside reads its remaining words from memory anyway (CanUseSubmittedCommands). if (ResnapshotEnabled() && suspensions.load(std::memory_order_relaxed) != suspensions_before) { const u64 now = suspensions.load(std::memory_order_acquire); + size_t first_entry = dma_pushbuffer_subindex + (command_resume_offset != 0 ? 1 : 0); for (CommandList& list : dma_pushbuffer) { if (!list.submitted_offsets.empty()) { - RefreshSnapshot(list, now); + RefreshSnapshot(list, now, first_entry); } + first_entry = 0; } } @@ -643,10 +647,11 @@ void DmaPusher::NoteDmaSemaphore(u32 method, u32 argument) { } } -void DmaPusher::RefreshSnapshot(CommandList& list, u64 now) { +void DmaPusher::RefreshSnapshot(CommandList& list, u64 now, size_t first_entry) { VideoCommon::Nx2Profile::Scope profile_scope{VideoCommon::Nx2Profile::Zone::Resnapshot}; // TEST list.snapshot_suspensions = now; - for (size_t i = 0; i < list.command_lists.size() && i < list.submitted_offsets.size(); ++i) { + for (size_t i = first_entry; i < list.command_lists.size() && i < list.submitted_offsets.size(); + ++i) { const CommandListHeader& entry = list.command_lists[i]; const u32 offset = list.submitted_offsets[i]; if (offset + entry.size > list.submitted_words.size()) { diff --git a/src/video_core/dma_pusher.h b/src/video_core/dma_pusher.h index f0f2094802..7851aa1460 100644 --- a/src/video_core/dma_pusher.h +++ b/src/video_core/dma_pusher.h @@ -223,8 +223,9 @@ class DmaPusher final { /// A copy engine method sent to the split's back: a launch that releases its semaphore tells /// the front what it writes (GpuSplit::NoteSemaphoreWrite). void NoteDmaSemaphore(u32 method, u32 argument); - /// Copies the list's words from memory again, as of suspension count `now`. - void RefreshSnapshot(CommandList& list, u64 now); + /// Copies the list's words from memory again, as of suspension count `now`, from entry + /// `first_entry` on (the entries before it ran already). + void RefreshSnapshot(CommandList& list, u64 now, size_t first_entry = 0); /// Copies the list's deferred entries (CommandList::deferred_entries) from memory. void RefreshDeferredEntries(CommandList& list); diff --git a/src/video_core/gpu_split.cpp b/src/video_core/gpu_split.cpp index cda0e266bc..5e4f9d939e 100644 --- a/src/video_core/gpu_split.cpp +++ b/src/video_core/gpu_split.cpp @@ -157,6 +157,7 @@ void GpuSplit::Stop() { front_macros = false; published.store(0, std::memory_order_relaxed); consumed.store(0, std::memory_order_relaxed); + back_idle_at.store(~u64{0}, std::memory_order_relaxed); back_waiting.store(false, std::memory_order_relaxed); front_waiting.store(false, std::memory_order_relaxed); tick_requested.store(false, std::memory_order_relaxed); @@ -568,6 +569,8 @@ void GpuSplit::BackLoop(std::stop_token stop_token) { MaybeTick(); u64 available = published.load(std::memory_order_acquire); if (available == read_position) { + // The front publishes what it holds at its next commit (a few words) from now on. + back_idle_at.store(read_position, std::memory_order_relaxed); Nx2Profile::Scope scope{Nx2Profile::Zone::SplitWait}; const GpuWaitStats::IdleScope idle{GpuWaitStats::g_back_idle_ns}; // TEST (profile_publishes): out of work from now until this block ends. diff --git a/src/video_core/gpu_split.h b/src/video_core/gpu_split.h index 0c85729da7..a1cc9a1fd2 100644 --- a/src/video_core/gpu_split.h +++ b/src/video_core/gpu_split.h @@ -394,7 +394,12 @@ class GpuSplit { NoteFirstUnpublished(); // TEST } write_position += words; - if (write_position - published_local >= publish_every) { + const u64 unpublished = write_position - published_local; + // The back waits for work at the last position published (it stored it in back_idle_at + // before it spun): it gets these words after a few, not after publish_every. + if (unpublished >= publish_every || + (unpublished >= PublishAfterWait && + back_idle_at.load(std::memory_order_relaxed) == published_local)) { Publish(); } } @@ -493,6 +498,10 @@ class GpuSplit { /// The front sleeps in Backoff (#161: instead of yielding). alignas(128) std::atomic front_waiting{}; std::atomic front_wake{}; + /// The back's position when it last ran out of work, stored before it waits for more: while + /// it equals published_local, the back waits for the words the front holds (Commit). The back + /// writes it once per wait, the front reads it at every Commit. + alignas(128) std::atomic back_idle_at{~u64{0}}; /// TEST (profile_publishes): when the back last ran out of work, 0 while it runs. alignas(128) std::atomic back_idle_since{}; std::jthread back_thread; From 78f08aed9ea9fd124d548ecb719a551a01c0cb66 Mon Sep 17 00:00:00 2001 From: jpkto <255363096+jpkto@users.noreply.github.com> Date: Sun, 8 Nov 2026 04:56:57 +0100 Subject: [PATCH 040/105] video_core: keep the split front's QMD view in sync, place copies and uploads In test49 the QMD view never took effect. Per 10 s, about 185,000 signals were not valued because the table was not synced, and the front took the back's table over 0 to 2 times. The takeover needed Ran(Position()) at a QMD method: the back had to have run every word sent, including the ones the front had not published yet. At a QMD method the front is in the middle of a macro's stream, so this almost never held. After the rare takeovers, the next signal was lost again to queued work that might write anywhere (2 of 2): uploads, copies and the GPU's sync operations all counted as anywhere. The view now starts in sync. An engine's front drains the split when it takes the engine's macros over. With the back idle there, the view takes the back's table over instead of dropping it. From then on it stays the back's table as long as each descriptor the front reads is the one the back will read, with no need for an idle back. The front takes a descriptor from: - the signal's own inline QMD, as before; - memory, once the back ran every write sent there (placed in its granule, or unplaced), on pages GPU work never wrote, as before; - new: the inline QMD a queued LOAD_INLINE_QMD_DATA writes there, if that is the last write sent there. The front keeps the last 256 inline QMDs it sent, with their positions (NoteInlineWrite), so a decrement sent before the back wrote its QMD is valued. Where the front can't tell, it drops its table as before. It takes the back's over again the next time the back has run everything sent: between commands and in the loop that polls suspended channels, where the back is often idle (6,700 waits a second in test49), and before QMD methods. The front now places more writes (GpuSplit::Writes), so they stop the view only where they overlap a QMD or a word it reads: - Copy engine launches: every byte any layout may write (MaxwellDMA::WritesOfLaunch, from a copy of the engine's registers that DmaPusher keeps), and the semaphore. They also write the caches where they read GPU-written memory (Caches). - Inline uploads of the compute and inline-to-memory engines, and block linear 3D uploads (Upload::UploadedRange, which ProcessData now shares). - The GPU's sync operations (TickWork) write guest memory only where GPU work wrote, so they are Caches. The back runs them between any two entries anyway (RequestTick). - Writes of 16 KiB to 16 MiB are noted by 64 KiB granule in coarse slots. Longer ones count as unplaced. A new class, Unplaced, covers guest memory anywhere without the GPU's mappings: engine work nobody placed (blits), longer writes, and launches the view could not value. Anywhere stays for what may unmap (the functions run in order). The view needs no queued mapping change (LastMappingChange). A maximum's or a reduction's base now comes from the last release sent to the word only if nothing sent since may have written there, placed or not, since copies are now placed. Tests: - QMD harness (FrontQmdView against a model of the back): new scripted cases (a decrement before the back wrote its inline QMD, writes after it, copies elsewhere and over a QMD, a queued mapping change with the back never idle) and four fuzz kinds of 100 to 200 runs. With the back always 4 to 200 entries behind and placeable writes, all 900,487 QMD entries stayed valued without a single takeover. With EDEN_NX2_DRAIN_CHECK, all 1,476,108 check lists and 655,480 notes matched across the four kinds, including 9,242 takeovers while behind. - A new harness ran the real MaxwellDMA and Upload::State on a real MemoryManager, with random registers over random memory. Over 2,000 copies and 2,000 uploads (5.6 million bytes changed), no byte changed outside the range the front places. - Split harness: the new classes and coarse slots; 20 runs, 10 pinned to one CPU, 10 with EDEN_NX2_GPU_PROFILE. Co-Authored-By: Claude Opus 5.5 --- src/video_core/dma_pusher.cpp | 106 +++++++++--- src/video_core/dma_pusher.h | 19 ++- src/video_core/engines/engine_upload.cpp | 40 ++++- src/video_core/engines/engine_upload.h | 11 ++ src/video_core/engines/kepler_compute.cpp | 91 +++++----- src/video_core/engines/maxwell_3d.cpp | 17 +- src/video_core/engines/maxwell_dma.cpp | 77 +++++++++ src/video_core/engines/maxwell_dma.h | 17 ++ src/video_core/front_qmd.h | 195 ++++++++++++++++------ src/video_core/gpu_split.cpp | 27 ++- src/video_core/gpu_split.h | 52 ++++-- src/video_core/gpu_thread.cpp | 13 +- 12 files changed, 507 insertions(+), 158 deletions(-) diff --git a/src/video_core/dma_pusher.cpp b/src/video_core/dma_pusher.cpp index 17392a0f9a..617331e6b1 100644 --- a/src/video_core/dma_pusher.cpp +++ b/src/video_core/dma_pusher.cpp @@ -22,7 +22,9 @@ #include "video_core/dma_pusher.h" #include "video_core/command_fetch_order.h" #include "video_core/control/channel_state.h" +#include "video_core/engines/engine_upload.h" #include "video_core/engines/kepler_compute.h" +#include "video_core/engines/kepler_memory.h" #include "video_core/engines/maxwell_3d.h" #include "video_core/engines/maxwell_dma.h" #include "video_core/gpu.h" @@ -64,6 +66,30 @@ void NoteListRead(MemoryManager& memory, s32 channel, GPUVAddr address, u64 devi unpublished, channel); } +static_assert(DmaPusher::DmaRegisterCount == Engines::MaxwellDMA::RegisterCount); +static_assert(DmaPusher::InlineToMemoryRegisterCount == Engines::KeplerMemory::Regs::NUM_REGS); + +/// What the back writes running `method` of an engine the front sends methods to as they are +/// (GpuSplit::Writes). A method the engine only keeps for later (its method sink) writes nothing +/// yet. Copy launches write the caches where they read GPU-written memory; what they copy to and +/// release, and what inline-to-memory uploads write, DmaPusher::NoteEngineWrites places. +VideoCommon::GpuSplit::Writes EngineWrites(const Engines::EngineInterface& engine, + Engines::EngineTypes type, u32 method) { + using Writes = VideoCommon::GpuSplit::Writes; + if (!engine.execution_mask[method]) { + return Writes::Noted; + } + switch (type) { + case Engines::EngineTypes::MaxwellDMA: + return Writes::Caches; + case Engines::EngineTypes::KeplerMemory: + return Writes::Noted; + default: + // Blits and the engines without a front: where they write, the front does not know. + return Writes::Unplaced; + } +} + } // Anonymous namespace bool DmaPusher::IsMemoryWriter(const Engines::EngineInterface& engine, Engines::EngineTypes type, @@ -508,6 +534,7 @@ void DmaPusher::CallMethod(u32 argument) { return; } const auto engine_type = subchannel_type[dma_state.subchannel]; + ShadowEngineMethod(engine_type, dma_state.method, argument); const bool writer = split.Active() && IsMemoryWriter(*subchannel, engine_type, dma_state.method); SCOPE_EXIT { @@ -535,15 +562,10 @@ void DmaPusher::CallMethod(u32 argument) { } } } - // A method the engine only keeps for later (its method sink) writes nothing yet. - using Writes = VideoCommon::GpuSplit::Writes; split.Method(subchannel, dma_state.method, argument, dma_state.is_last_call, WantsSegment() ? &segment : nullptr, - subchannel->execution_mask[dma_state.method] ? Writes::Anywhere - : Writes::Noted); - if (engine_type == Engines::EngineTypes::MaxwellDMA) { - NoteDmaSemaphore(dma_state.method, argument); - } + EngineWrites(*subchannel, engine_type, dma_state.method)); + NoteEngineWrites(engine_type, dma_state.method); return; } if (!subchannel->execution_mask[dma_state.method]) { @@ -601,14 +623,25 @@ void DmaPusher::CallMultiMethod(const u32* base_start, u32 num_methods) { } } split.MultiMethod(subchannel, dma_state.method, base_start, num_methods, - dma_state.method_count, WantsSegment() ? &segment : nullptr); - if (engine_type == Engines::EngineTypes::MaxwellDMA) { - for (u32 i = 0; i < num_methods; ++i) { - NoteDmaSemaphore(dma_state.method, base_start[i]); + dma_state.method_count, WantsSegment() ? &segment : nullptr, + EngineWrites(*subchannel, engine_type, dma_state.method)); + // The back runs them one by one: a copy engine launch each. + const bool launches = engine_type == Engines::EngineTypes::MaxwellDMA && + dma_state.method == Engines::MaxwellDMA::LaunchDmaMethod; + for (u32 i = 0; i < num_methods; ++i) { + ShadowEngineMethod(engine_type, dma_state.method, base_start[i]); + if (launches) { + NoteEngineWrites(engine_type, dma_state.method); } } + if (!launches && num_methods != 0) { + NoteEngineWrites(engine_type, dma_state.method); + } return; } + for (u32 i = 0; i < num_methods; ++i) { + ShadowEngineMethod(engine_type, dma_state.method, base_start[i]); + } VideoCommon::Nx2Profile::Scope method_scope{VideoCommon::Nx2Profile::Zone::Method}; subchannel->ConsumeSink(system); subchannel->current_dma_segment = dma_state.dma_get + dma_state.dma_word_offset; @@ -616,32 +649,53 @@ void DmaPusher::CallMultiMethod(const u32* base_start, u32 num_methods) { } } -void DmaPusher::NoteDmaSemaphore(u32 method, u32 argument) { - using Engines::MaxwellDMA; - if (method >= MaxwellDMA::SemaphoreMethod && method < MaxwellDMA::SemaphoreMethod + 3) { - dma_semaphore[method - MaxwellDMA::SemaphoreMethod] = argument; +void DmaPusher::ShadowEngineMethod(Engines::EngineTypes type, u32 method, u32 argument) { + // As the engines' CallMethod and ConsumeSink store them. + if (type == Engines::EngineTypes::MaxwellDMA) { + if (method < dma_registers.size()) { + dma_registers[method] = argument; + } + } else if (type == Engines::EngineTypes::KeplerMemory) { + if (method < inline_to_memory_registers.size()) { + inline_to_memory_registers[method] = argument; + } + } +} + +void DmaPusher::NoteEngineWrites(Engines::EngineTypes type, u32 method) { + auto& split = VideoCommon::GpuSplit::Get(); + if (type == Engines::EngineTypes::KeplerMemory) { + if (method != KEPLERMEMORY_REG_INDEX(data)) { + return; + } + // Upload::State::ProcessData, of the upload KeplerMemory's exec started. + Engines::Upload::Registers upload; + std::memcpy(&upload, &inline_to_memory_registers[KEPLERMEMORY_REG_INDEX(upload)], + sizeof(upload)); + const bool linear = (inline_to_memory_registers[KEPLERMEMORY_REG_INDEX(exec)] & 1) != 0; + const auto range = Engines::Upload::UploadedRange(upload, linear); + split.NoteWrite(memory_manager, range.address, range.size); return; } - if (method != MaxwellDMA::LaunchDmaMethod) { + using Engines::MaxwellDMA; + if (type != Engines::EngineTypes::MaxwellDMA || method != MaxwellDMA::LaunchDmaMethod) { return; } - // As MaxwellDMA::ReleaseSemaphore writes it. - using SemaphoreType = MaxwellDMA::LaunchDMA::SemaphoreType; - const auto type = std::bit_cast(argument).semaphore_type.Value(); - if (type != SemaphoreType::RELEASE_ONE_WORD_SEMAPHORE && - type != SemaphoreType::RELEASE_FOUR_WORD_SEMAPHORE) { + const auto writes = MaxwellDMA::WritesOfLaunch(dma_registers); + split.NoteWrite(memory_manager, writes.destination.address, writes.destination.size); + if (writes.semaphore.size == 0) { return; } - const GPUVAddr address = MaxwellDMA::PackedGPUVAddr{dma_semaphore[0], dma_semaphore[1]}; - const auto device = memory_manager.GpuToCpuAddress(address); + // As MaxwellDMA::ReleaseSemaphore writes it. + split.NoteWrite(memory_manager, writes.semaphore.address, writes.semaphore.size); + const auto device = memory_manager.GpuToCpuAddress(writes.semaphore.address); if (!device) { return; } - auto& split = VideoCommon::GpuSplit::Get(); - split.NoteSemaphoreWrite(*device, dma_semaphore[2], true); + split.NoteSemaphoreWrite(*device, dma_registers[MaxwellDMA::SemaphoreMethod + 2], true); VideoCommon::Nx2AcquireStats::NoteRelease(*device, VideoCommon::Nx2AcquireStats::Releaser::Copy); - if (type == SemaphoreType::RELEASE_FOUR_WORD_SEMAPHORE) { + if (writes.semaphore.size > sizeof(u32)) { // The payload as 64 bits (a timestamp follows). split.NoteSemaphoreWrite(*device + 4, 0, true); } diff --git a/src/video_core/dma_pusher.h b/src/video_core/dma_pusher.h index 7851aa1460..48f66b8716 100644 --- a/src/video_core/dma_pusher.h +++ b/src/video_core/dma_pusher.h @@ -220,9 +220,13 @@ class DmaPusher final { /// EDEN_NX2_DRAIN_CHECK: checks an entry on pages no GPU work had written against the compute /// and copy work it did not wait for (Nx2DrainCheck). void CheckUnwaitedWriters(GPUVAddr address, size_t size); - /// A copy engine method sent to the split's back: a launch that releases its semaphore tells - /// the front what it writes (GpuSplit::NoteSemaphoreWrite). - void NoteDmaSemaphore(u32 method, u32 argument); + /// Keeps the front's copy of the copy and inline-to-memory engines' registers, with the split + /// and without. + void ShadowEngineMethod(Engines::EngineTypes type, u32 method, u32 argument); + /// An engine method was just sent to the split's back, as it is: places what a copy launch or + /// an inline-to-memory upload writes (GpuSplit::NoteWrite), and tells what a launch releases + /// (GpuSplit::NoteSemaphoreWrite). + void NoteEngineWrites(Engines::EngineTypes type, u32 method); /// Copies the list's words from memory again, as of suspension count `now`, from entry /// `first_entry` on (the entries before it ran already). void RefreshSnapshot(CommandList& list, u64 now, size_t first_entry = 0); @@ -310,8 +314,13 @@ class DmaPusher final { std::array subchannels{}; std::array subchannel_type; - /// The copy engine's semaphore address (high, low) and payload as sent (NoteDmaSemaphore). - std::array dma_semaphore{}; + /// The registers of the channel's copy engine (MaxwellDMA::RegisterCount) and its + /// inline-to-memory engine (KeplerMemory::Regs::NUM_REGS), as the methods sent left them + /// (ShadowEngineMethod): where the back's copies and uploads write. + static constexpr size_t DmaRegisterCount = 0x800; + static constexpr size_t InlineToMemoryRegisterCount = 0x7f; + std::array dma_registers{}; + std::array inline_to_memory_registers{}; Engines::Puller puller; std::mutex sync_mutex; diff --git a/src/video_core/engines/engine_upload.cpp b/src/video_core/engines/engine_upload.cpp index 0ffdae8036..e4799db593 100644 --- a/src/video_core/engines/engine_upload.cpp +++ b/src/video_core/engines/engine_upload.cpp @@ -16,6 +16,32 @@ namespace Tegra::Engines::Upload { +namespace { + +/// The power of two bytes a swizzled upload moves per element (ProcessData). +u32 SwizzleBppShift(const Registers& regs) { + return Common::FoldRight( + 4U, [](u32 x, u32 y) { return (std::min)(x, static_cast(std::countr_zero(y))); }, + regs.dest.width, regs.line_length_in, regs.dest.x, static_cast(regs.dest.Address())); +} + +} // Anonymous namespace + +WrittenRange UploadedRange(const Registers& regs, bool is_linear) { + const GPUVAddr address = regs.dest.Address(); + if (is_linear) { + if (regs.line_count == 0 || regs.line_length_in == 0) { + return {}; + } + // The last line ends last, whatever the pitch. + return {address, u64{regs.dest.pitch} * (regs.line_count - 1) + regs.line_length_in}; + } + const u32 bpp_shift = SwizzleBppShift(regs); + return {address, Tegra::Texture::CalculateSize( + true, 1U << bpp_shift, regs.dest.width >> bpp_shift, regs.dest.height, + regs.dest.depth, regs.dest.BlockHeight(), regs.dest.BlockDepth())}; +} + State::State(MemoryManager& memory_manager_, Registers& regs_) : regs{regs_}, memory_manager{memory_manager_} {} @@ -77,15 +103,11 @@ void State::ProcessData(std::span read_buffer) { rasterizer->AccelerateInlineToMemory(dest_line, regs.line_length_in, buffer); } } else { - u32 width = regs.dest.width; - u32 x_elements = regs.line_length_in; - u32 x_offset = regs.dest.x; - const u32 bpp_shift = Common::FoldRight( - 4U, [](u32 x, u32 y) { return (std::min)(x, static_cast(std::countr_zero(y))); }, - width, x_elements, x_offset, static_cast(address)); - width >>= bpp_shift; - x_elements >>= bpp_shift; - x_offset >>= bpp_shift; + // As UploadedRange places it. + const u32 bpp_shift = SwizzleBppShift(regs); + const u32 width = regs.dest.width >> bpp_shift; + const u32 x_elements = regs.line_length_in >> bpp_shift; + const u32 x_offset = regs.dest.x >> bpp_shift; const u32 bytes_per_pixel = 1U << bpp_shift; const std::size_t dst_size = Tegra::Texture::CalculateSize( true, bytes_per_pixel, width, regs.dest.height, regs.dest.depth, diff --git a/src/video_core/engines/engine_upload.h b/src/video_core/engines/engine_upload.h index 55a82f938b..8169ce8cba 100644 --- a/src/video_core/engines/engine_upload.h +++ b/src/video_core/engines/engine_upload.h @@ -57,6 +57,17 @@ struct Registers { } dest; }; +/// Guest memory [address, address + size) an upload writes (size 0: none). +struct WrittenRange { + GPUVAddr address{}; + u64 size{}; +}; + +/// Where State::ProcessData writes an upload with registers `regs` (linear: lines of +/// line_length_in bytes, pitch apart; else swizzled into the destination surface): what the split +/// GPU thread's front notes for the back to write running the upload's data methods. +[[nodiscard]] WrittenRange UploadedRange(const Registers& regs, bool is_linear); + class State { public: explicit State(MemoryManager& memory_manager_, Registers& regs_); diff --git a/src/video_core/engines/kepler_compute.cpp b/src/video_core/engines/kepler_compute.cpp index bbfb3bc1f3..932ff30960 100644 --- a/src/video_core/engines/kepler_compute.cpp +++ b/src/video_core/engines/kepler_compute.cpp @@ -1437,8 +1437,7 @@ namespace { // The QMD scheduling of every channel, run on the front as the back will run it. VideoCommon::FrontQmdView& GetFrontQmd() { - static VideoCommon::FrontQmdView view; - return view; + return VideoCommon::FrontQmdView::Get(); } // An engine's GPU address space as its back reads QMDs and their release words (FrontQmdView). @@ -1486,32 +1485,12 @@ struct FrontQmdMemory { } // What the back writes to guest memory running `method` (GpuSplit::Writes), as -// ProcessNativeMethod runs it: nothing for register writes and the macro code and bindings (the -// front's), what FrontNoteWrites notes for reports, inline QMDs and QMD launches, anywhere for -// inline uploads (and with EDEN_NATIVE_DRAW_DIAGNOSTICS, the methods it traces). -[[nodiscard]] VideoCommon::GpuSplit::Writes FrontWrites(const std::bitset<0xe00>& plain_methods, - u32 method) { +// ProcessNativeMethod runs it: below the macro methods, nothing for register writes, the macro +// code and bindings (the front's) and the methods EDEN_NATIVE_DRAW_DIAGNOSTICS traces, and what +// FrontNoteWrites notes for reports, inline QMDs, QMD launches and inline uploads. +[[nodiscard]] VideoCommon::GpuSplit::Writes FrontWrites(u32 method) { using Writes = VideoCommon::GpuSplit::Writes; - if (method >= plain_methods.size()) { - return Writes::Anywhere; - } - if (plain_methods[method]) { - return Writes::Noted; - } - switch (method) { - case 0x45: - case 0x46: - case 0x47: - case 0x48: - case 0xae: - case 0xaf: - case 0xb0: - case 0x107: - case 0x6c3: - return Writes::Noted; - default: - return Writes::Anywhere; - } + return method < 0xe00 ? Writes::Noted : Writes::Unplaced; } } // Anonymous namespace @@ -1543,8 +1522,9 @@ KeplerCompute::FrontState& KeplerCompute::Front(VideoCommon::GpuSplit& split) { front->bind_pointer = native_bind_pointer; front->macro = native_macro; front_macros = true; - // A new session's or a restarted split's: the QMDs the back scheduled meanwhile. - GetFrontQmd().Lose(); + // A new session's or a restarted split's: the QMDs the back scheduled meanwhile, in the + // table the engines share. The back is idle, its table the current one. + GetFrontQmd().TakeOver(native_qmd); } return *front; } @@ -1613,7 +1593,7 @@ void KeplerCompute::FrontCallMethod(Core::System& system, VideoCommon::GpuSplit& const GPUVAddr* segment) { auto& fs = Front(split); if (method >= fs.registers.size()) { - split.Method(this, method, argument, is_last_call, segment); + split.Method(this, method, argument, is_last_call, segment, FrontWrites(method)); return; } if (method >= 0xe00) { @@ -1632,10 +1612,9 @@ void KeplerCompute::FrontCallMethod(Core::System& system, VideoCommon::GpuSplit& } FrontApply(fs, method, argument); if (IsQmdMethod(method)) { - GetFrontQmd().Sync(split, native_qmd); + GetFrontQmd().MaybeTakeOver(split, native_qmd); } - split.Method(this, method, argument, is_last_call, segment, - FrontWrites(native_plain_methods, method)); + split.Method(this, method, argument, is_last_call, segment, FrontWrites(method)); FrontNoteWrites(split, fs, method, argument, false); FrontQmdCheck(split); } @@ -1673,6 +1652,17 @@ void KeplerCompute::FrontNoteWrites(VideoCommon::GpuSplit& split, const FrontSta case 0x107: FrontInlineQmd(split, fs, argument); break; + case KEPLER_COMPUTE_REG_INDEX(data_upload): { + // Upload::State::ProcessData, of the upload exec_upload started: the whole of it at each + // data method, at most. + Upload::Registers upload; + std::memcpy(&upload, fs.registers.data() + KEPLER_COMPUTE_REG_INDEX(upload), + sizeof(upload)); + const bool linear = (fs.registers[KEPLER_COMPUTE_REG_INDEX(exec_upload)] & 1) != 0; + const auto range = Upload::UploadedRange(upload, linear); + split.NoteWrite(memory_manager, range.address, range.size); + break; + } case 0xae: case 0xaf: case 0xb0: @@ -1737,18 +1727,23 @@ void KeplerCompute::FrontInlineQmd(VideoCommon::GpuSplit& split, const FrontStat u32 last_word) { const GPUVAddr address = ((GPUVAddr{fs.registers[0xc6]} << 32) | fs.registers[0xc7]) << 8; if (!memory_manager.IsFullyMappedRange(address, 256)) { - // The back writes nothing either, unless a map still queued changes that. - if (!split.Ran(split.LastUnplacedWrite())) { + // The back writes nothing either, unless a change of the mappings still queued changes + // that. + if (!split.Ran(split.LastMappingChange())) { GetFrontQmd().Lose(); - split.NoteAnyWrite(); + split.NoteUnplacedWrite(); } return; } split.NoteWrite(memory_manager, address, 256); + // The words the back writes (registers 0xc8 to 0x107). + NativeQmdScheduler::Descriptor qmd; + std::memcpy(qmd.data(), fs.registers.data() + 0xc8, sizeof(qmd)); + qmd[63] = last_word; + if (const auto device = memory_manager.GpuToCpuAddress(address)) { + GetFrontQmd().NoteInlineWrite(split, *device, qmd); + } if ((fs.registers[0xa7] & (1U << 24)) != 0 || (fs.registers[0xcc] & (1U << 12)) == 0) { - NativeQmdScheduler::Descriptor qmd; - std::memcpy(qmd.data(), fs.registers.data() + 0xc8, sizeof(qmd)); - qmd[63] = last_word; GetFrontQmd().Signal(split, FrontQmdMemory{memory_manager}, address, 3, &qmd); } } @@ -1764,19 +1759,23 @@ void KeplerCompute::FrontCallMultiMethod(Core::System& system, VideoCommon::GpuS return; } if (method >= fs.registers.size()) { - split.MultiMethod(this, method, base_start, amount, methods_pending, segment); + split.MultiMethod(this, method, base_start, amount, methods_pending, segment, + FrontWrites(method)); return; } for (u32 i = 0; i < amount; ++i) { FrontApply(fs, method, base_start[i]); } if (IsQmdMethod(method)) { - GetFrontQmd().Sync(split, native_qmd); + GetFrontQmd().MaybeTakeOver(split, native_qmd); } split.MultiMethod(this, method, base_start, amount, methods_pending, segment, - FrontWrites(native_plain_methods, method)); - // The back runs them one by one; what they write is the back's once it ran them all. - for (u32 i = 0; i < amount; ++i) { + FrontWrites(method)); + // The back runs them one by one; what they write is the back's once it ran them all. Each + // upload data word may write the upload's range: once covers them. + const u32 notes = + method == KEPLER_COMPUTE_REG_INDEX(data_upload) ? (std::min)(amount, 1U) : amount; + for (u32 i = 0; i < notes; ++i) { FrontNoteWrites(split, fs, method, base_start[i], false); } FrontQmdCheck(split); @@ -1801,9 +1800,9 @@ void KeplerCompute::FrontRunMacro(Core::System& system, VideoCommon::GpuSplit& s } FrontApply(fs, address, value); if (IsQmdMethod(address)) { - GetFrontQmd().Sync(split, native_qmd); + GetFrontQmd().MaybeTakeOver(split, native_qmd); } - split.MacroWrite(this, address, value, FrontWrites(native_plain_methods, address)); + split.MacroWrite(this, address, value, FrontWrites(address)); FrontNoteWrites(split, fs, address, value, true); FrontQmdCheck(split); return true; diff --git a/src/video_core/engines/maxwell_3d.cpp b/src/video_core/engines/maxwell_3d.cpp index c0b876ebe8..b7094415de 100644 --- a/src/video_core/engines/maxwell_3d.cpp +++ b/src/video_core/engines/maxwell_3d.cpp @@ -1955,18 +1955,15 @@ void Maxwell3D::FrontNoteWrites(VideoCommon::GpuSplit& split, const FrontState& 16); break; case MAXWELL3D_REG_INDEX(inline_data): { - // Upload::State::ProcessData: lines of line_length_in bytes, pitch apart. + // Upload::State::ProcessData: lines of line_length_in bytes, pitch apart, or swizzled + // into the destination surface. const auto launch = std::bit_cast(fs.regs[MAXWELL3D_REG_INDEX(launch_dma)]); - const u32 lines = fs.regs[MAXWELL3D_REG_INDEX(upload.line_count)]; - const u32 length = fs.regs[MAXWELL3D_REG_INDEX(upload.line_length_in)]; - if (launch.memory_layout != Regs::LaunchDMA::Layout::Pitch) { - split.NoteAnyWrite(); // Swizzled into the destination surface - } else if (lines != 0 && length != 0) { - split.NoteWrite(memory_manager, address(MAXWELL3D_REG_INDEX(upload.dest.address_high)), - u64{fs.regs[MAXWELL3D_REG_INDEX(upload.dest.pitch)]} * (lines - 1) + - length); - } + Upload::Registers upload; + std::memcpy(&upload, fs.regs.data() + MAXWELL3D_REG_INDEX(upload), sizeof(upload)); + const auto range = + Upload::UploadedRange(upload, launch.memory_layout == Regs::LaunchDMA::Layout::Pitch); + split.NoteWrite(memory_manager, range.address, range.size); break; } case MAXWELL3D_REG_INDEX(render_enable.mode): { diff --git a/src/video_core/engines/maxwell_dma.cpp b/src/video_core/engines/maxwell_dma.cpp index 0b0541799b..c6b79a4c20 100644 --- a/src/video_core/engines/maxwell_dma.cpp +++ b/src/video_core/engines/maxwell_dma.cpp @@ -4,8 +4,12 @@ // SPDX-FileCopyrightText: Copyright 2018 yuzu Emulator Project // SPDX-License-Identifier: GPL-2.0-or-later +#include +#include #include +#include #include "common/algorithm.h" +#include "common/alignment.h" #include "common/assert.h" #include "common/logging.h" #include @@ -35,6 +39,79 @@ void MaxwellDMA::BindRasterizer(VideoCore::RasterizerInterface* rasterizer_) { rasterizer = rasterizer_; } +MaxwellDMA::LaunchWrites MaxwellDMA::WritesOfLaunch(std::span registers) { + const auto word = [registers](size_t offset) { return registers[offset / sizeof(u32)]; }; + const auto address = [&word](size_t offset) { + return static_cast(PackedGPUVAddr{word(offset), word(offset + sizeof(u32))}); + }; + // The power of two bytes a swizzling copy moves per element (as the copies compute it). + const auto bpp_shift = [](auto... values) { + return Common::FoldRight( + 4U, [](u32 x, u32 y) { return (std::min)(x, static_cast(std::countr_zero(y))); }, + static_cast(values)...); + }; + LaunchWrites writes{}; + const auto launch = std::bit_cast(word(offsetof(Regs, launch_dma))); + if (launch.semaphore_type == LaunchDMA::SemaphoreType::RELEASE_ONE_WORD_SEMAPHORE) { + writes.semaphore = {address(offsetof(Regs, semaphore)), sizeof(u32)}; + } else if (launch.semaphore_type == LaunchDMA::SemaphoreType::RELEASE_FOUR_WORD_SEMAPHORE) { + writes.semaphore = {address(offsetof(Regs, semaphore)), 4 * sizeof(u32)}; + } + const GPUVAddr in = address(offsetof(Regs, offset_in)); + const GPUVAddr out = address(offsetof(Regs, offset_out)); + const s32 pitch_out = static_cast(word(offsetof(Regs, pitch_out))); + const u32 length = word(offsetof(Regs, line_length_in)); + const u32 lines = word(offsetof(Regs, line_count)); + RemapConst remap; + std::memcpy(&remap, ®isters[offsetof(Regs, remap_const) / sizeof(u32)], sizeof(remap)); + DMA::Parameters dst; + std::memcpy(&dst, ®isters[offsetof(Regs, dst_params) / sizeof(u32)], sizeof(dst)); + DMA::Parameters src; + std::memcpy(&src, ®isters[offsetof(Regs, src_params) / sizeof(u32)], sizeof(src)); + const bool remapping = launch.remap_enable != 0; + const u32 base_bpp = !remapping ? 1U + : (remap.num_dst_components_minus_one + 1) * + (remap.component_size_minus_one + 1); + if (launch.multi_line_enable) { + const bool src_pitch = launch.src_memory_layout == LaunchDMA::MemoryLayout::PITCH; + const bool dst_pitch = launch.dst_memory_layout == LaunchDMA::MemoryLayout::PITCH; + if (!dst_pitch) { + // Swizzled into the destination surface (CopyBlockLinearToBlockLinear, + // CopyPitchToBlockLinear, or the texture cache's BufferToImage). + u32 shift = 0; + if (!remapping) { + shift = src_pitch ? bpp_shift(dst.width, length, dst.origin.x.Value(), out) + : bpp_shift(src.width, dst.width, length, src.origin.x.Value(), + dst.origin.x.Value(), in, out); + } + writes.destination = {out, CalculateSize(true, base_bpp << shift, dst.width >> shift, + dst.height, dst.depth, dst.block_size.height, + dst.block_size.depth)}; + } else if (!src_pitch) { + // Lines the pitch apart (CopyBlockLinearToPitch, or the cache's ImageToBuffer). + const u64 pitch = static_cast(std::abs(s64{pitch_out})); + writes.destination = {out, (std::max)(pitch, u64{length}) * lines}; + } else if (lines != 0 && length != 0) { + // Lines from the first to the last, whichever way the pitch goes. + const s64 last = s64{lines - 1} * pitch_out; + const u64 span = static_cast(last < 0 ? -last : last); + writes.destination = {last < 0 ? out - span : out, span + length}; + } + } else if (length != 0) { + if (remapping && remap.dst_x == RemapConst::Swizzle::CONST_A) { + // A fill: BufferClear counts words, the guest memory write takes length components. + writes.destination = {out, u64{length} * sizeof(u32)}; + } else { + // A copy; with a block linear page kind on either side, in 16-byte pieces swizzled + // within their 512-byte blocks. + const GPUVAddr begin = out & ~GPUVAddr{0x1ff}; + const GPUVAddr end = Common::AlignUp(out + length, GPUVAddr{0x200}); + writes.destination = {begin, end - begin}; + } + } + return writes; +} + void MaxwellDMA::ConsumeSinkImpl(Core::System& system) { for (auto [method, value] : method_sink) { regs.reg_array[method] = value; diff --git a/src/video_core/engines/maxwell_dma.h b/src/video_core/engines/maxwell_dma.h index 82ac274598..52b6ade13c 100644 --- a/src/video_core/engines/maxwell_dma.h +++ b/src/video_core/engines/maxwell_dma.h @@ -8,6 +8,7 @@ #include #include +#include #include #include "common/bit_field.h" @@ -244,6 +245,21 @@ class MaxwellDMA final : public EngineInterface { void BindRasterizer(VideoCore::RasterizerInterface* rasterizer); + /// The methods of the register file (CallMethod ignores the ones past it). + static constexpr u32 RegisterCount = 0x800; + /// Guest memory [address, address + size) a launch writes (size 0: none). + struct WriteRange { + GPUVAddr address{}; + u64 size{}; + }; + struct LaunchWrites { + WriteRange destination; ///< Every byte Launch() may write, whatever layout it picks + WriteRange semaphore; ///< ReleaseSemaphore's + }; + /// What a launch with register file `registers` writes (the split GPU thread's front keeps a + /// copy of the registers to place the copies the back makes). + [[nodiscard]] static LaunchWrites WritesOfLaunch(std::span registers); + explicit MaxwellDMA(MemoryManager& memory_manager_); ~MaxwellDMA() override; @@ -333,6 +349,7 @@ class MaxwellDMA final : public EngineInterface { #undef ASSERT_REG_POSITION static_assert(SemaphoreMethod * sizeof(u32) == offsetof(Regs, semaphore)); static_assert(LaunchDmaMethod * sizeof(u32) == offsetof(Regs, launch_dma)); + static_assert(RegisterCount == NUM_REGS); }; } // namespace Tegra::Engines diff --git a/src/video_core/front_qmd.h b/src/video_core/front_qmd.h index c45b506386..52b5996a19 100644 --- a/src/video_core/front_qmd.h +++ b/src/video_core/front_qmd.h @@ -4,6 +4,7 @@ #pragma once #include +#include #include #include #include @@ -19,12 +20,21 @@ namespace VideoCommon { /// front as its back will run it, on a table of its own: what the launches release is known when /// they are sent, for the front's view of the semaphore words (GpuSplit::NoteSemaphoreWrite). /// -/// The front reads the descriptors when it sends a signal, the back when it runs it, both from -/// guest memory without the GPU caches. They read the same where no map still queued can change -/// and nothing queued writes there but through the caches (GpuSplit::MemoryWritesTarget) onto -/// pages GPU work never wrote (Nx2MayBeGpuWritten). Where the front can't tell, its table may -/// differ from the back's until it takes the back's over again (Sync), and the launches count as -/// writing guest memory anywhere. +/// The table is the back's from the start: an engine's front takes it over with the back drained +/// when it takes the engine's macros over (TakeOver). It stays the back's as long as the front +/// reads the descriptors the back will read. The front reads one when it sends a signal, the back +/// when it runs it, both from guest memory without the GPU caches. The front takes a descriptor: +/// - from the signal's own inline QMD, which the back writes right before; +/// - from memory, once the back ran every write sent there (GpuSplit::MemoryWritesTarget: the +/// writes placed in its granule and the unplaced ones), on pages GPU work never wrote +/// (Nx2MayBeGpuWritten); +/// - from the inline QMD a LOAD_INLINE_QMD_DATA still queued writes there, when that is the last +/// write sent there (NoteInlineWrite). +/// No change of the GPU's mappings may be queued either (GpuSplit::LastMappingChange). Where the +/// front can't tell what the back reads, its table may no longer be the back's: it stops valuing, +/// the launches count as writing guest memory anywhere, and it takes the back's table over again +/// the next time the back has run everything sent, between commands or at a QMD method +/// (MaybeTakeOver). /// /// `Memory`, the channel's GPU address space as the back reads it, has: /// - bool IsMapped(GPUVAddr, u64), void Read(GPUVAddr, void*, u64), u32 ReadWord(GPUVAddr) (as @@ -35,16 +45,31 @@ class FrontQmdView { public: using Descriptor = Tegra::NativeQmdScheduler::Descriptor; - /// Before a QMD method is sent: takes over the table `back` shares with the other engines once - /// the back ran everything sent. - void Sync(const GpuSplit& split, const Tegra::NativeQmdScheduler& back) { + /// The view the compute engines of every channel feed: they share the back's table. + static FrontQmdView& Get() { + static FrontQmdView view; + return view; + } + + /// The back ran everything sent and runs nothing (the split drained, or it waits for work): + /// takes over the table `back` shares with the other engines. + void TakeOver(const Tegra::NativeQmdScheduler& back) { + const auto start = std::chrono::steady_clock::now(); // TEST + scheduler.CopyStates(back); + synced = true; + Nx2AcquireStats::NoteQmdSync(std::chrono::steady_clock::now() - start); + } + + /// The front, between commands and before QMD methods: takes the back's table over if the + /// view lost it and the back has run everything sent. + void MaybeTakeOver(const GpuSplit& split, const Tegra::NativeQmdScheduler& back) { if (!synced && split.Ran(split.Position())) [[unlikely]] { - const auto start = std::chrono::steady_clock::now(); // TEST - scheduler.CopyStates(back); - synced = true; - Nx2AcquireStats::NoteQmdSync(std::chrono::steady_clock::now() - start); + TakeOver(back); } } + void MaybeTakeOver(const GpuSplit& split) { + MaybeTakeOver(split, back_table); + } /// TEST (EDEN_NX2_DRAIN_CHECK): the releases noted since the last call, and whether every /// signal since was valued; nothing if no signal was sent since. @@ -63,15 +88,27 @@ class FrontQmdView { return taken; } - /// The engines schedule QMDs the front does not see (the split stopped, a front starts). + /// The engines schedule QMDs the front does not see (the split stopped). void Lose() { synced = false; + inline_writes_count = 0; } [[nodiscard]] bool Synced() const { return synced; } + /// LOAD_INLINE_QMD_DATA was just sent: running it, the back writes `qmd` to the QMD at + /// `device`. Until then the front reads it from here (ReadDescriptor). + void NoteInlineWrite(const GpuSplit& split, u64 device, const Descriptor& qmd) { + InlineWrite& written = inline_writes[inline_writes_count % inline_writes.size()]; + ++inline_writes_count; + written.device = device; + written.position = split.Position(); + written.qmd = qmd; + written.overwritten = false; + } + /// The scheduling of PCAS action `argument` on the QMD at `address` that the back runs at the /// entry just sent (KeplerCompute::ProcessNativeMethod); `inline_qmd`: the QMD it writes there /// right before. Notes what the launches it runs release. @@ -82,44 +119,33 @@ class FrontQmdView { using Lost = Nx2AcquireStats::QmdLost; checks_pending = Nx2DrainCheck::g_enabled; // TEST if (!synced) { - split.NoteAnyWrite(); - checks_unvalued = true; // TEST - Nx2AcquireStats::NoteQmdSignal(Lost::NotSynced); + Unvalued(split, Lost::NotSynced); return; } - releases.clear(); - // A map still queued may change what the back reads, mapped or not. - bool exact = split.Ran(split.LastUnplacedWrite()); - std::optional lost; // TEST: why not - if (!exact) { - lost = Lost::Unplaced; + // A change of the mappings still queued may unmap what the back reads or releases to. + if (!split.Ran(split.LastMappingChange())) { + synced = false; + Unvalued(split, Lost::Unplaced); + return; } + releases.clear(); + std::optional lost; const auto read = [&](u64 qmd_address, Descriptor& descriptor) { if (!memory.IsMapped(qmd_address, sizeof(descriptor))) { return false; } if (inline_qmd != nullptr && qmd_address == address) { descriptor = *inline_qmd; - } else { - memory.Read(qmd_address, descriptor.data(), sizeof(descriptor)); - const auto device = memory.Device(qmd_address); - const bool written = - !device.has_value() || memory.GpuWritten(*device, sizeof(descriptor)); - const bool pending = - !split.Ran(memory.WritesTarget(split, qmd_address, sizeof(descriptor))); - if (exact && (written || pending)) { - lost = written ? Lost::GpuWritten : Lost::Pending; - } - exact = exact && !written && !pending; + } else if (const auto unknown = ReadDescriptor(split, memory, qmd_address, descriptor); + unknown && !lost) { + lost = unknown; } // A release this signal ran before writes into the QMD. for (const Release& earlier : releases) { - if (exact && earlier.address + 16 > qmd_address && + if (!lost && earlier.address + 16 > qmd_address && earlier.address < qmd_address + sizeof(descriptor)) { lost = Lost::ReleasedInto; } - exact = exact && (earlier.address + 16 <= qmd_address || - earlier.address >= qmd_address + sizeof(descriptor)); } return true; }; @@ -132,13 +158,12 @@ class FrontQmdView { return true; }; scheduler.Signal(address, argument, read, release, [](const Descriptor&) { return true; }); - Nx2AcquireStats::NoteQmdSignal(lost); - if (!exact) { + if (lost) { synced = false; - split.NoteAnyWrite(); - checks_unvalued = true; // TEST + Unvalued(split, *lost); return; } + Nx2AcquireStats::NoteQmdSignal(std::nullopt); for (const Release& noted : releases) { NoteRelease(split, memory, noted.address, noted.payload, noted.flags); } @@ -151,6 +176,63 @@ class FrontQmdView { u32 flags; }; + /// An inline QMD a LOAD_INLINE_QMD_DATA sent writes (NoteInlineWrite). + struct InlineWrite { + u64 device{}; + u64 position{}; + Descriptor qmd{}; + bool overwritten{}; ///< A release of the same entry writes into it after + }; + + /// A signal the front can't value: its releases may write guest memory anywhere (they change + /// no mappings). + void Unvalued(GpuSplit& split, Nx2AcquireStats::QmdLost lost) { + split.NoteUnplacedWrite(); + checks_unvalued = true; // TEST + Nx2AcquireStats::NoteQmdSignal(lost); + } + + /// Reads into `descriptor` the QMD at `qmd_address` as the back will read it running the entry + /// just sent, or tells why the front can't. + template + [[nodiscard]] std::optional ReadDescriptor( + const GpuSplit& split, const Memory& memory, GPUVAddr qmd_address, + Descriptor& descriptor) const { + using Lost = Nx2AcquireStats::QmdLost; + memory.Read(qmd_address, descriptor.data(), sizeof(descriptor)); + const auto device = memory.Device(qmd_address); + if (!device || memory.GpuWritten(*device, sizeof(descriptor))) { + return Lost::GpuWritten; + } + // The last write sent there, placed or not (the inline write covers the whole QMD). + const u64 target = memory.WritesTarget(split, qmd_address, sizeof(descriptor)); + if (split.Ran(target)) { + return std::nullopt; + } + if (const InlineWrite* written = QueuedInlineWrite(split, *device); + written != nullptr && written->position == target) { + descriptor = written->qmd; + return std::nullopt; + } + return target == split.LastUnplacedWrite() ? Lost::Unplaced : Lost::Pending; + } + + /// The last inline QMD sent to the QMD at `device` that the back has not written yet. + [[nodiscard]] const InlineWrite* QueuedInlineWrite(const GpuSplit& split, u64 device) const { + const size_t count = (std::min)(inline_writes_count, inline_writes.size()); + for (size_t i = 1; i <= count; ++i) { + const InlineWrite& written = + inline_writes[(inline_writes_count - i) % inline_writes.size()]; + if (split.Ran(written.position)) { + break; // The back wrote it, and the older ones + } + if (written.device == device) { + return written.overwritten ? nullptr : &written; + } + } + return nullptr; + } + /// A release the back runs with the entry just sent, as ProcessNativeMethod's: what the word /// holds once the back ran it. template @@ -159,16 +241,27 @@ class FrontQmdView { const bool four_words = (flags >> 30) == 0; const bool reduction = (flags & (1U << 26)) != 0; const bool maximum = !reduction && !four_words && (flags & (1U << 19)) != 0; + const u64 size = four_words ? 16 : sizeof(u32); const auto device = memory.Device(address); // The word a reduction or a maximum combines the payload with, before this write. std::optional base; if (device && (reduction || maximum)) { base = WordAt(split, memory, address, *device); } - memory.NoteWrite(split, address, four_words ? 16 : sizeof(u32)); + memory.NoteWrite(split, address, size); if (!device) { return; } + // An inline QMD this entry wrote before is no longer what memory holds there. + for (size_t i = 1; i <= (std::min)(inline_writes_count, inline_writes.size()); ++i) { + InlineWrite& written = inline_writes[(inline_writes_count - i) % inline_writes.size()]; + if (written.position != split.Position()) { + break; + } + if (written.device < *device + size && *device < written.device + sizeof(Descriptor)) { + written.overwritten = true; + } + } // TEST: what was noted (EDEN_NX2_DRAIN_CHECK, EDEN_NX2_ACQUIRE_STATS). const auto check = [this](u64 word, u32 value, bool known, bool at_least) { using Note = Nx2AcquireStats::QmdNote; @@ -204,8 +297,8 @@ class FrontQmdView { } /// The word the back finds at `address` (`device`) when it runs the entry just sent, where - /// the front knows it: the last release sent there while the back may not have run it, else - /// memory, once nothing queued may write there. + /// the front knows it: the last release sent there, if nothing sent since may have written + /// over it, else memory, once the back ran every write sent there. template [[nodiscard]] std::optional WordAt(const GpuSplit& split, const Memory& memory, GPUVAddr address, u64 device) const { @@ -213,23 +306,29 @@ class FrontQmdView { if (memory.InSyncpointShim(address, sizeof(u32))) { return std::nullopt; } + // The last write sent to the word's granule, placed or not. + const u64 target = memory.WritesTarget(split, address, sizeof(u32)); if (const auto queued = split.QueuedSemaphoreWriteAt(device)) { - if (queued->known && !queued->at_least && - queued->position >= split.LastUnplacedWrite()) { + if (queued->known && !queued->at_least && queued->position >= target) { return queued->payload; } return std::nullopt; } - if (memory.GpuWritten(device, sizeof(u32)) || - !split.Ran(memory.WritesTarget(split, address, sizeof(u32)))) { + if (memory.GpuWritten(device, sizeof(u32)) || !split.Ran(target)) { return std::nullopt; } return memory.ReadWord(address); } Tegra::NativeQmdScheduler scheduler{Tegra::NativeQmdScheduler::OwnTable{}}; + /// The table the engines of every channel share, which the back's scheduling uses. + Tegra::NativeQmdScheduler back_table; std::vector releases; bool synced{}; + /// The last inline QMDs sent, oldest first from inline_writes_count on: a back a millisecond + /// behind has some ten to write, a few times that in long frames. + std::array inline_writes{}; + size_t inline_writes_count{}; /// TEST (EDEN_NX2_DRAIN_CHECK): see TakeChecks. std::vector checks; bool checks_pending{}; diff --git a/src/video_core/gpu_split.cpp b/src/video_core/gpu_split.cpp index 5e4f9d939e..c5bb0ed5a7 100644 --- a/src/video_core/gpu_split.cpp +++ b/src/video_core/gpu_split.cpp @@ -147,8 +147,10 @@ void GpuSplit::Stop() { semaphore_writes_limit = 1024; last_any_write = 0; last_unplaced_write = 0; + last_mapping_change = 0; last_noted_write = 0; write_positions.fill(0); + coarse_positions.fill(0); published_local = 0; consumed_cache = 0; publish_every = PublishEvery; @@ -310,8 +312,17 @@ void ForEachDeviceRange(const Tegra::MemoryManager& memory, GPUVAddr address, u6 } // Anonymous namespace void GpuSplit::NoteWrite(const Tegra::MemoryManager& memory, GPUVAddr address, u64 size) { - if (size > MaxNotedBytes || address + size < address) { - NoteAnyWrite(); + if (size == 0) { + return; + } + if (size > MaxCoarseNotedBytes || address + size < address) { + NoteUnplacedWrite(); + return; + } + if (size > MaxNotedBytes) { + ForEachDeviceRange(memory, address, size, [this](DAddr device, u64 bytes) { + NoteCoarseDeviceWrite(device, bytes); + }); return; } ForEachDeviceRange(memory, address, size, @@ -326,6 +337,14 @@ void GpuSplit::NoteDeviceWrite(DAddr address, u64 size) { last_noted_write = write_position; } +void GpuSplit::NoteCoarseDeviceWrite(DAddr address, u64 size) { + const u64 last = (address + size - 1) >> CoarseGranuleBits; + for (u64 granule = address >> CoarseGranuleBits; granule <= last; ++granule) { + coarse_positions[CoarseSlot(granule)] = write_position; + } + last_noted_write = write_position; +} + u64 GpuSplit::WritesTarget(const Tegra::MemoryManager& memory, GPUVAddr address, u64 size) const { if (size > MaxNotedBytes || address + size < address) { return std::max(last_any_write, last_noted_write); @@ -356,6 +375,10 @@ u64 GpuSplit::DeviceWritesTarget(DAddr address, u64 size) const { for (u64 granule = address >> WriteGranuleBits; granule <= last; ++granule) { target = std::max(target, write_positions[WriteSlot(granule)]); } + const u64 last_coarse = (address + size - 1) >> CoarseGranuleBits; + for (u64 granule = address >> CoarseGranuleBits; granule <= last_coarse; ++granule) { + target = std::max(target, coarse_positions[CoarseSlot(granule)]); + } return target; } diff --git a/src/video_core/gpu_split.h b/src/video_core/gpu_split.h index a1cc9a1fd2..9dea61c85a 100644 --- a/src/video_core/gpu_split.h +++ b/src/video_core/gpu_split.h @@ -133,7 +133,11 @@ class GpuSplit { /// What the sender of an entry tells about the guest memory the back writes running it, for /// the front's reads of guest memory (WaitForWrites, MemoryWritesTarget). enum class Writes : u8 { - Anywhere, ///< Unknown: a read anywhere waits for the entry + /// Unknown: a read anywhere waits for the entry, and it may change the GPU's mappings + /// (unmaps; GPU maps drain the split) + Anywhere, + /// Guest memory anywhere, the mappings not: engine work whose writes nobody placed + Unplaced, /// The GPU caches anywhere (shader stores, copies into buffers), guest memory only through /// them, where GPU work wrote (Nx2MayBeGpuWritten): a read of the caches waits for it, a /// read of other guest memory not (MemoryWritesTarget) @@ -295,10 +299,18 @@ class GpuSplit { return write_position - published_local; } - /// The entry just sent (Writes::Noted) writes [address, address + size) of `memory`. + /// The entry just sent (Writes::Noted) writes [address, address + size) of `memory`, as + /// mapped now. Up to MaxNotedBytes by 256-byte granule, up to MaxCoarseNotedBytes by 64 KiB + /// granule, longer writes count as unplaced. void NoteWrite(const Tegra::MemoryManager& memory, GPUVAddr address, u64 size); - /// The entry just sent may write guest memory anywhere. + /// The entry just sent may write guest memory anywhere and change the GPU's mappings + /// (Writes::Anywhere). void NoteAnyWrite() { + NoteUnplacedWrite(); + last_mapping_change = write_position; + } + /// The entry just sent may write guest memory anywhere (Writes::Unplaced). + void NoteUnplacedWrite() { last_any_write = write_position; last_unplaced_write = write_position; } @@ -309,15 +321,22 @@ class GpuSplit { void NoteWrites(Writes writes) { if (writes == Writes::Anywhere) { NoteAnyWrite(); + } else if (writes == Writes::Unplaced) { + NoteUnplacedWrite(); } else if (writes == Writes::Caches) { NoteCacheWrite(); } } /// The position after the last entry sent that may write guest memory where the front does - /// not know (NoteAnyWrite), outside what the caches hold. + /// not know (NoteAnyWrite, NoteUnplacedWrite), outside what the caches hold. [[nodiscard]] u64 LastUnplacedWrite() const { return (std::min)(last_unplaced_write, write_position); } + /// The position after the last entry sent that may change the GPU's mappings (NoteAnyWrite): + /// once the back ran it, what is mapped now is what the back finds mapped. + [[nodiscard]] u64 LastMappingChange() const { + return (std::min)(last_mapping_change, write_position); + } /// Once the back ran everything before the position returned, no entry sent so far writes /// guest memory in [address, address + size) of `memory` but through the GPU caches /// (Writes::Caches): what the back reads there without them, as the QMD scheduler does, is in @@ -350,18 +369,27 @@ class GpuSplit { static constexpr size_t PublishEvery = 256; static constexpr size_t PublishAfterWait = 8; - /// NoteWrite places writes by 256-byte granule of device memory, up to MaxNotedBytes long - /// (longer ones count as anywhere), in write_positions slots hashed by granule: granules that - /// share a slot share its position, so a read may wait for a write elsewhere, never miss one. - /// 16384 slots: the constant buffer words of a few hundred queued draws share few with a QMD. + /// NoteWrite places writes by 256-byte granule of device memory, up to MaxNotedBytes long, + /// in write_positions slots hashed by granule: granules that share a slot share its position, + /// so a read may wait for a write elsewhere, never miss one. 16384 slots: the constant buffer + /// words of a few hundred queued draws share few with a QMD. Writes up to MaxCoarseNotedBytes + /// (copies, uploads) go by 64 KiB granule into coarse_positions, longer ones count as + /// unplaced. static constexpr u32 WriteGranuleBits = 8; static constexpr u64 MaxNotedBytes = 0x4000; static constexpr u32 WriteSlotBits = 14; + static constexpr u32 CoarseGranuleBits = 16; + static constexpr u64 MaxCoarseNotedBytes = u64{16} << 20; + static constexpr u32 CoarseSlotBits = 12; static size_t WriteSlot(u64 granule) { return static_cast((granule * 0x9E3779B97F4A7C15ULL) >> (64 - WriteSlotBits)); } + static size_t CoarseSlot(u64 granule) { + return static_cast((granule * 0x9E3779B97F4A7C15ULL) >> (64 - CoarseSlotBits)); + } void NoteDeviceWrite(DAddr address, u64 size); + void NoteCoarseDeviceWrite(DAddr address, u64 size); /// The position WaitForWrites waits for. [[nodiscard]] u64 WritesTarget(const Tegra::MemoryManager& memory, GPUVAddr address, u64 size) const; @@ -460,12 +488,16 @@ class GpuSplit { std::unordered_map semaphore_writes; size_t semaphore_writes_limit = 1024; ///< Forget what the back ran past this many /// After the last entry that may write guest memory or the GPU caches anywhere (NoteAnyWrite, - /// NoteCacheWrite), the last one that may write guest memory itself anywhere (NoteAnyWrite), - /// the last one NoteWrite placed, and the last placed write in each slot's granules. + /// NoteUnplacedWrite, NoteCacheWrite), the last one that may write guest memory itself + /// anywhere (NoteAnyWrite, NoteUnplacedWrite), the last one that may change the GPU's + /// mappings (NoteAnyWrite), the last one NoteWrite placed, and the last placed write in each + /// slot's granules, fine and coarse. u64 last_any_write{}; u64 last_unplaced_write{}; + u64 last_mapping_change{}; u64 last_noted_write{}; std::array write_positions{}; + std::array coarse_positions{}; u64 published_local{}; u64 consumed_cache{}; u64 first_unpublished_ticks{}; ///< TEST: when the first word not published yet was sent diff --git a/src/video_core/gpu_thread.cpp b/src/video_core/gpu_thread.cpp index 711c62a5f6..215adb925d 100644 --- a/src/video_core/gpu_thread.cpp +++ b/src/video_core/gpu_thread.cpp @@ -20,6 +20,7 @@ #include "core/frontend/graphics_context.h" #include "video_core/control/scheduler.h" #include "video_core/dma_pusher.h" +#include "video_core/front_qmd.h" #include "video_core/gpu.h" #include "video_core/gpu_thread.h" #include "video_core/gpu_split.h" @@ -90,6 +91,10 @@ void ThreadManager::StartThread(VideoCore::RendererBase& renderer, Core::Fronten last_progress = std::chrono::steady_clock::now(); } } + if (split.Active()) { + // The back may have run everything sent meanwhile. + VideoCommon::FrontQmdView::Get().MaybeTakeOver(split); + } if (!state.queue.TryPop(next)) { // The CPU usually releases the semaphore soon after: poll for half a // millisecond, then in short sleeps. Ocarina of Time waits here ~330 times a @@ -118,8 +123,9 @@ void ThreadManager::StartThread(VideoCore::RendererBase& renderer, Core::Fronten } last_progress = std::chrono::steady_clock::now(); if (split.Active()) { - // Between commands the back may have run out of work. + // Between commands the back may have run out of work, and everything sent. split.PublishSoon(); + VideoCommon::FrontQmdView::Get().MaybeTakeOver(split); // Only command lists are parsed here; everything else runs on the back thread // in order, and so does the signal that the command ran. auto* const rasterizer_ptr = renderer.ReadRasterizer(); @@ -128,7 +134,10 @@ void ThreadManager::StartThread(VideoCore::RendererBase& renderer, Core::Fronten Tegra::Nx2ListStats::NoteListQueued(submit_list->channel, -1); scheduler.Push(system.GPU(), submit_list->channel, std::move(submit_list->entries)); } else if (std::holds_alternative(next.data)) { - split.Defer([gpu = &system.GPU()] { gpu->TickWork(); }); + // The sync operations (the back may run them between any two entries too, + // GpuSplit::RequestTick): flushes write guest memory where GPU work wrote. + split.Defer([gpu = &system.GPU()] { gpu->TickWork(); }, + VideoCommon::GpuSplit::Writes::Caches); } else if (auto* run = std::get_if(&next.data)) { if (run->before_later_reads) { // GPU memory maps unmap caches (UnmapMemory downloads): the back's state. From 3319c785cb54e3a898468adcc61ec26831f99433 Mon Sep 17 00:00:00 2001 From: jpkto <255363096+jpkto@users.noreply.github.com> Date: Sun, 8 Nov 2026 05:20:52 +0100 Subject: [PATCH 041/105] TEST: check the QMD view's descriptors and table, count why it stops EDEN_NX2_DRAIN_CHECK now also checks what the QMD view reads. The back records each descriptor its scheduling reads. The front records the one it took: from memory, from the signal's inline QMD, or from an inline QMD still queued. After the QMD methods, the back compares the two lists together with the releases. Every 512th time, while the view is synced, a copy of the front's table goes along, and the back compares it with the table the engines share. With fences that delay their operations (High GPU accuracy, or the balanced or accurate fence behavior), the back writes maxima and reductions later, so the checks count as not valued there. The QMD part of the "Drain checks" line is now: QMD view: methods valued N with R releases, released otherwise than noted W, with D descriptors read, read otherwise X; tables compared T, differing Y; not valued U W, X and Y must stay 0. The first 16 differing descriptors or tables are logged. EDEN_NX2_ACQUIRE_STATS, "QMD view" line: - "work that may change the mappings queued" is now its own reason, apart from work that may write anywhere. Both say what sent the last such write: CPU functions (RunInOrder), engines the front sends as they are (blits), oversized writes, inline QMDs. - "descriptors read from an inline QMD still queued". - Takeovers by site: at a front's start, between commands, at QMD methods. EDEN_NX2_GPU_PROFILE adds "early N/s" after the late publishes: publishes made because the back waited, before 256 words. Checked with the QMD harness, which now runs the real check functions on its model of the back. Over the four fuzz kinds, 1,476,108 valued methods, 768,232 releases, 1,635,746 descriptors and 2,597 tables were compared: none differed. Each loss to unplaced work or to mappings had its kind. Split harness: 20 runs, 10 pinned to one CPU, 10 with EDEN_NX2_GPU_PROFILE, all passed. A small program printed the new report lines. Co-Authored-By: Claude Opus 5.5 --- src/video_core/dma_pusher.cpp | 2 + src/video_core/engines/kepler_compute.cpp | 41 ++++-- src/video_core/front_qmd.h | 49 +++++-- src/video_core/gpu_split.cpp | 7 + src/video_core/gpu_split.h | 42 ++++++ src/video_core/gpu_thread.cpp | 7 +- src/video_core/native_qmd.h | 10 ++ src/video_core/nx2_drain_check.h | 166 +++++++++++++++++++--- src/video_core/nx2_profile.h | 10 +- 9 files changed, 284 insertions(+), 50 deletions(-) diff --git a/src/video_core/dma_pusher.cpp b/src/video_core/dma_pusher.cpp index 617331e6b1..6cf0208122 100644 --- a/src/video_core/dma_pusher.cpp +++ b/src/video_core/dma_pusher.cpp @@ -565,6 +565,7 @@ void DmaPusher::CallMethod(u32 argument) { split.Method(subchannel, dma_state.method, argument, dma_state.is_last_call, WantsSegment() ? &segment : nullptr, EngineWrites(*subchannel, engine_type, dma_state.method)); + split.TagUnplaced(VideoCommon::UnplacedKind::Engine); // TEST NoteEngineWrites(engine_type, dma_state.method); return; } @@ -625,6 +626,7 @@ void DmaPusher::CallMultiMethod(const u32* base_start, u32 num_methods) { split.MultiMethod(subchannel, dma_state.method, base_start, num_methods, dma_state.method_count, WantsSegment() ? &segment : nullptr, EngineWrites(*subchannel, engine_type, dma_state.method)); + split.TagUnplaced(VideoCommon::UnplacedKind::Engine); // TEST // The back runs them one by one: a copy engine launch each. const bool launches = engine_type == Engines::EngineTypes::MaxwellDMA && dma_state.method == Engines::MaxwellDMA::LaunchDmaMethod; diff --git a/src/video_core/engines/kepler_compute.cpp b/src/video_core/engines/kepler_compute.cpp index 932ff30960..17f2ac4b32 100644 --- a/src/video_core/engines/kepler_compute.cpp +++ b/src/video_core/engines/kepler_compute.cpp @@ -350,6 +350,11 @@ bool KeplerCompute::ProcessNativeMethod(Core::System& system, u32 method, u32 ar [&](u64 address, NativeQmdScheduler::Descriptor& descriptor) { if (!memory_manager.IsFullyMappedRange(address, sizeof(descriptor))) return false; memory_manager.ReadBlockUnsafe(address, descriptor.data(), sizeof(descriptor)); + if (VideoCommon::Nx2DrainCheck::g_enabled) [[unlikely]] { + // TEST: what the front's QMD view compares the descriptors it read with. + VideoCommon::Nx2DrainCheck::NoteQmdRead(address, descriptor, + VideoCommon::GpuSplit::OnBackThread()); + } if (static u32 logged{}; (descriptor[53] & (1U << 23)) && logged < 8) { ++logged; LOG_INFO(HW_GPU, "Native QMD {:#x} releases: {:08x} | {:08x} | {:08x}", address, @@ -1493,6 +1498,16 @@ struct FrontQmdMemory { return method < 0xe00 ? Writes::Noted : Writes::Unplaced; } +// TEST (EDEN_NX2_DRAIN_CHECK): whether QMD maximum and reduction releases land as the back runs +// them (FenceManager::SignalFence runs its operation at once), for the QMD view check. +[[nodiscard]] bool QmdReleasesLandAtOnce() { + return Settings::IsGPUFenceBehaviorDefault() + ? !Settings::IsGPULevelHigh() + : !Settings::IsGPUFenceBehaviorBalanced() && !Settings::IsGPUFenceBehaviorAccurate(); +} + +using QmdTakeOver = VideoCommon::Nx2AcquireStats::QmdTakeOver; // TEST + } // Anonymous namespace // Split GPU thread with macros on its front half (gpu_split.h, EDEN_NX2_SPLIT=2): the front @@ -1524,7 +1539,7 @@ KeplerCompute::FrontState& KeplerCompute::Front(VideoCommon::GpuSplit& split) { front_macros = true; // A new session's or a restarted split's: the QMDs the back scheduled meanwhile, in the // table the engines share. The back is idle, its table the current one. - GetFrontQmd().TakeOver(native_qmd); + GetFrontQmd().TakeOver(native_qmd, QmdTakeOver::FrontStart); } return *front; } @@ -1594,6 +1609,7 @@ void KeplerCompute::FrontCallMethod(Core::System& system, VideoCommon::GpuSplit& auto& fs = Front(split); if (method >= fs.registers.size()) { split.Method(this, method, argument, is_last_call, segment, FrontWrites(method)); + split.TagUnplaced(VideoCommon::UnplacedKind::Engine); // TEST return; } if (method >= 0xe00) { @@ -1612,15 +1628,16 @@ void KeplerCompute::FrontCallMethod(Core::System& system, VideoCommon::GpuSplit& } FrontApply(fs, method, argument); if (IsQmdMethod(method)) { - GetFrontQmd().MaybeTakeOver(split, native_qmd); + GetFrontQmd().MaybeTakeOver(split, native_qmd, QmdTakeOver::QmdMethod); } split.Method(this, method, argument, is_last_call, segment, FrontWrites(method)); FrontNoteWrites(split, fs, method, argument, false); FrontQmdCheck(split); } -// TEST (EDEN_NX2_DRAIN_CHECK): has the back compare what it releases for the QMD methods just -// sent with what the front's QMD view noted for them. +// TEST (EDEN_NX2_DRAIN_CHECK): has the back compare what it reads and releases for the QMD +// methods just sent with what the front's QMD view read and noted for them, and now and then its +// QMD table with the view's. void KeplerCompute::FrontQmdCheck(VideoCommon::GpuSplit& split) { if (!VideoCommon::Nx2DrainCheck::g_enabled) [[likely]] { return; @@ -1630,8 +1647,14 @@ void KeplerCompute::FrontQmdCheck(VideoCommon::GpuSplit& split) { return; } split.DeferFunction( - [predicted = std::move(checks->releases), valued = checks->valued] { - VideoCommon::Nx2DrainCheck::CheckQmdReleases(predicted, valued); + [predicted = std::move(checks->releases), reads = std::move(checks->reads), + table = std::move(checks->table), valued = checks->valued && QmdReleasesLandAtOnce()] { + VideoCommon::Nx2DrainCheck::CheckQmdReleases(predicted, reads, valued); + if (table && valued) { + const NativeQmdScheduler shared; // The table the engines share + VideoCommon::Nx2DrainCheck::NoteQmdTables(table->SameStates(shared), + table->StateCount()); + } }, VideoCommon::GpuSplit::Writes::Noted); } @@ -1732,6 +1755,7 @@ void KeplerCompute::FrontInlineQmd(VideoCommon::GpuSplit& split, const FrontStat if (!split.Ran(split.LastMappingChange())) { GetFrontQmd().Lose(); split.NoteUnplacedWrite(); + split.TagUnplaced(VideoCommon::UnplacedKind::InlineQmd); // TEST } return; } @@ -1761,13 +1785,14 @@ void KeplerCompute::FrontCallMultiMethod(Core::System& system, VideoCommon::GpuS if (method >= fs.registers.size()) { split.MultiMethod(this, method, base_start, amount, methods_pending, segment, FrontWrites(method)); + split.TagUnplaced(VideoCommon::UnplacedKind::Engine); // TEST return; } for (u32 i = 0; i < amount; ++i) { FrontApply(fs, method, base_start[i]); } if (IsQmdMethod(method)) { - GetFrontQmd().MaybeTakeOver(split, native_qmd); + GetFrontQmd().MaybeTakeOver(split, native_qmd, QmdTakeOver::QmdMethod); } split.MultiMethod(this, method, base_start, amount, methods_pending, segment, FrontWrites(method)); @@ -1800,7 +1825,7 @@ void KeplerCompute::FrontRunMacro(Core::System& system, VideoCommon::GpuSplit& s } FrontApply(fs, address, value); if (IsQmdMethod(address)) { - GetFrontQmd().MaybeTakeOver(split, native_qmd); + GetFrontQmd().MaybeTakeOver(split, native_qmd, QmdTakeOver::QmdMethod); } split.MacroWrite(this, address, value, FrontWrites(address)); FrontNoteWrites(split, fs, address, value, true); diff --git a/src/video_core/front_qmd.h b/src/video_core/front_qmd.h index 52b5996a19..af7c471d86 100644 --- a/src/video_core/front_qmd.h +++ b/src/video_core/front_qmd.h @@ -6,6 +6,7 @@ #include #include #include +#include #include #include @@ -52,29 +53,33 @@ class FrontQmdView { } /// The back ran everything sent and runs nothing (the split drained, or it waits for work): - /// takes over the table `back` shares with the other engines. - void TakeOver(const Tegra::NativeQmdScheduler& back) { + /// takes over the table `back` shares with the other engines. `where`: TEST. + void TakeOver(const Tegra::NativeQmdScheduler& back, Nx2AcquireStats::QmdTakeOver where) { const auto start = std::chrono::steady_clock::now(); // TEST scheduler.CopyStates(back); synced = true; - Nx2AcquireStats::NoteQmdSync(std::chrono::steady_clock::now() - start); + Nx2AcquireStats::NoteQmdSync(where, std::chrono::steady_clock::now() - start); } /// The front, between commands and before QMD methods: takes the back's table over if the /// view lost it and the back has run everything sent. - void MaybeTakeOver(const GpuSplit& split, const Tegra::NativeQmdScheduler& back) { + void MaybeTakeOver(const GpuSplit& split, const Tegra::NativeQmdScheduler& back, + Nx2AcquireStats::QmdTakeOver where) { if (!synced && split.Ran(split.Position())) [[unlikely]] { - TakeOver(back); + TakeOver(back, where); } } - void MaybeTakeOver(const GpuSplit& split) { - MaybeTakeOver(split, back_table); + void MaybeTakeOver(const GpuSplit& split, Nx2AcquireStats::QmdTakeOver where) { + MaybeTakeOver(split, back_table, where); } - /// TEST (EDEN_NX2_DRAIN_CHECK): the releases noted since the last call, and whether every - /// signal since was valued; nothing if no signal was sent since. + /// TEST (EDEN_NX2_DRAIN_CHECK): the releases noted since the last call, the descriptors read + /// for them, and whether every signal since was valued; every 512th time while the view is + /// synced, a copy of its table; nothing if no signal was sent since. struct Checks { std::vector releases; + std::vector reads; + std::shared_ptr table; bool valued; }; [[nodiscard]] std::optional TakeChecks() { @@ -82,8 +87,15 @@ class FrontQmdView { return std::nullopt; } checks_pending = false; - Checks taken{std::move(checks), !checks_unvalued}; + Checks taken{std::move(checks), std::move(check_reads), nullptr, !checks_unvalued}; + if (taken.valued && synced && ++table_checks % 512 == 0) { + auto copy = + std::make_shared(Tegra::NativeQmdScheduler::OwnTable{}); + copy->CopyStates(scheduler); + taken.table = std::move(copy); + } checks.clear(); + check_reads.clear(); checks_unvalued = false; return taken; } @@ -125,7 +137,7 @@ class FrontQmdView { // A change of the mappings still queued may unmap what the back reads or releases to. if (!split.Ran(split.LastMappingChange())) { synced = false; - Unvalued(split, Lost::Unplaced); + Unvalued(split, Lost::Mappings); return; } releases.clear(); @@ -140,6 +152,9 @@ class FrontQmdView { unknown && !lost) { lost = unknown; } + if (checks_pending) { + check_reads.push_back({qmd_address, descriptor}); // TEST + } // A release this signal ran before writes into the QMD. for (const Release& earlier : releases) { if (!lost && earlier.address + 16 > qmd_address && @@ -187,9 +202,14 @@ class FrontQmdView { /// A signal the front can't value: its releases may write guest memory anywhere (they change /// no mappings). void Unvalued(GpuSplit& split, Nx2AcquireStats::QmdLost lost) { + // TEST: what sent the write the view could not read past. + const UnplacedKind kind = lost == Nx2AcquireStats::QmdLost::Mappings + ? split.LastMappingKind() + : split.LastUnplacedKind(); split.NoteUnplacedWrite(); - checks_unvalued = true; // TEST - Nx2AcquireStats::NoteQmdSignal(lost); + split.TagUnplaced(UnplacedKind::QmdLaunch); // TEST + checks_unvalued = true; // TEST + Nx2AcquireStats::NoteQmdSignal(lost, kind); } /// Reads into `descriptor` the QMD at `qmd_address` as the back will read it running the entry @@ -212,6 +232,7 @@ class FrontQmdView { if (const InlineWrite* written = QueuedInlineWrite(split, *device); written != nullptr && written->position == target) { descriptor = written->qmd; + Nx2AcquireStats::NoteQmdInlineRead(); // TEST return std::nullopt; } return target == split.LastUnplacedWrite() ? Lost::Unplaced : Lost::Pending; @@ -331,6 +352,8 @@ class FrontQmdView { size_t inline_writes_count{}; /// TEST (EDEN_NX2_DRAIN_CHECK): see TakeChecks. std::vector checks; + std::vector check_reads; + u64 table_checks{}; bool checks_pending{}; bool checks_unvalued{}; }; diff --git a/src/video_core/gpu_split.cpp b/src/video_core/gpu_split.cpp index c5bb0ed5a7..f35c5db263 100644 --- a/src/video_core/gpu_split.cpp +++ b/src/video_core/gpu_split.cpp @@ -148,6 +148,8 @@ void GpuSplit::Stop() { last_any_write = 0; last_unplaced_write = 0; last_mapping_change = 0; + last_unplaced_kind = UnplacedKind::Other; + last_mapping_kind = UnplacedKind::Other; last_noted_write = 0; write_positions.fill(0); coarse_positions.fill(0); @@ -202,6 +204,10 @@ void GpuSplit::NoteFirstUnpublished() { first_unpublished_ticks = Nx2Profile::Ticks(); } +void GpuSplit::NoteEarlyPublish() { + Nx2Profile::Add(Nx2Profile::Counter::EarlyPublishes, 1); +} + void GpuSplit::NoteLatePublish() { const u64 idle_since = back_idle_since.load(std::memory_order_relaxed); if (idle_since == 0 || first_unpublished_ticks == 0) { @@ -317,6 +323,7 @@ void GpuSplit::NoteWrite(const Tegra::MemoryManager& memory, GPUVAddr address, u } if (size > MaxCoarseNotedBytes || address + size < address) { NoteUnplacedWrite(); + TagUnplaced(UnplacedKind::Oversized); // TEST return; } if (size > MaxNotedBytes) { diff --git a/src/video_core/gpu_split.h b/src/video_core/gpu_split.h index 9dea61c85a..92a7cefff9 100644 --- a/src/video_core/gpu_split.h +++ b/src/video_core/gpu_split.h @@ -32,6 +32,22 @@ class EngineInterface; namespace VideoCommon { +/// TEST (EDEN_NX2_ACQUIRE_STATS): what sent the last write the front could not place +/// (GpuSplit::TagUnplaced), for why the QMD view could not value a signal. +enum class UnplacedKind : u8 { + Other, ///< A deferred call nobody tagged + RunCommand, ///< A function the CPU queued in order (unmaps, events) + Engine, ///< An engine method the front sends as it is (blits) + Oversized, ///< A placed write longer than the coarse slots take + QmdLaunch, ///< A QMD launch the view could not value + InlineQmd, ///< An inline QMD behind a change of the mappings + Count, +}; + +inline constexpr std::array(UnplacedKind::Count)> + unplaced_kind_names{"other", "CPU functions", "engines", "oversized writes", + "QMD launches", "inline QMDs"}; + /// Why the split's front waited for its back (EDEN_NX2_GPU_PROFILE counts each). enum class DrainSite : u32 { Other, @@ -308,11 +324,30 @@ class GpuSplit { void NoteAnyWrite() { NoteUnplacedWrite(); last_mapping_change = write_position; + last_mapping_kind = UnplacedKind::Other; // TEST } /// The entry just sent may write guest memory anywhere (Writes::Unplaced). void NoteUnplacedWrite() { last_any_write = write_position; last_unplaced_write = write_position; + last_unplaced_kind = UnplacedKind::Other; // TEST + } + /// TEST: what sent the entry just sent, if it may write anywhere or change the mappings. + void TagUnplaced(UnplacedKind kind) { + if (last_unplaced_write == write_position) { + last_unplaced_kind = kind; + } + if (last_mapping_change == write_position) { + last_mapping_kind = kind; + } + } + /// TEST: what sent the last entry that may write anywhere (LastUnplacedWrite), and the last + /// that may change the mappings (LastMappingChange). + [[nodiscard]] UnplacedKind LastUnplacedKind() const { + return last_unplaced_kind; + } + [[nodiscard]] UnplacedKind LastMappingKind() const { + return last_mapping_kind; } /// The entry just sent may write the GPU caches anywhere (Writes::Caches). void NoteCacheWrite() { @@ -428,6 +463,9 @@ class GpuSplit { if (unpublished >= publish_every || (unpublished >= PublishAfterWait && back_idle_at.load(std::memory_order_relaxed) == published_local)) { + if (profile_publishes && unpublished < publish_every) [[unlikely]] { + NoteEarlyPublish(); // TEST + } Publish(); } } @@ -438,6 +476,8 @@ class GpuSplit { bool profile_publishes{}; void NoteFirstUnpublished(); void NoteLatePublish(); + /// TEST: Commit publishes before publish_every, because the back waits for the words. + void NoteEarlyPublish(); void WaitForSpace(size_t words); void SelectEngine(Tegra::Engines::EngineInterface* engine) { if (engine != front_engine) [[unlikely]] { @@ -495,6 +535,8 @@ class GpuSplit { u64 last_any_write{}; u64 last_unplaced_write{}; u64 last_mapping_change{}; + UnplacedKind last_unplaced_kind{}; ///< TEST: see TagUnplaced + UnplacedKind last_mapping_kind{}; ///< TEST u64 last_noted_write{}; std::array write_positions{}; std::array coarse_positions{}; diff --git a/src/video_core/gpu_thread.cpp b/src/video_core/gpu_thread.cpp index 215adb925d..27400e46b1 100644 --- a/src/video_core/gpu_thread.cpp +++ b/src/video_core/gpu_thread.cpp @@ -93,7 +93,8 @@ void ThreadManager::StartThread(VideoCore::RendererBase& renderer, Core::Fronten } if (split.Active()) { // The back may have run everything sent meanwhile. - VideoCommon::FrontQmdView::Get().MaybeTakeOver(split); + VideoCommon::FrontQmdView::Get().MaybeTakeOver( + split, VideoCommon::Nx2AcquireStats::QmdTakeOver::BetweenCommands); } if (!state.queue.TryPop(next)) { // The CPU usually releases the semaphore soon after: poll for half a @@ -125,7 +126,8 @@ void ThreadManager::StartThread(VideoCore::RendererBase& renderer, Core::Fronten if (split.Active()) { // Between commands the back may have run out of work, and everything sent. split.PublishSoon(); - VideoCommon::FrontQmdView::Get().MaybeTakeOver(split); + VideoCommon::FrontQmdView::Get().MaybeTakeOver( + split, VideoCommon::Nx2AcquireStats::QmdTakeOver::BetweenCommands); // Only command lists are parsed here; everything else runs on the back thread // in order, and so does the signal that the command ran. auto* const rasterizer_ptr = renderer.ReadRasterizer(); @@ -145,6 +147,7 @@ void ThreadManager::StartThread(VideoCore::RendererBase& renderer, Core::Fronten split.Drain(VideoCommon::DrainSite::Map); } else { split.DeferFunction(std::move(run->func)); + split.TagUnplaced(VideoCommon::UnplacedKind::RunCommand); // TEST } } else if (const auto* flush = std::get_if(&next.data)) { // Writes guest memory with what the caches hold of it. diff --git a/src/video_core/native_qmd.h b/src/video_core/native_qmd.h index bbf4266b44..42dd1ed305 100644 --- a/src/video_core/native_qmd.h +++ b/src/video_core/native_qmd.h @@ -51,6 +51,14 @@ class NativeQmdScheduler { *table = *other.table; } } + /// TEST (EDEN_NX2_DRAIN_CHECK): whether `other` holds the same QMD states, while nothing + /// runs on either; how many QMDs this one holds. + [[nodiscard]] bool SameStates(const NativeQmdScheduler& other) const { + return *table == *other.table; + } + [[nodiscard]] std::size_t StateCount() const { + return table->size(); + } // Optional diagnostics: called with "load"/"count"/"defer"/"run" events. std::function trace; // Optional diagnostics: called with the QMD, slot and raw words of a release it skips. @@ -166,6 +174,8 @@ class NativeQmdScheduler { bool loaded{}; bool scheduled{}; bool completed{}; + + bool operator==(const State&) const = default; ///< TEST: SameStates }; // One table for every channel: a QMD's dependency counter is state of the QMD itself, and // title 1 schedules a QMD on one channel while another decrements it (a decrement that diff --git a/src/video_core/nx2_drain_check.h b/src/video_core/nx2_drain_check.h index 6555ac040c..c900adc238 100644 --- a/src/video_core/nx2_drain_check.h +++ b/src/video_core/nx2_drain_check.h @@ -19,6 +19,7 @@ #include "common/common_types.h" #include "common/logging.h" +#include "video_core/gpu_split.h" #include "video_core/nx2.h" // EDEN_NX2_DRAIN_CHECK: what the split GPU thread's front would see without some of its waits for @@ -34,7 +35,10 @@ // the drained word does not meet must stay 0; such a pass is corrected (the channel waits), and // a wait the drained word meets ends (counted: a drain would have passed or resumed it); // - the releases the front noted for the QMD launches it valued (FrontQmdView), against the ones -// the back then ran, compared on the back right after the QMD methods. Wrong ones must stay 0. +// the back then ran, compared on the back right after the QMD methods, and the descriptors the +// front read for them (from memory, the signal's inline QMD or a queued one) against the ones +// the back read. Every 512th time, the front's whole QMD table against the back's. Wrong ones +// must stay 0. namespace VideoCommon::Nx2DrainCheck { // A plain flag: the front checks it at every command list and acquire. @@ -66,6 +70,13 @@ struct State { std::atomic qmd_wrong{}; std::atomic qmd_unvalued{}; std::atomic qmd_logged{}; + /// Descriptors compared for the valued methods, and those the back read otherwise; QMD tables + /// compared, and those that differed. + std::atomic qmd_reads{}; + std::atomic qmd_reads_wrong{}; + std::atomic qmd_reads_logged{}; + std::atomic qmd_tables{}; + std::atomic qmd_tables_wrong{}; std::mutex released_mutex; std::unordered_set released; ///< Device addresses of the semaphores the GPU released std::chrono::steady_clock::time_point last_report{}; ///< The front's @@ -136,12 +147,24 @@ struct QmdRelease { bool operator==(const QmdRelease&) const = default; }; +/// A QMD the scheduling read (NativeQmdScheduler::Signal's read): its GPU address and words. +struct QmdRead { + u64 address; + std::array words; +}; + /// The releases the back ran for QMD launches since the last check (the back's thread only). inline std::vector& BackQmdReleases() { static std::vector released; return released; } +/// The QMDs the back's scheduling read since the last check (the back's thread only). +inline std::vector& BackQmdReads() { + static std::vector reads; + return reads; +} + /// The split's back runs a release of a QMD launch (KeplerCompute::ProcessNativeMethod). inline void NoteQmdRelease(u64 device_address, u32 value, bool on_back) { if (on_back) { @@ -149,17 +172,67 @@ inline void NoteQmdRelease(u64 device_address, u32 value, bool on_back) { } } -/// The back, right after QMD methods the front sent: what it released for them against what the -/// front noted (`predicted`, in order), or nothing to compare where the front could not value -/// them (`valued` false). -inline void CheckQmdReleases(std::span predicted, bool valued) { +/// The split's back's scheduling read a QMD (KeplerCompute::ProcessNativeMethod). +inline void NoteQmdRead(u64 address, const std::array& words, bool on_back) { + if (on_back) { + BackQmdReads().push_back({address, words}); + } +} + +/// The back compared the front's QMD table with its own at the same place (`same`). +inline void NoteQmdTables(bool same, size_t states) { + State& state = Get(); + state.qmd_tables.fetch_add(1, std::memory_order_relaxed); + if (!same) { + state.qmd_tables_wrong.fetch_add(1, std::memory_order_relaxed); + if (state.qmd_reads_logged.fetch_add(1, std::memory_order_relaxed) < 16) { + LOG_ERROR(HW_GPU, "QMD view check: the front's table ({} QMDs) differs from the back's", + states); + } + } +} + +/// The back, right after QMD methods the front sent: what it read and released for them against +/// what the front read and noted (`front_reads`, `predicted`, in order), or nothing to compare +/// where the front could not value them (`valued` false). +inline void CheckQmdReleases(std::span predicted, + std::span front_reads, bool valued) { State& state = Get(); auto& released = BackQmdReleases(); + auto& back_reads = BackQmdReads(); if (!valued) { state.qmd_unvalued.fetch_add(1, std::memory_order_relaxed); released.clear(); + back_reads.clear(); return; } + state.qmd_reads.fetch_add(back_reads.size(), std::memory_order_relaxed); + bool same_reads = back_reads.size() == front_reads.size(); + for (size_t i = 0; same_reads && i < back_reads.size(); ++i) { + same_reads = back_reads[i].address == front_reads[i].address && + back_reads[i].words == front_reads[i].words; + } + if (!same_reads) { + state.qmd_reads_wrong.fetch_add(1, std::memory_order_relaxed); + if (state.qmd_reads_logged.fetch_add(1, std::memory_order_relaxed) < 16) { + std::string text; + for (size_t i = 0; i < (std::max)(back_reads.size(), front_reads.size()); ++i) { + const QmdRead* back = i < back_reads.size() ? &back_reads[i] : nullptr; + const QmdRead* front = i < front_reads.size() ? &front_reads[i] : nullptr; + text += fmt::format(" | {:#x} by the back, {:#x} by the front", + back ? back->address : 0, front ? front->address : 0); + for (size_t word = 0; back && front && word < 64; ++word) { + if (back->words[word] != front->words[word]) { + text += fmt::format(", word {} {:#x} for {:#x}", word, back->words[word], + front->words[word]); + break; + } + } + } + LOG_ERROR(HW_GPU, "QMD view check: the back read other descriptors{}", text); + } + } + back_reads.clear(); state.qmd_methods.fetch_add(1, std::memory_order_relaxed); state.qmd_releases.fetch_add(released.size(), std::memory_order_relaxed); bool same = released.size() == predicted.size(); @@ -262,16 +335,22 @@ inline void MaybeReport() { const u64 qmd_releases = take(state.qmd_releases); const u64 qmd_wrong = take(state.qmd_wrong); const u64 qmd_unvalued = take(state.qmd_unvalued); + const u64 qmd_reads = take(state.qmd_reads); + const u64 qmd_reads_wrong = take(state.qmd_reads_wrong); + const u64 qmd_tables = take(state.qmd_tables); + const u64 qmd_tables_wrong = take(state.qmd_tables_wrong); LOG_INFO(HW_GPU, "Drain checks in {:.1f} s: command lists that did not wait for writers sent after the " "last sync {}, GPU-written by those writers {} | drained acquires at addresses the " "GPU never released {}, at ones it released {} | drained polls of suspended acquires " "at addresses never released {}, at released ones {} | semaphore view: acquires " "passed {}, suspended {}; polls resumed {}, kept waiting {} | QMD view: methods " - "valued {} with {} releases, released otherwise than noted {}; not valued {}", + "valued {} with {} releases, released otherwise than noted {}, with {} descriptors " + "read, read otherwise {}; tables compared {}, differing {}; not valued {}", seconds, unwaited, written, acquires_never, acquires_released, polls_never, polls_released, acquire_passes, acquire_waits, poll_passes, poll_waits, qmd_methods, - qmd_releases, qmd_wrong, qmd_unvalued); + qmd_releases, qmd_wrong, qmd_reads, qmd_reads_wrong, qmd_tables, qmd_tables_wrong, + qmd_unvalued); } } // namespace VideoCommon::Nx2DrainCheck @@ -314,10 +393,19 @@ enum class Releaser : u32 { /// Why the front's QMD view could not value a signal (FrontQmdView::Signal). enum class QmdLost : u32 { NotSynced, ///< Its table is not the back's yet (the back has not caught up since) - Unplaced, ///< Work that may write guest memory anywhere (or map) was still queued + Unplaced, ///< Work that may write guest memory anywhere was still queued GpuWritten, ///< A QMD it read is on a page GPU work wrote Pending, ///< A write into a QMD it read was still queued ReleasedInto, ///< A release of the same signal writes into a QMD it read + Mappings, ///< Work that may change the GPU's mappings was still queued + Count, +}; + +/// Where the front's QMD view took the back's table over (FrontQmdView::TakeOver). +enum class QmdTakeOver : u32 { + FrontStart, ///< An engine's front started, with the back drained + BetweenCommands, ///< The back had run everything sent (GPU thread loop) + QmdMethod, ///< The same, at a QMD method Count, }; @@ -357,8 +445,12 @@ struct State { u64 passed_by_qmd{}; ///< Of the passes by a queued release, at words QMD launches release u64 qmd_valued{}; std::array(QmdLost::Count)> qmd_lost{}; + /// Of qmd_lost[Unplaced] and qmd_lost[Mappings]: by what sent the last such write + std::array(UnplacedKind::Count)> qmd_lost_unplaced{}; + std::array(UnplacedKind::Count)> qmd_lost_mappings{}; std::array(QmdNote::Count)> qmd_notes{}; - u64 qmd_syncs{}; + u64 qmd_inline_reads{}; ///< Descriptors read from an inline QMD still queued + std::array(QmdTakeOver::Count)> qmd_syncs{}; std::chrono::steady_clock::duration qmd_sync_time{}; std::unordered_map noted; ///< Releaser bits by word std::chrono::steady_clock::time_point last_report{}; @@ -391,16 +483,29 @@ inline void NoteRelease(u64 device_address, Releaser releaser) { } } -/// The QMD view valued a signal (`lost` empty) or not. -inline void NoteQmdSignal(std::optional lost) { +/// The QMD view valued a signal (`lost` empty) or not; `kind`: what sent the last write that +/// may write anywhere (Unplaced) or change the mappings (Mappings). +inline void NoteQmdSignal(std::optional lost, UnplacedKind kind = UnplacedKind::Other) { if (!g_enabled) { return; } State& state = Get(); - if (lost) { - ++state.qmd_lost[static_cast(*lost)]; - } else { + if (!lost) { ++state.qmd_valued; + return; + } + ++state.qmd_lost[static_cast(*lost)]; + if (*lost == QmdLost::Unplaced) { + ++state.qmd_lost_unplaced[static_cast(kind)]; + } else if (*lost == QmdLost::Mappings) { + ++state.qmd_lost_mappings[static_cast(kind)]; + } +} + +/// The QMD view read a descriptor from an inline QMD still queued. +inline void NoteQmdInlineRead() { + if (g_enabled) { + ++Get().qmd_inline_reads; } } @@ -411,11 +516,11 @@ inline void NoteQmdNote(QmdNote note) { } } -/// The QMD view took the back's table over, in `time`. -inline void NoteQmdSync(std::chrono::steady_clock::duration time) { +/// The QMD view took the back's table over, at `where`, in `time`. +inline void NoteQmdSync(QmdTakeOver where, std::chrono::steady_clock::duration time) { if (g_enabled) { State& state = Get(); - ++state.qmd_syncs; + ++state.qmd_syncs[static_cast(where)]; state.qmd_sync_time += time; } } @@ -524,14 +629,29 @@ inline void MaybeReport() { state.waits_over_1ms); const auto& lost = state.qmd_lost; const auto& notes = state.qmd_notes; + const auto& syncs = state.qmd_syncs; + const auto kinds = [](const auto& counts) { + std::string text; + for (size_t kind = 0; kind < counts.size(); ++kind) { + if (counts[kind] != 0) { + text += fmt::format("{}{} {}", text.empty() ? " (" : ", ", + unplaced_kind_names[kind], counts[kind]); + } + } + return text.empty() ? text : text + ")"; + }; LOG_INFO(HW_GPU, "QMD view in {:.1f} s (EDEN_NX2_ACQUIRE_STATS): signals valued {}; not valued: table " - "not synced yet {}, work that may write anywhere queued {}, QMD on a GPU-written page " - "{}, a write into the QMD queued {}, a release of the signal into it {} | releases " - "noted exact {}, at least {}, unknown {} | tables taken over {} in {:.3f} ms", - seconds, state.qmd_valued, lost[0], lost[1], lost[2], lost[3], lost[4], notes[0], - notes[1], notes[2], state.qmd_syncs, - std::chrono::duration(state.qmd_sync_time).count()); + "not synced yet {}, work that may write anywhere queued {}{}, work that may change " + "the mappings queued {}{}, QMD on a GPU-written page {}, a write into the QMD queued " + "{}, a release of the signal into it {} | descriptors read from an inline QMD still " + "queued {} | releases noted exact {}, at least {}, unknown {} | tables taken over " + "{} in {:.3f} ms (at a front's start {}, between commands {}, at QMD methods {})", + seconds, state.qmd_valued, lost[0], lost[1], kinds(state.qmd_lost_unplaced), lost[5], + kinds(state.qmd_lost_mappings), lost[2], lost[3], lost[4], state.qmd_inline_reads, + notes[0], notes[1], notes[2], syncs[0] + syncs[1] + syncs[2], + std::chrono::duration(state.qmd_sync_time).count(), syncs[0], + syncs[1], syncs[2]); auto noted_words = std::move(state.noted); state = State{}; state.noted = std::move(noted_words); diff --git a/src/video_core/nx2_profile.h b/src/video_core/nx2_profile.h index 8f6cd2e3bd..821491df6d 100644 --- a/src/video_core/nx2_profile.h +++ b/src/video_core/nx2_profile.h @@ -111,7 +111,7 @@ inline u64 Ticks() { struct ThreadData { std::array(Zone::Count)> exclusive_ns{}; std::array(Zone::Count)> count{}; - std::array counters{}; ///< See Counter + std::array counters{}; ///< See Counter std::array(DrainSite::Count)> drain_ns{}; std::array(DrainSite::Count)> drain_count{}; u64 child_ns{}; ///< Time of the zones nested in the current one @@ -170,6 +170,7 @@ enum class Counter : u32 { LatePublishTicks, LatePublishes, ResnapshotWords, ///< TEST: command list words copied again at resumes + EarlyPublishes, ///< TEST: publishes to a back waiting for work, before publish_every }; /// Method calls on this thread by method (3D registers on the back, macros on the front), and @@ -289,12 +290,13 @@ inline void Report(const char* thread_name = "GPU thread") { line += fmt::format(" | acquires {:.0f} ({:.0f} waited)", static_cast(data.counters[6]) * 1e9 / window, static_cast(data.counters[7]) * 1e9 / window); - if (data.counters[9] != 0 || data.counters[10] != 0) { + if (data.counters[9] != 0 || data.counters[10] != 0 || data.counters[11] != 0) { // TEST: the back idle while words it could run waited on the front. - line += fmt::format(" | publishes to an idle back late by {:.1f} ms ({:.0f}/s) | " - "resnapshot words {:.0f}/s", + line += fmt::format(" | publishes to an idle back late by {:.1f} ms ({:.0f}/s), early " + "{:.0f}/s | resnapshot words {:.0f}/s", static_cast(data.counters[8]) * 1000.0 / window_ticks, static_cast(data.counters[9]) * 1e9 / window, + static_cast(data.counters[11]) * 1e9 / window, static_cast(data.counters[10]) * 1e9 / window); } std::string drains; From b964e6b40c71ad8083539101cf2dab5da95d702d Mon Sep 17 00:00:00 2001 From: jpkto <255363096+jpkto@users.noreply.github.com> Date: Sun, 8 Nov 2026 05:00:09 +0100 Subject: [PATCH 042/105] video_core: profile the 3D methods on the path unprofiled runs take With EDEN_NX2_GPU_PROFILE set, Maxwell3D turned its fast method path off, and the split's back thread wrapped each 3D method call in a zone with two time stamp reads. Ocarina of Time makes about 6 million of these calls a second. So profiled runs measured a slower path than the game normally runs, and much of what they measured was the measuring. A benchmark that runs the real GpuSplit::Execute and Maxwell3D on a method stream shaped like Ocarina's (null rasterizer) shows the gap: on the test machine a method took 78 to 90 ns profiled and 21 to 24 ns unprofiled. On the 7800X3D the same overhead is roughly 70 to 150 ms of the back thread's 913 ms a second in test49. Now: - fast_maxwell3d no longer depends on the profile. - CallMethod counts the 3D register writes (the write and same-value counters and the per-method counts) as CallMethodSlow did, for both paths. CallConstBufferWords counts each word. - The calls are timed by sampling (Nx2Profile::SampledScope). Outside every other zone, one call in about 32 is timed, at random intervals of 1 to 63 calls, and it stands for the calls since the last timed one. The cost of the two time stamp reads is taken off each sample, since the untimed calls don't pay it. A sample over 100,000 ticks (an interrupted call) counts once. - Calls made inside another zone (a run of words, a back macro, the unsplit GPU thread's method zone) are that zone's time, as before. What the profile shows changes in two ways. The "method" zone is now an estimate of the normal path's time. And the sunk state writes that the back applies before a method (ConsumeSink) are no longer inside it: they count as "other". Tested: - Compile checks; clang strict warnings on the changed lines. - Split harness: 20 of 20 runs, and 10 of 10 pinned to one CPU. - The maxwell_fast_method test compiles. - Benchmark, profiled: 303 instead of 351 instructions per method, 23 instead of 80 to 90 ns. Unprofiled: 254.5 instead of 252.5 instructions. - The sampled estimate against timing every call (same build, interval 1), 24 runs each: median 32 and 32.5 ticks per method. Rebased onto upstream #714, which made EDEN_NX2_3D_METHODS the Maxwell3D member trace_first_method_writes: the fast path test uses that member. Co-Authored-By: Claude Opus 5.5 --- src/video_core/engines/maxwell_3d.cpp | 41 +++++++--- src/video_core/engines/maxwell_3d.h | 5 ++ src/video_core/nx2_profile.h | 108 +++++++++++++++++++++++++- 3 files changed, 142 insertions(+), 12 deletions(-) diff --git a/src/video_core/engines/maxwell_3d.cpp b/src/video_core/engines/maxwell_3d.cpp index b7094415de..aa1a8a3272 100644 --- a/src/video_core/engines/maxwell_3d.cpp +++ b/src/video_core/engines/maxwell_3d.cpp @@ -60,8 +60,8 @@ Maxwell3D::Maxwell3D(MemoryManager& memory_manager_) execution_mask.reset(); for (size_t i = 0; i < execution_mask.size(); i++) execution_mask[i] = IsMethodExecutable(u32(i)); - fast_maxwell3d = !record_write_history && - !trace_first_method_writes && !VideoCommon::Nx2Profile::Enabled(); + // Profiled runs take the same path: CallMethod counts and times its calls itself. + fast_maxwell3d = !record_write_history && !trace_first_method_writes; } Maxwell3D::~Maxwell3D() = default; @@ -480,6 +480,14 @@ void Maxwell3D::CallMethod(Core::System& system, u32 method, u32 method_argument if (front_macros && !VideoCommon::GpuSplit::OnBackThread()) { ReleaseFront(); } + if (VideoCommon::Nx2Profile::Enabled()) [[unlikely]] { + return ProfiledCallMethod(system, method, method_argument, is_last_call); + } + DispatchMethod(system, method, method_argument, is_last_call); +} + +void Maxwell3D::DispatchMethod(Core::System& system, u32 method, u32 method_argument, + bool is_last_call) { if (!fast_maxwell3d || executing_macro != 0 || method >= MacroRegistersStart || method == MAXWELL3D_REG_INDEX(shadow_ram_control)) { return CallMethodSlow(system, method, method_argument, is_last_call); @@ -490,6 +498,20 @@ void Maxwell3D::CallMethod(Core::System& system, u32 method, u32 method_argument CallMethodFast(method, method_argument, is_last_call); } +// EDEN_NX2_GPU_PROFILE: the path unprofiled runs take, counted. GpuSplit's back calls the 3D engine +// outside every zone (fast_maxwell3d), millions of times a second: a sample of those calls is +// timed (SampledScope). Called inside another zone (a back macro, a run of words, the unsplit GPU +// thread's method zone), the call is that zone's time, as before. +void Maxwell3D::ProfiledCallMethod(Core::System& system, u32 method, u32 method_argument, + bool is_last_call) { + const VideoCommon::Nx2Profile::SampledScope scope{VideoCommon::Nx2Profile::Zone::Method}; + if (method < MacroRegistersStart) { + VideoCommon::Nx2Profile::CountRegisterWrite(method, + regs.reg_array[method] == method_argument); + } + DispatchMethod(system, method, method_argument, is_last_call); +} + namespace { // Executable methods the draw manager does nothing for (ProcessDirtyRegisters is all they do). @@ -632,8 +654,15 @@ void Maxwell3D::CallConstBufferWords(Core::System& system, const u32* methods, } return; } + // Counted and timed as CallMethod counts and times each word (EDEN_NX2_GPU_PROFILE). + const VideoCommon::Nx2Profile::SampledScope scope{VideoCommon::Nx2Profile::Zone::Method, count}; + const bool profile = VideoCommon::Nx2Profile::Enabled(); // As CallMethodFast for each word: the first may issue a deferred draw, before any upload. for (u32 i = 0; i < count; ++i) { + if (profile) [[unlikely]] { + VideoCommon::Nx2Profile::CountRegisterWrite(methods[i], + regs.reg_array[methods[i]] == arguments[i]); + } ProcessDirtyRegisters(methods[i], ProcessShadowRam(methods[i], arguments[i])); } memory_manager.WriteWordsCached(address + offset, values.data(), count); @@ -662,14 +691,6 @@ void Maxwell3D::CallMethodSlow(Core::System& system, u32 method, u32 method_argu } ASSERT(method < Regs::NUM_REGS && "Invalid Maxwell3D register, increase the size of the Regs structure"); - if (VideoCommon::Nx2Profile::Enabled()) [[unlikely]] { - VideoCommon::Nx2Profile::Add(VideoCommon::Nx2Profile::Counter::MethodWrites, 1); - const bool same = regs.reg_array[method] == method_argument; - if (same) { - VideoCommon::Nx2Profile::Add(VideoCommon::Nx2Profile::Counter::SameValueWrites, 1); - } - VideoCommon::Nx2Profile::CountMethod(method, same); - } if (method == MAXWELL3D_REG_INDEX(shadow_ram_control) && front_macros) [[unlikely]] { back_control_writes[back_control_count++ % back_control_writes.size()] = { method_argument, static_cast(shadow_state.shadow_ram_control), is_last_call ? 1U : 0U}; diff --git a/src/video_core/engines/maxwell_3d.h b/src/video_core/engines/maxwell_3d.h index 8f38bf5ff5..1a5511b531 100644 --- a/src/video_core/engines/maxwell_3d.h +++ b/src/video_core/engines/maxwell_3d.h @@ -3289,6 +3289,11 @@ class Maxwell3D final : public EngineInterface { void ProcessMethodCall(u32 method, u32 argument, u32 nonshadow_argument, bool is_last_call); + /// CallMethod after the split's checks: CallMethodFast, or CallMethodSlow where it can't. + void DispatchMethod(Core::System& system, u32 method, u32 method_argument, bool is_last_call); + /// CallMethod with EDEN_NX2_GPU_PROFILE: DispatchMethod, counted and timed. + void ProfiledCallMethod(Core::System& system, u32 method, u32 method_argument, + bool is_last_call); /// CallMethod with every check, for the cases and tools CallMethodFast leaves out. void CallMethodSlow(Core::System& system, u32 method, u32 method_argument, bool is_last_call); void CallMethodFast(u32 method, u32 method_argument, bool is_last_call); diff --git a/src/video_core/nx2_profile.h b/src/video_core/nx2_profile.h index 821491df6d..eaadbcfa9e 100644 --- a/src/video_core/nx2_profile.h +++ b/src/video_core/nx2_profile.h @@ -118,6 +118,10 @@ struct ThreadData { u64 window_start{}; u64 window_start_ticks{}; std::optional cpu_start_ns; + u32 depth{}; ///< Zones open (Scope, timed SampledScope) + u32 samples_left{1}; ///< SampledScope: entries until the next timed one + u32 sample_weight{1}; ///< SampledScope: the entries the next timed one stands for + u32 sample_seed{0x2545F491}; }; inline thread_local ThreadData t_data; @@ -128,8 +132,10 @@ class Scope { if (!enabled) { return; } - parent_child_ns = t_data.child_ns; - t_data.child_ns = 0; + auto& data = t_data; + ++data.depth; + parent_child_ns = data.child_ns; + data.child_ns = 0; start = Ticks(); } @@ -143,6 +149,7 @@ class Scope { data.exclusive_ns[index] += elapsed - data.child_ns; ++data.count[index]; data.child_ns = parent_child_ns + elapsed; + --data.depth; } Scope(const Scope&) = delete; @@ -155,6 +162,89 @@ class Scope { u64 parent_child_ns{}; }; +/// The ticks two time stamp reads in a row take: what a timed entry measures on top of its own +/// time (the least of a few tries). +inline u64 TimingOverheadTicks() { + static const u64 overhead = [] { + u64 least = ~u64{0}; + for (int i = 0; i < 64; ++i) { + const u64 first = Ticks(); + const u64 second = Ticks(); + least = (std::min)(least, second - first); + } + return least; + }(); + return overhead; +} + +/// A zone entered millions of times a second, such as the split's back's 3D register writes +/// (Maxwell3D::CallMethod): timing every entry took two time stamp reads each, which cost more +/// than most entries and slowed the very runs it measured. Entered outside every other zone, one +/// entry in about SampleEvery is timed and stands for the entries since the last timed one; all +/// are counted. Entered inside another zone it does nothing: the entry is that zone's time. +class SampledScope { +public: + static constexpr u32 SampleEvery = 32; + /// About 25 us at 4 GHz (time stamp ticks): far longer than these entries take. + static constexpr u64 LongTicks = 100'000; + + /// `entries`: what this entry counts as (a run of constant buffer words counts each word). + explicit SampledScope(Zone zone, u32 entries = 1) { + if (!Enabled()) [[likely]] { + return; + } + auto& data = t_data; + if (data.depth != 0) { + return; + } + index = static_cast(zone); + data.count[index] += entries; + if (--data.samples_left != 0) { + return; + } + weight = data.sample_weight; + // The next interval: 1 to 2 * SampleEvery - 1 entries, so that no fixed period lines up + // with a repeating pattern of entries (xorshift32). + u32 seed = data.sample_seed; + seed ^= seed << 13; + seed ^= seed >> 17; + seed ^= seed << 5; + data.sample_seed = seed; + data.sample_weight = 1 + seed % (2 * SampleEvery - 1); + data.samples_left = data.sample_weight; + ++data.depth; + parent_child_ns = data.child_ns; + data.child_ns = 0; + start = Ticks(); + } + + ~SampledScope() { + if (weight == 0) { + return; + } + const u64 elapsed = Ticks() - start; + auto& data = t_data; + // Less the reads' own cost, which the untimed entries did not pay. + const u64 overhead = TimingOverheadTicks(); + const u64 own = elapsed - data.child_ns; + const u64 measured = own > overhead ? own - overhead : 0; + // An entry this long was most likely interrupted or preempted: it counts once, not for + // the untimed entries too. + data.exclusive_ns[index] += measured > LongTicks ? measured : measured * weight; + data.child_ns = parent_child_ns + elapsed; + --data.depth; + } + + SampledScope(const SampledScope&) = delete; + SampledScope& operator=(const SampledScope&) = delete; + +private: + size_t index{}; + u64 weight{}; ///< 0: not timed + u64 start{}; + u64 parent_child_ns{}; +}; + /// Amounts counted on the GPU thread, logged per second with the zones. enum class Counter : u32 { DownloadBytes, ///< Buffer data copied back from the host GPU at fences @@ -187,6 +277,18 @@ inline void CountMethod(u32 method, bool same) { } } +/// A 3D register write (Maxwell3D::CallMethod below the macro range), `same` if the register +/// held the value already: the write counters and the per-method counts. +inline void CountRegisterWrite(u32 method, bool same) { + if (!Enabled()) { + return; + } + auto& counters = t_data.counters; + ++counters[static_cast(Counter::MethodWrites)]; + counters[static_cast(Counter::SameValueWrites)] += same ? 1 : 0; + CountMethod(method, same); +} + inline void CountDmaRead(u32 macro) { if (Enabled() && macro < t_dma_reads.size()) { ++t_dma_reads[macro]; @@ -329,7 +431,9 @@ inline void Report(const char* thread_name = "GPU thread") { t_method_count = {}; t_method_same = {}; t_dma_reads = {}; + const u32 depth = data.depth; data = {}; + data.depth = depth; data.window_start = now; data.window_start_ticks = now_ticks; data.cpu_start_ns = cpu_now; From 47af075e495a799b77f93d203c8dd3493ded253a Mon Sep 17 00:00:00 2001 From: jpkto <255363096+jpkto@users.noreply.github.com> Date: Sun, 8 Nov 2026 05:09:53 +0100 Subject: [PATCH 043/105] engines: write 3D state registers in one call The split's back thread writes about 5 million 3D registers a second in Ocarina of Time, about half of them state registers (not in execution_mask). Each took four calls: GpuSplit::Execute to CallMethod, then CallMethodFast, then ProcessDirtyRegisters. It also checked two function statics, Nx2Perf::Verify and SkipSameValueWrites. MSVC checks a function static's initialization on every call by reading a thread-local. Now: - CallMethod writes a state register itself (WriteStateRegister), when the fast path applies and neither EDEN_NX2_PERF_VERIFY nor the profile is on. Everything else goes the way it went before. - ProcessShadowRam, ProcessDirtyRegisters and WriteStateRegister are defined in the class, so the hot paths inline them. This includes ConsumeSinkImpl, CallConstBufferWords and the executable methods in CallMethodFast. - EDEN_NX2_PERF_VERIFY and EDEN_NX2_SKIP_SAME are read once at startup into variables. The buffer cache and memory tracker also check Nx2Perf::Verify on hot paths. The write is the same: the shadow RAM step, the same-value check, the deferred instanced draw check, the register and its dirty flags. The condition keeps every method the old dispatch sent elsewhere: macro calls, SET_SHADOW_RAM_CONTROL, a macro collecting parameters, and the debug tools that turn fast_maxwell3d off. Tested: - Method benchmark (the real GpuSplit::Execute and Maxwell3D, Ocarina-like stream): 219 instead of 254.5 instructions per method on average, counted with callgrind. Wall time is about 10% lower on the test machine, which is noisy. GCC's function statics need no thread-local read, so MSVC should gain more. - 400,000 random writes run through a fast-path engine and a slow-path engine, with shadow RAM mode changes, constant buffer words, binds and draws. Registers, shadow RAM, dirty flags and draw state matched at 3,555 checkpoints, with EDEN_NX2_SKIP_SAME on and off and with the profile on. The check reports a mismatch when the shadow RAM step is left out of the state write. - The benchmark with EDEN_NX2_PERF_VERIFY=1 logged no difference. - Compile checks; clang strict warnings on the changed lines. - Split harness: 20 of 20 runs, and 10 of 10 pinned to one CPU. - The maxwell_fast_method test compiles. Rebased onto upstream #714: EDEN_NX2_SKIP_SAME and EDEN_NX2_PERF_VERIFY were function statics, read on first use, after nx2_switches.txt loads. Read at static initialization, as this commit first had it, they would no longer work from that file. So EDEN_NX2_SKIP_SAME is a Maxwell3D member read when the engine is made (as #714 made EDEN_NX2_3D_METHODS), and EDEN_NX2_PERF_VERIFY a plain flag read again by VideoCore::CreateGPU, as upstream reads its diagnostic switches since #714, so it works from nx2_switches.txt too, not only from the environment. Co-Authored-By: Claude Opus 5.5 --- src/video_core/engines/maxwell_3d.cpp | 58 ++++++++++----------------- src/video_core/engines/maxwell_3d.h | 54 +++++++++++++++++++++---- src/video_core/nx2_perf.h | 22 +++++++--- src/video_core/video_core.cpp | 2 + 4 files changed, 86 insertions(+), 50 deletions(-) diff --git a/src/video_core/engines/maxwell_3d.cpp b/src/video_core/engines/maxwell_3d.cpp index aa1a8a3272..09b7160f79 100644 --- a/src/video_core/engines/maxwell_3d.cpp +++ b/src/video_core/engines/maxwell_3d.cpp @@ -316,16 +316,6 @@ size_t Maxwell3D::EstimateIndexBufferSize() { return std::min(memory_manager.GetMemoryLayoutSize(start_address, byte_size * max_size) / byte_size, lower_cap); } -u32 Maxwell3D::ProcessShadowRam(u32 method, u32 argument) { - // Keep track of the register value in shadow_state when requested. - auto const c = shadow_state.shadow_ram_control; - if (c == Regs::ShadowRamControl::Track || c == Regs::ShadowRamControl::TrackWithFilter) - return shadow_state.reg_array[method] = argument; - else if (c == Regs::ShadowRamControl::Replay) - return shadow_state.reg_array[method]; - return argument; -} - void Maxwell3D::ConsumeSinkImpl(Core::System& system) { if (front_macros && !VideoCommon::GpuSplit::OnBackThread()) { ReleaseFront(); @@ -354,18 +344,6 @@ void Maxwell3D::ConsumeSinkImpl(Core::System& system) { method_sink.clear(); } -void Maxwell3D::ProcessDirtyRegisters(u32 method, u32 argument) { - // The instances after the first are drawn together once their loop ends, and they must see - // the state they were begun with: NVN's draw macros reset the base instance (and its copy in - // c[0][0x644]) right after the loop, before the macro returns. - if (draw_manager.HasDeferredDraw() && !DrawManager::IsInstanceLoopMethod(method)) [[unlikely]] { - draw_manager.DrawDeferred(*this, true); - } - regs.reg_array[method] = argument; - for (auto const& table : dirty.tables) - dirty.flags[table[method]] = true; -} - void Maxwell3D::ProcessMethodCall(u32 method, u32 argument, u32 nonshadow_argument, bool is_last_call) { if (GetNativeMacroEngine().native_turing && method == MAXWELL3D_REG_INDEX(clear_surface)) { LOG_DEBUG(HW_GPU, "Native GPU surface clear: value={:#x}", argument); @@ -476,16 +454,6 @@ void Maxwell3D::TraceFirstMethodWrite(u32 method, u32 argument) { } } -void Maxwell3D::CallMethod(Core::System& system, u32 method, u32 method_argument, bool is_last_call) { - if (front_macros && !VideoCommon::GpuSplit::OnBackThread()) { - ReleaseFront(); - } - if (VideoCommon::Nx2Profile::Enabled()) [[unlikely]] { - return ProfiledCallMethod(system, method, method_argument, is_last_call); - } - DispatchMethod(system, method, method_argument, is_last_call); -} - void Maxwell3D::DispatchMethod(Core::System& system, u32 method, u32 method_argument, bool is_last_call) { if (!fast_maxwell3d || executing_macro != 0 || method >= MacroRegistersStart || @@ -498,6 +466,25 @@ void Maxwell3D::DispatchMethod(Core::System& system, u32 method, u32 method_argu CallMethodFast(method, method_argument, is_last_call); } +void Maxwell3D::CallMethod(Core::System& system, u32 method, u32 method_argument, + bool is_last_call) { + if (front_macros && !VideoCommon::GpuSplit::OnBackThread()) { + ReleaseFront(); + } + // About half of the split's back's 3D writes (~5 million a second in Ocarina of Time) are state + // registers: CallMethodFast's state write, here without the calls to it. Not in + // execution_mask, they are not SET_SHADOW_RAM_CONTROL either. + if (fast_maxwell3d && executing_macro == 0 && method < MacroRegistersStart && + !execution_mask[method] && !VideoCommon::Nx2Perf::Verify() && + !VideoCommon::Nx2Profile::Enabled()) [[likely]] { + return WriteStateRegister(method, method_argument); + } + if (VideoCommon::Nx2Profile::Enabled()) [[unlikely]] { + return ProfiledCallMethod(system, method, method_argument, is_last_call); + } + DispatchMethod(system, method, method_argument, is_last_call); +} + // EDEN_NX2_GPU_PROFILE: the path unprofiled runs take, counted. GpuSplit's back calls the 3D engine // outside every zone (fast_maxwell3d), millions of times a second: a sample of those calls is // timed (SampledScope). Called inside another zone (a back macro, a run of words, the unsplit GPU @@ -533,13 +520,10 @@ bool IsStoreOnlyExecutable(u32 method) { // no write history, trace or profile, not a macro call or SET_SHADOW_RAM_CONTROL. The hot // executable methods go straight to their handler. void Maxwell3D::CallMethodFast(u32 method, u32 method_argument, bool is_last_call) { - const u32 argument = ProcessShadowRam(method, method_argument); if (!execution_mask[method]) { - if (!SkipSameValueWrites() || regs.reg_array[method] != argument) { - ProcessDirtyRegisters(method, argument); - } - return; + return WriteStateRegister(method, method_argument); } + const u32 argument = ProcessShadowRam(method, method_argument); ProcessDirtyRegisters(method, argument); constexpr u32 cb_data = MAXWELL3D_REG_INDEX(const_buffer.buffer); if (method - cb_data < 16) { diff --git a/src/video_core/engines/maxwell_3d.h b/src/video_core/engines/maxwell_3d.h index 1a5511b531..61057d0453 100644 --- a/src/video_core/engines/maxwell_3d.h +++ b/src/video_core/engines/maxwell_3d.h @@ -3281,9 +3281,44 @@ class Maxwell3D final : public EngineInterface { void ProcessMacro(Core::System& system, u32 method, const u32* base_start, u32 amount, bool is_last_call); - u32 ProcessShadowRam(u32 method, u32 argument); + /// The value a write leaves in the register, as the shadow RAM mode has it: the argument + /// (tracked in shadow_state when tracking), or the shadow copy when replaying. + u32 ProcessShadowRam(u32 method, u32 argument) { + const auto control = shadow_state.shadow_ram_control; + if (control == Regs::ShadowRamControl::Track || + control == Regs::ShadowRamControl::TrackWithFilter) { + return shadow_state.reg_array[method] = argument; + } + if (control == Regs::ShadowRamControl::Replay) { + return shadow_state.reg_array[method]; + } + return argument; + } - void ProcessDirtyRegisters(u32 method, u32 argument); + /// Stores a register's value and sets its dirty flags. Defined here, as the hot method paths + /// call it for nearly every write. + void ProcessDirtyRegisters(u32 method, u32 argument) { + // The instances after the first are drawn together once their loop ends, and they must + // see the state they were begun with: NVN's draw macros reset the base instance (and its + // copy in c[0][0x644]) right after the loop, before the macro returns. + if (draw_manager.HasDeferredDraw() && !DrawManager::IsInstanceLoopMethod(method)) + [[unlikely]] { + draw_manager.DrawDeferred(*this, true); + } + regs.reg_array[method] = argument; + for (const auto& table : dirty.tables) { + dirty.flags[table[method]] = true; + } + } + + /// CallMethodFast for a state register (not in execution_mask): a write of the value the + /// register holds changes nothing (EDEN_NX2_SKIP_SAME=0 keeps it). + void WriteStateRegister(u32 method, u32 method_argument) { + const u32 argument = ProcessShadowRam(method, method_argument); + if (!SkipSameValueWrites() || regs.reg_array[method] != argument) { + ProcessDirtyRegisters(method, argument); + } + } void ConsumeSinkImpl(Core::System& system) override; @@ -3315,13 +3350,16 @@ class Maxwell3D final : public EngineInterface { void CallMacroMethod(Core::System& system, u32 method, std::span parameters); /// Whether state register writes that keep the register's value are dropped (CallMethod). - static bool SkipSameValueWrites() { - static const bool skip = [] { - const char* value = std::getenv("EDEN_NX2_SKIP_SAME"); - return value == nullptr || value[0] != '0'; - }(); - return skip; + bool SkipSameValueWrites() const { + return skip_same_value_writes; } + /// EDEN_NX2_SKIP_SAME=0 keeps them. Read per engine: a function's static is checked for its + /// initialization on every call (MSVC reads a thread-local for it), nearly every 3D write, + /// and static initialization runs before the frontend loads nx2_switches.txt. + const bool skip_same_value_writes = [] { + const char* value = std::getenv("EDEN_NX2_SKIP_SAME"); + return value == nullptr || value[0] != '0'; + }(); /// Handles writes to the macro uploading register. void ProcessMacroUpload(u32 data); diff --git a/src/video_core/nx2_perf.h b/src/video_core/nx2_perf.h index 0fe16cfb8e..2b17a690c9 100644 --- a/src/video_core/nx2_perf.h +++ b/src/video_core/nx2_perf.h @@ -7,12 +7,24 @@ namespace VideoCommon::Nx2Perf { +[[nodiscard]] inline bool VerifySwitchSet() { + const char* value = std::getenv("EDEN_NX2_PERF_VERIFY"); + return value != nullptr && value[0] == '1'; +} + +/// EDEN_NX2_PERF_VERIFY=1. A plain flag: Maxwell3D::CallMethod checks it on every 3D write, +/// and a function's static is checked for its initialization on every call (MSVC reads a +/// thread-local for it). +inline bool g_verify = VerifySwitchSet(); + +/// Called by VideoCore::CreateGPU: static initialization runs before the frontend loads +/// nx2_switches.txt, so a switch set only there takes effect from that point on. +inline void ReadSwitch() { + g_verify = VerifySwitchSet(); +} + inline bool Verify() { - static const bool verify = [] { - const char* value = std::getenv("EDEN_NX2_PERF_VERIFY"); - return value != nullptr && value[0] == '1'; - }(); - return verify; + return g_verify; } } // namespace VideoCommon::Nx2Perf diff --git a/src/video_core/video_core.cpp b/src/video_core/video_core.cpp index 73831ee5e1..c05250ecbf 100644 --- a/src/video_core/video_core.cpp +++ b/src/video_core/video_core.cpp @@ -17,6 +17,7 @@ #include "video_core/nx2_download_trace.h" #include "video_core/nx2_drain_check.h" #include "video_core/nx2_list_stats.h" +#include "video_core/nx2_perf.h" #include "video_core/nx2_profile.h" #include "video_core/nx2_read_trace.h" #include "video_core/renderer_base.h" @@ -61,6 +62,7 @@ void CreateGPU(std::optional& gpu, Core::Frontend::EmuWindow& emu_wi VideoCommon::ReadTrace::ReadSwitch(); Tegra::Nx2ListStats::ReadSwitches(); VideoCommon::Nx2AcquireStats::ReadSwitch(); + VideoCommon::Nx2Perf::ReadSwitch(); const auto nvdec_value = Settings::values.nvdec_emulation.GetValue(); const bool use_nvdec = nvdec_value != Settings::NvdecEmulation::Off; From 7c382303ce4aeb7299a8ec56555b35ddbdec6d32 Mon Sep 17 00:00:00 2001 From: jpkto <255363096+jpkto@users.noreply.github.com> Date: Sun, 8 Nov 2026 05:13:14 +0100 Subject: [PATCH 044/105] buffer_cache: compare a small uniform buffer where it lies in guest memory The stream path for small uniform buffers runs for most of the uniform buffers of every draw. It copied the buffer's bytes from guest memory into a scratch array, compared them with the bytes its slot streamed last, then copied them into the stream buffer and, when they differed, into the slot. That made three copies and a compare for a changed buffer, and a copy and a compare for an unchanged one. When the range lies in one piece of host memory (GetSpan), the bytes are now compared where they are. Most of these buffers lie within a page. When they differ, the slot takes them, and the stream allocation is copied from the slot. So an unchanged buffer costs a compare and a changed one two copies; each saves one copy. Other ranges (across non-contiguous pages, or unmapped) are gathered into scratch as before. The bound bytes are the same as before. The slot also still holds exactly what its stream allocation holds, even if the guest writes the memory meanwhile: the allocation is filled from the slot, not from guest memory. EDEN_NX2_UBOREUSE is now read once at startup, not by a function static checked per uniform buffer (MSVC reads a thread-local for that check). Tested: - uniform_stream_reuse test, built with Catch2 and run. Its fake memory now has a GetSpan that refuses ranges across 4 KiB, so both paths run (the test checks each took over 1,000 reads). Every bound buffer is checked against guest memory over 20,000 random binds, with mutations, new ticks and resets. It fails when the slot isn't updated on a change. - Microbenchmark of old against new (fake runtime, six slots, 64 MiB stream), in ns per bind: - 256 bytes unchanged: 27.6 to 8.9 - 1 KiB unchanged: 41 to 30; 1 KiB, a quarter changed: 187 to 162 - 2 KiB unchanged: 73 to 49; 2 KiB, all changed: 534 to 310 - small buffers whose bytes all change: about the same. - Compile checks of the buffer cache users; clang strict warnings on the changed lines. Rebased onto upstream #714: EDEN_NX2_UBOREUSE was a function static, read on first use, after nx2_switches.txt loads. It is now a plain flag that VideoCore::CreateGPU reads again, so it keeps working from that file. Co-Authored-By: Claude Opus 5.5 --- src/tests/video_core/uniform_stream_reuse.cpp | 14 ++++++ .../buffer_cache/uniform_stream_reuse.h | 49 +++++++++++++------ src/video_core/video_core.cpp | 2 + 3 files changed, 50 insertions(+), 15 deletions(-) diff --git a/src/tests/video_core/uniform_stream_reuse.cpp b/src/tests/video_core/uniform_stream_reuse.cpp index 6059b70eeb..a0265b123b 100644 --- a/src/tests/video_core/uniform_stream_reuse.cpp +++ b/src/tests/video_core/uniform_stream_reuse.cpp @@ -19,9 +19,21 @@ void Check(bool condition, const char* message) { struct Memory { std::array bytes{}; + u32 gathered{}; ///< Reads through ReadBlockUnsafe + u32 in_place{}; ///< Ranges GetSpan handed out void ReadBlockUnsafe(u64 address, void* destination, u32 size) { Check(address + size <= bytes.size(), "read outside guest memory"); std::memcpy(destination, bytes.data() + address, size); + ++gathered; + } + /// As the device memory manager: a range in one piece of host memory, here one that doesn't + /// cross a 4 KiB page (as if the pages after it were mapped elsewhere). + u8* GetSpan(u64 address, size_t size) { + if (address / 4096 != (address + size - 1) / 4096) { + return nullptr; + } + ++in_place; + return bytes.data() + address; } }; @@ -113,4 +125,6 @@ TEST_CASE("Uniform stream reuse preserves data through mutation and session rese bind(stage, slot, address, size); bind(stage, slot, address, size); } + Check(memory.gathered > 1000 && memory.in_place > 1000, + "the reads did not take both the gathered and the in-place path"); } diff --git a/src/video_core/buffer_cache/uniform_stream_reuse.h b/src/video_core/buffer_cache/uniform_stream_reuse.h index 7ba31c4717..4ee035436a 100644 --- a/src/video_core/buffer_cache/uniform_stream_reuse.h +++ b/src/video_core/buffer_cache/uniform_stream_reuse.h @@ -31,13 +31,24 @@ concept StreamUniformReusable = requires(R& runtime, u32 size) { { runtime.CurrentTick() } -> std::convertible_to; }; +[[nodiscard]] inline bool UniformStreamReuseSwitch() { + const char* const value = std::getenv("EDEN_NX2_UBOREUSE"); + return value == nullptr || value[0] != '0'; +} + /// EDEN_NX2_UBOREUSE=0 turns the reuse off (every small uniform buffer is copied, as before). +/// A plain flag: checked for every uniform buffer of every draw, where a function's static is +/// checked for its initialization on every call (MSVC reads a thread-local for it). +inline bool g_uniform_stream_reuse = UniformStreamReuseSwitch(); + +/// Called by VideoCore::CreateGPU: static initialization runs before the frontend loads +/// nx2_switches.txt, so a switch set only there takes effect from that point on. +inline void ReadUniformStreamReuseSwitch() { + g_uniform_stream_reuse = UniformStreamReuseSwitch(); +} + [[nodiscard]] inline bool UniformStreamReuseEnabled() noexcept { - static const bool enabled = [] { - const char* const value = std::getenv("EDEN_NX2_UBOREUSE"); - return value == nullptr || value[0] != '0'; - }(); - return enabled; + return g_uniform_stream_reuse; } template @@ -97,24 +108,32 @@ template } Reuse& reuse = UniformStreamReuseState(); typename Reuse::Slot& slot = reuse.slots[stage][index]; - u8* const scratch = reuse.scratch.data(); - device_memory.ReadBlockUnsafe(device_addr, scratch, size); - - const bool same_bytes = - slot.size == size && std::memcmp(slot.bytes.data(), scratch, size) == 0; + // Guest memory in one piece of host memory (most of these buffers lie within a page) is + // compared where it is; other ranges are gathered into scratch first. + const u8* guest = nullptr; + if constexpr (requires { device_memory.GetSpan(device_addr, size_t{size}); }) { + guest = device_memory.GetSpan(device_addr, size_t{size}); + } + if (guest == nullptr) { + device_memory.ReadBlockUnsafe(device_addr, reuse.scratch.data(), size); + guest = reuse.scratch.data(); + } + const bool same_bytes = slot.size == size && std::memcmp(slot.bytes.data(), guest, size) == 0; if (same_bytes && slot.tick == runtime.CurrentTick()) { runtime.RebindStreamUniform(slot.ref, size); return true; } + if (!same_bytes) { + // The slot takes the bytes first and the stream gets the slot's: the guest may write its + // memory meanwhile, and the slot must hold what its allocation holds. + slot.size = size; + std::memcpy(slot.bytes.data(), guest, size); + } const std::span span = runtime.BindMappedUniformBuffer(stage, binding_index, size); - std::memcpy(span.data(), scratch, size); + std::memcpy(span.data(), slot.bytes.data(), size); slot.ref = runtime.LastStreamUniformRef(); // After the allocation: the tick the stream region is tagged with is the one current now. slot.tick = runtime.CurrentTick(); - if (!same_bytes) { - slot.size = size; - std::memcpy(slot.bytes.data(), scratch, size); - } return true; } diff --git a/src/video_core/video_core.cpp b/src/video_core/video_core.cpp index c05250ecbf..251b10e679 100644 --- a/src/video_core/video_core.cpp +++ b/src/video_core/video_core.cpp @@ -11,6 +11,7 @@ #include "core/core.h" #include "core/frontend/emu_window.h" #include "core/frontend/graphics_context.h" +#include "video_core/buffer_cache/uniform_stream_reuse.h" #include "video_core/host1x/gpu_device_memory_manager.h" #include "video_core/host1x/host1x.h" #include "video_core/nx2_cmdpage_diag.h" @@ -63,6 +64,7 @@ void CreateGPU(std::optional& gpu, Core::Frontend::EmuWindow& emu_wi Tegra::Nx2ListStats::ReadSwitches(); VideoCommon::Nx2AcquireStats::ReadSwitch(); VideoCommon::Nx2Perf::ReadSwitch(); + VideoCommon::ReadUniformStreamReuseSwitch(); const auto nvdec_value = Settings::values.nvdec_emulation.GetValue(); const bool use_nvdec = nvdec_value != Settings::NvdecEmulation::Off; From 20628c90aeb019640e9c3213f8bd3fde43470a5f Mon Sep 17 00:00:00 2001 From: jpkto <255363096+jpkto@users.noreply.github.com> Date: Sun, 8 Nov 2026 05:28:23 +0100 Subject: [PATCH 045/105] video_core: leave TickWork without its lock when every sync request ran GPU::TickWork runs after every draw and dispatch (about 118,000 times a second in Ocarina of Time) to serve the sync operations other threads request, such as the flush behind a guest read. Nearly always there are none, but each call still locked and unlocked sync_request_mutex to find the queue empty. The fence of the last request (last_sync_fence) is now an atomic, still written under the mutex. TickWork first compares it with the fence of the last request run (current_sync_fence). When they are equal, every request made has run and none is running, and it returns. A request made meanwhile is not lost. It is the same as one queued just after the locked check used to be: it runs at the next tick, and every maker asks for one (TickGPU, or GpuSplit::RequestTick). On the split, the back sees the new fence after taking the tick request: the maker stores the fence before releasing the mutex, and the tick request after that. RunOnGpuThread asks for no tick and still runs at the next draw's. Tested: - Checked that every RequestSyncOperation caller ticks after it, except RunOnGpuThread, as described. - Compile check; clang strict warnings on the changed lines. Co-Authored-By: Claude Opus 5.5 --- src/video_core/gpu.cpp | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/video_core/gpu.cpp b/src/video_core/gpu.cpp index 2c0c79fed0..8efd8bc750 100644 --- a/src/video_core/gpu.cpp +++ b/src/video_core/gpu.cpp @@ -153,7 +153,8 @@ struct GPU::Impl { template [[nodiscard]] u64 RequestSyncOperation(Func&& action) { std::unique_lock lck{sync_request_mutex}; - const u64 fence = ++last_sync_fence; + const u64 fence = last_sync_fence.load(std::memory_order_relaxed) + 1; + last_sync_fence.store(fence, std::memory_order_release); sync_requests.emplace_back(std::forward(action)); return fence; } @@ -179,6 +180,12 @@ struct GPU::Impl { /// Tick pending requests within the GPU. void TickWork() { + // After every draw and dispatch, nearly always with nothing to do: every request made + // has run. One made meanwhile runs at the next tick, which its maker asks for. + if (current_sync_fence.load(std::memory_order_acquire) == + last_sync_fence.load(std::memory_order_acquire)) { + return; + } std::unique_lock lck{sync_request_mutex}; while (!sync_requests.empty()) { auto request = std::move(sync_requests.front()); @@ -463,7 +470,7 @@ struct GPU::Impl { std::list> sync_requests; std::atomic current_sync_fence{}; std::atomic ampere_anchor{}; - u64 last_sync_fence{}; + std::atomic last_sync_fence{}; ///< The last request's fence (written under the mutex) VideoCommon::PendingMaps pending_maps; std::mutex sync_request_mutex; From 20eba75f8066f616b6017019a0baa162a1d5a9d0 Mon Sep 17 00:00:00 2001 From: jpkto <255363096+jpkto@users.noreply.github.com> Date: Sun, 8 Nov 2026 05:26:22 +0100 Subject: [PATCH 046/105] TEST: fine profile zones for the split's back (EDEN_NX2_GPU_PROFILE=2) The back thread's profile leaves large parts unexplained: "other" (102 ms a second in test49), the "draw" zone's own time (39 ms), "bind stage buffers" (106 ms), "update buffers" (55 ms) and "read handles" (41 ms). The 3D methods' handlers were all one "method" zone. With EDEN_NX2_GPU_PROFILE=2 these zones are split further. Each new zone is nested in the zone named, so it takes its time out of that zone: - method: "cb bind", "cb upload" (constant buffer words written to memory), "query report", "counter reset", "render enable". - other: "sink" (state writes held for the next executable method) and "macro end" (a front macro's deferred draw). - draw: "draw setup" (pipeline lookup to Configure, with the cache locks), "draw queries" (query and transform feedback state) and "tick work" (GPU::TickWork after each draw and dispatch). - bind stage buffers: "uniform buffers". - update buffers: "geometry buffers". - read handles: "storage bindings". A second line, "GPU back thread fine counters per second", counts the uniform buffer binds by path (bound again, streamed, cached, other), the bytes they cover, the storage buffers bound and the texture handles read. EDEN_NX2_GPU_PROFILE=1 is unchanged and stays comparable with earlier runs. The fine zones cost more time stamp reads, about 30% more time per method in the method benchmark, so =2 is for the breakdown, not the totals. Tested: - Compile checks of the profile header's users and the changed files; clang strict warnings on the changed lines. - The zone and name lists are checked to have the same length (static_assert). - Method benchmark with =2: the new method zones count exactly the calls the null rasterizer saw (binds, queries, counter resets, conditions). - Split harness: 20 of 20 runs, and 10 of 10 pinned to one CPU. - uniform_stream_reuse test passes. Rebased onto upstream #714: Nx2Profile::ReadSwitch reads EDEN_NX2_GPU_PROFILE=2 again with the profile switch, so the fine zones work from nx2_switches.txt too. Co-Authored-By: Claude Opus 5.5 --- src/video_core/buffer_cache/buffer_cache.h | 25 +++++- .../buffer_cache/uniform_stream_reuse.h | 2 + src/video_core/engines/maxwell_3d.cpp | 11 +++ src/video_core/nx2_profile.h | 83 ++++++++++++++++++- .../renderer_vulkan/vk_graphics_pipeline.cpp | 23 +++-- .../renderer_vulkan/vk_rasterizer.cpp | 18 +++- 6 files changed, 148 insertions(+), 14 deletions(-) diff --git a/src/video_core/buffer_cache/buffer_cache.h b/src/video_core/buffer_cache/buffer_cache.h index 00cc702ba4..0b4cdf8465 100644 --- a/src/video_core/buffer_cache/buffer_cache.h +++ b/src/video_core/buffer_cache/buffer_cache.h @@ -1609,7 +1609,13 @@ void BufferCache

::BindHostGeometryBuffers(bool is_indexed) { template void BufferCache

::BindHostStageBuffers(size_t stage) { - BindHostGraphicsUniformBuffers(stage); + { + const Nx2Profile::FineScope uniform_scope{Nx2Profile::Zone::UniformBuffers}; + BindHostGraphicsUniformBuffers(stage); + } + Nx2Profile::AddFine( + Nx2Profile::FineCounter::StorageBinds, + static_cast(std::popcount(channel_state->enabled_storage_buffers[stage]))); if (channel_state->enabled_storage_buffers[stage] != 0) { BindHostGraphicsStorageBuffers(stage); } @@ -2357,6 +2363,8 @@ void BufferCache

::BindHostGraphicsUniformBuffer(size_t stage, u32 index, u32 } channel_state->fast_bound_uniform_buffers[stage] |= 1u << binding_index; channel_state->uniform_buffer_binding_sizes[stage][binding_index] = size; + Nx2Profile::AddFine(Nx2Profile::FineCounter::UniformOther, 1); + Nx2Profile::AddFine(Nx2Profile::FineCounter::UniformBytes, size); return; } const Binding& binding = channel_state->uniform_buffers[stage][index]; @@ -2404,6 +2412,7 @@ void BufferCache

::BindHostGraphicsUniformBuffer(size_t stage, u32 index, u32 const bool use_fast_buffer = needs_alignment_stream || (has_host_buffer && size <= channel_state->uniform_buffer_skip_cache_size && !memory_tracker.IsRegionGpuModified(device_addr, size)); + Nx2Profile::AddFine(Nx2Profile::FineCounter::UniformBytes, size); if (use_fast_buffer) { if constexpr (IS_OPENGL) { if (runtime.HasFastBufferSubData()) { @@ -2419,6 +2428,7 @@ void BufferCache

::BindHostGraphicsUniformBuffer(size_t stage, u32 index, u32 } const auto span = ImmediateBufferWithData(device_addr, size); runtime.PushFastUniformBuffer(stage, binding_index, span); + Nx2Profile::AddFine(Nx2Profile::FineCounter::UniformOther, 1); return; } } @@ -2431,16 +2441,29 @@ void BufferCache

::BindHostGraphicsUniformBuffer(size_t stage, u32 index, u32 // stream allocation again instead of allocating and copying (uniform_stream_reuse.h). // EDEN_NX2_UBOREUSE=0 turns it off. if (VideoCommon::UniformStreamReuseEnabled()) { + // TEST: rebinds counted (EDEN_NX2_GPU_PROFILE=2). + const u64 rebinds = + VideoCommon::UniformStreamReuseState() + .rebinds; streamed = VideoCommon::BindSmallUniformReusing( runtime, device_memory, stage, index, binding_index, device_addr, size); + const u64 rebound = + VideoCommon::UniformStreamReuseState() + .rebinds - + rebinds; + Nx2Profile::AddFine(Nx2Profile::FineCounter::UniformRebinds, rebound); + Nx2Profile::AddFine(Nx2Profile::FineCounter::UniformStreamed, + streamed ? 1 - rebound : 0); } } if (!streamed) { const std::span span = runtime.BindMappedUniformBuffer(stage, binding_index, size); device_memory.ReadBlockUnsafe(device_addr, span.data(), size); + Nx2Profile::AddFine(Nx2Profile::FineCounter::UniformStreamed, 1); } return; } + Nx2Profile::AddFine(Nx2Profile::FineCounter::UniformCached, 1); // Classic cached path if (SynchronizeBuffer(buffer, device_addr, size)) { ++channel_state->uniform_cache_hits[0]; diff --git a/src/video_core/buffer_cache/uniform_stream_reuse.h b/src/video_core/buffer_cache/uniform_stream_reuse.h index 4ee035436a..43a85244f2 100644 --- a/src/video_core/buffer_cache/uniform_stream_reuse.h +++ b/src/video_core/buffer_cache/uniform_stream_reuse.h @@ -76,6 +76,7 @@ class UniformStreamReuse { std::array, NumStages> slots{}; std::array scratch{}; + u64 rebinds{}; ///< TEST: allocations bound again (EDEN_NX2_GPU_PROFILE=2 counts them) }; template @@ -121,6 +122,7 @@ template const bool same_bytes = slot.size == size && std::memcmp(slot.bytes.data(), guest, size) == 0; if (same_bytes && slot.tick == runtime.CurrentTick()) { runtime.RebindStreamUniform(slot.ref, size); + ++reuse.rebinds; return true; } if (!same_bytes) { diff --git a/src/video_core/engines/maxwell_3d.cpp b/src/video_core/engines/maxwell_3d.cpp index 09b7160f79..f6d1a54ae9 100644 --- a/src/video_core/engines/maxwell_3d.cpp +++ b/src/video_core/engines/maxwell_3d.cpp @@ -317,6 +317,7 @@ size_t Maxwell3D::EstimateIndexBufferSize() { } void Maxwell3D::ConsumeSinkImpl(Core::System& system) { + const VideoCommon::Nx2Profile::FineScope profile_scope{VideoCommon::Nx2Profile::Zone::Sink}; if (front_macros && !VideoCommon::GpuSplit::OnBackThread()) { ReleaseFront(); } @@ -649,6 +650,7 @@ void Maxwell3D::CallConstBufferWords(Core::System& system, const u32* methods, } ProcessDirtyRegisters(methods[i], ProcessShadowRam(methods[i], arguments[i])); } + const VideoCommon::Nx2Profile::FineScope upload_scope{VideoCommon::Nx2Profile::Zone::CbUpload}; memory_manager.WriteWordsCached(address + offset, values.data(), count); regs.const_buffer.offset += static_cast(bytes); } @@ -808,6 +810,8 @@ void Maxwell3D::StampQueryResult(Core::System& system, u64 payload, bool long_qu } void Maxwell3D::ProcessQueryGet() { + const VideoCommon::Nx2Profile::FineScope profile_scope{ + VideoCommon::Nx2Profile::Zone::QueryReport}; VideoCommon::QueryPropertiesFlags flags{}; if (regs.report_semaphore.query.short_query == 0) { flags |= VideoCommon::QueryPropertiesFlags::HasTimeout; @@ -842,6 +846,8 @@ void Maxwell3D::ProcessQueryGet() { } void Maxwell3D::ProcessQueryCondition() { + const VideoCommon::Nx2Profile::FineScope profile_scope{ + VideoCommon::Nx2Profile::Zone::RenderEnable}; if (rasterizer->AccelerateConditionalRendering()) { execute_on = true; return; @@ -908,6 +914,8 @@ void Maxwell3D::ProcessQueryCondition() { } void Maxwell3D::ProcessCounterReset() { + const VideoCommon::Nx2Profile::FineScope profile_scope{ + VideoCommon::Nx2Profile::Zone::CounterReset}; const auto query_type = [clear_report = regs.clear_report_value]() { switch (clear_report) { case Tegra::Engines::Maxwell3D::Regs::ClearReport::ZPassPixelCount: @@ -936,6 +944,7 @@ void Maxwell3D::ProcessSyncPoint() { } void Maxwell3D::ProcessCBBind(size_t stage_index) { + const VideoCommon::Nx2Profile::FineScope profile_scope{VideoCommon::Nx2Profile::Zone::CbBind}; // Bind the buffer currently in CB_ADDRESS to the specified index in the desired shader // stage. const auto& bind_data = regs.bind_groups[stage_index]; @@ -964,6 +973,7 @@ void Maxwell3D::ProcessCBBind(size_t stage_index) { } void Maxwell3D::ProcessCBMultiData(const u32* start_base, u32 amount) { + const VideoCommon::Nx2Profile::FineScope profile_scope{VideoCommon::Nx2Profile::Zone::CbUpload}; // Write the input value to the current const buffer at the current position. const GPUVAddr buffer_address = regs.const_buffer.Address(); ASSERT(buffer_address != 0); @@ -2163,6 +2173,7 @@ void Maxwell3D::FrontVerify(VideoCommon::GpuSplit& split, FrontState& fs) { } void Maxwell3D::FinishFrontMacro(Core::System& system) { + const VideoCommon::Nx2Profile::FineScope profile_scope{VideoCommon::Nx2Profile::Zone::MacroEnd}; // What ProcessMacro and CallMacroMethod do around the macro. ConsumeSink(system); draw_manager.DrawDeferred(*this); diff --git a/src/video_core/nx2_profile.h b/src/video_core/nx2_profile.h index eaadbcfa9e..5c21d65c4f 100644 --- a/src/video_core/nx2_profile.h +++ b/src/video_core/nx2_profile.h @@ -64,6 +64,20 @@ enum class Zone : u32 { ListGpuWait, AcquirePoll, ///< TEST: a suspended acquire's poll (CanResumeAcquire in ResumePending) Resnapshot, ///< TEST: command lists copied again at a resume (DmaPusher::RefreshSnapshot) + // TEST: fine zones, EDEN_NX2_GPU_PROFILE=2 only (FineScope): they split the zones above. + CbBind, ///< Of method: constant buffer binds (Maxwell3D::ProcessCBBind) + CbUpload, ///< Of method: constant buffer words written to memory + QueryReport, ///< Of method: report semaphores (Maxwell3D::ProcessQueryGet) + CounterReset, ///< Of method: counter resets (Maxwell3D::ProcessCounterReset) + RenderEnable, ///< Of method: render enable conditions (Maxwell3D::ProcessQueryCondition) + Sink, ///< State writes held for the next executable method (ConsumeSinkImpl) + MacroEnd, ///< A front macro's end on the back (Maxwell3D::FinishFrontMacro) + DrawSetup, ///< Of draw: after the pipeline lookup, before Configure (cache locks) + DrawQueries, ///< Of draw: query and transform feedback state before recording + TickWork, ///< Of draw and compute: the sync operations after them (GPU::TickWork) + UniformBuffers, ///< Of bind stage buffers: the uniform buffers + GeometryBuffers, ///< Of update buffers: index and vertex buffer binds + StorageBindings, ///< Of read handles: storage buffer addresses from constant buffers Count, }; @@ -76,21 +90,38 @@ inline constexpr std::array(Zone::Count)> zone_ "configure draw", "dynamic state", "draw record", "texture upload", "fence", "fence skipped", "method", "split drain", "split wait", "writer wait", "list submit wait", "list GPU wait", "acquire poll", "resnapshot", + "cb bind", "cb upload", "query report", "counter reset", "render enable", "sink", + "macro end", "draw setup", "draw queries", "tick work", "uniform buffers", + "geometry buffers", "storage bindings", }; +static_assert(zone_names.back() != nullptr, "a zone without a name"); // A plain flag: Enabled() runs several times per method write (no guard to check). inline bool g_enabled = std::getenv("EDEN_NX2_GPU_PROFILE") != nullptr; +/// TEST: EDEN_NX2_GPU_PROFILE=2 adds the fine zones (FineScope) and counters (FineCounter), at +/// the cost of more time stamp reads: =1 keeps the zones runs compare with. +[[nodiscard]] inline bool FineSwitchSet() { + const char* value = std::getenv("EDEN_NX2_GPU_PROFILE"); + return value != nullptr && value[0] == '2'; +} +inline bool g_fine = FineSwitchSet(); + /// Called by VideoCore::CreateGPU: static initialization runs before the frontend loads /// nx2_switches.txt, so a switch set only there takes effect from that point on. inline void ReadSwitch() { g_enabled = std::getenv("EDEN_NX2_GPU_PROFILE") != nullptr; + g_fine = FineSwitchSet(); } inline bool Enabled() { return g_enabled; } +inline bool Fine() { + return g_fine; +} + // Nanoseconds; from the host's own counter where the wall clock reads it directly (the zones // run ~150,000 times a second, and QueryPerformanceCounter cost ~5% of the GPU thread). inline u64 Now() { @@ -108,7 +139,27 @@ inline u64 Ticks() { #endif } +/// TEST: amounts counted with EDEN_NX2_GPU_PROFILE=2, logged per second in a line of their own. +enum class FineCounter : u32 { + UniformRebinds, ///< Small uniform buffers bound again from the same stream allocation + UniformStreamed, ///< Small uniform buffers copied into the stream buffer + UniformCached, ///< Uniform buffers bound from their host buffer + UniformOther, ///< Uniform buffers bound otherwise (root tables, OpenGL, fallbacks) + UniformBytes, ///< Bytes of the uniform buffers bound (all paths) + StorageBinds, ///< Graphics storage buffers bound + TextureHandles, ///< Graphics texture handles read + Count, +}; + +inline constexpr std::array(FineCounter::Count)> + fine_counter_names{ + "uniform rebinds", "uniform streamed", "uniform cached", "uniform other", + "uniform bytes", "storage binds", "texture handles", + }; +static_assert(fine_counter_names.back() != nullptr, "a fine counter without a name"); + struct ThreadData { + std::array(FineCounter::Count)> fine_counters{}; ///< TEST std::array(Zone::Count)> exclusive_ns{}; std::array(Zone::Count)> count{}; std::array counters{}; ///< See Counter @@ -128,7 +179,10 @@ inline thread_local ThreadData t_data; class Scope { public: - explicit Scope(Zone zone_) : zone{zone_}, enabled{Enabled()} { + explicit Scope(Zone zone_) : Scope{zone_, Enabled()} {} + + /// A zone timed only if `on` (a profiled run's choice, such as Fine()). + Scope(Zone zone_, bool on) : zone{zone_}, enabled{on} { if (!enabled) { return; } @@ -162,6 +216,12 @@ class Scope { u64 parent_child_ns{}; }; +/// TEST: a fine zone (EDEN_NX2_GPU_PROFILE=2). +class FineScope : public Scope { +public: + explicit FineScope(Zone zone_) : Scope{zone_, Fine()} {} +}; + /// The ticks two time stamp reads in a row take: what a timed entry measures on top of its own /// time (the least of a few tries). inline u64 TimingOverheadTicks() { @@ -333,6 +393,13 @@ inline void Add(Counter counter, u64 amount) { } } +/// TEST: EDEN_NX2_GPU_PROFILE=2 only. +inline void AddFine(FineCounter counter, u64 amount) { + if (Fine()) [[unlikely]] { + t_data.fine_counters[static_cast(counter)] += amount; + } +} + /// The front waited for the back since `start` (Ticks) for `site`. inline void AddDrain(DrainSite site, u64 start) { if (Enabled()) { @@ -422,6 +489,20 @@ inline void Report(const char* thread_name = "GPU thread") { static_cast(window) / 1e6); } LOG_INFO(HW_GPU, "{} per second:{}", thread_name, line); + if (Fine()) { + // TEST: the fine counters. + std::string fine; + for (size_t i = 0; i < data.fine_counters.size(); ++i) { + if (data.fine_counters[i] != 0) { + fine += fmt::format(" {} {:.0f},", fine_counter_names[i], + static_cast(data.fine_counters[i]) * 1e9 / window); + } + } + if (!fine.empty()) { + fine.pop_back(); + LOG_INFO(HW_GPU, "{} fine counters per second:{}", thread_name, fine); + } + } if (const std::string methods = TopMethods(t_method_count, &t_method_same, window); !methods.empty()) { LOG_INFO(HW_GPU, "{} methods per second:{}", thread_name, methods); } diff --git a/src/video_core/renderer_vulkan/vk_graphics_pipeline.cpp b/src/video_core/renderer_vulkan/vk_graphics_pipeline.cpp index 0cb6e61328..283d82dfe0 100644 --- a/src/video_core/renderer_vulkan/vk_graphics_pipeline.cpp +++ b/src/video_core/renderer_vulkan/vk_graphics_pipeline.cpp @@ -508,15 +508,18 @@ bool GraphicsPipeline::ConfigureImpl(bool is_indexed) { const bool via_header_index{regs.sampler_binding == Maxwell::SamplerBinding::ViaHeaderBinding}; const auto config_stage{[&](size_t stage) LAMBDA_FORCEINLINE { const Shader::Info& info{stage_infos[stage]}; - buffer_cache.UnbindGraphicsStorageBuffers(stage); - if constexpr (Spec::has_storage_buffers) { - size_t ssbo_index{}; - for (const auto& desc : info.storage_buffers_descriptors) { - ASSERT(desc.count == 1); - buffer_cache.BindGraphicsStorageBuffer(stage, ssbo_index, desc.cbuf_index, - desc.cbuf_offset, desc.is_written, - desc.indirect_offset); - ++ssbo_index; + { + const Profile::FineScope storage_scope{Profile::Zone::StorageBindings}; + buffer_cache.UnbindGraphicsStorageBuffers(stage); + if constexpr (Spec::has_storage_buffers) { + size_t ssbo_index{}; + for (const auto& desc : info.storage_buffers_descriptors) { + ASSERT(desc.count == 1); + buffer_cache.BindGraphicsStorageBuffer(stage, ssbo_index, desc.cbuf_index, + desc.cbuf_offset, desc.is_written, + desc.indirect_offset); + ++ssbo_index; + } } } const auto& cbufs{maxwell3d->state.shader_stages[stage].const_buffers}; @@ -608,6 +611,7 @@ bool GraphicsPipeline::ConfigureImpl(bool is_indexed) { config_stage(4); } } + Profile::AddFine(Profile::FineCounter::TextureHandles, views.size()); ASSERT(views.size() == num_image_elements); ASSERT(samplers.size() == num_textures); { @@ -725,6 +729,7 @@ bool GraphicsPipeline::ConfigureImpl(bool is_indexed) { { Profile::Scope scope{Profile::Zone::UpdateBuffers}; buffer_cache.UpdateGraphicsBuffers(is_indexed); + const Profile::FineScope geometry_scope{Profile::Zone::GeometryBuffers}; buffer_cache.BindHostGeometryBuffers(is_indexed); } diff --git a/src/video_core/renderer_vulkan/vk_rasterizer.cpp b/src/video_core/renderer_vulkan/vk_rasterizer.cpp index 467161c2b8..c65c5d1aa1 100644 --- a/src/video_core/renderer_vulkan/vk_rasterizer.cpp +++ b/src/video_core/renderer_vulkan/vk_rasterizer.cpp @@ -544,6 +544,8 @@ void RasterizerVulkan::PrepareDraw(bool is_indexed, Func&& draw_func, bool is_in } SCOPE_EXIT { + const VideoCommon::Nx2Profile::FineScope tick_scope{ + VideoCommon::Nx2Profile::Zone::TickWork}; gpu.TickWork(); }; bool submitted = false; @@ -609,6 +611,9 @@ void RasterizerVulkan::PrepareDraw(bool is_indexed, Func&& draw_func, bool is_in } return; } + // TEST: up to Configure (EDEN_NX2_GPU_PROFILE=2). + std::optional setup_scope; + setup_scope.emplace(Profile::Zone::DrawSetup); if (VideoCommon::OotNetwork::BypassEnabled(Settings::GetCurrentProgramID())) { TrackOotTonemappedImage(*pipeline); } @@ -779,6 +784,7 @@ void RasterizerVulkan::PrepareDraw(bool is_indexed, Func&& draw_func, bool is_in } } } + setup_scope.reset(); if (Profile::Scope scope{Profile::Zone::Configure}; !pipeline->Configure(is_indexed)) { report_failure("configure failed"); TraceRenderState(is_indexed, "configure-failed", nullptr); @@ -832,9 +838,13 @@ void RasterizerVulkan::PrepareDraw(bool is_indexed, Func&& draw_func, bool is_in UpdateDynamicStates(pipeline); } - query_cache.NotifySegment(true); - HandleTransformFeedback(); - query_cache.CounterEnable(VideoCommon::QueryType::ZPassPixelCount64, maxwell3d->regs.zpass_pixel_count_enable); + { + const Profile::FineScope queries_scope{Profile::Zone::DrawQueries}; + query_cache.NotifySegment(true); + HandleTransformFeedback(); + query_cache.CounterEnable(VideoCommon::QueryType::ZPassPixelCount64, + maxwell3d->regs.zpass_pixel_count_enable); + } { Profile::Scope scope{Profile::Zone::DrawRecord}; scheduler.TraceGraphicsPipeline(pipeline->ShaderHashes()); @@ -1755,6 +1765,8 @@ void RasterizerVulkan::DispatchCompute() { // As after a draw: run the sync operations threads wait for, such as a guest read's flush, // rather than only at the next draw. SCOPE_EXIT { + const VideoCommon::Nx2Profile::FineScope tick_scope{ + VideoCommon::Nx2Profile::Zone::TickWork}; gpu.TickWork(); }; VideoCommon::Nx2Profile::Scope profile_scope{VideoCommon::Nx2Profile::Zone::Compute}; From 90e33429812d9d1b5b397aec3f9d3c6a48cfcb2e Mon Sep 17 00:00:00 2001 From: jpkto <255363096+jpkto@users.noreply.github.com> Date: Sun, 8 Nov 2026 05:46:39 +0100 Subject: [PATCH 047/105] video_core: mark the buffer uses of indirect draws, obtained buffers and conditional rendering An upload may run in the upload command buffer, ahead of its submission's work, only when no work recorded before it uses the bytes it writes. The buffer cache learns of such uses from Buffer::MarkUsage, and some GPU work reads or writes guest buffers without calling it: - Indirect draws read their arguments and their count. BindHostDrawIndirectBuffers synchronized both and marked nothing. - ObtainBuffer and ObtainCPUBuffer marked only for MarkAsWritten. These went unmarked: indirect dispatch arguments (DiscardWrite), DMA buffer-to-image copies, query result copies, conditional rendering predicates and their resolve pass (all DoNothing). - Conditional rendering on a guest predicate (CompareValueImpl) reads it again at every render pass that resumes it, in later submissions too. The usage marks were cleared only when TickFrame found a whole buffer idle, so other marks on the same pages usually hid these gaps. Once marks are kept per submission, an upload could run ahead of such a read or write. What changed: - ObtainCPUBuffer marks the range it hands out, for every operation. - BindHostDrawIndirectBuffers marks the arguments and the count. - The query cache keeps the address of a guest predicate. Each time it begins conditional rendering again, it marks the predicate through BufferCache::MarkUsed. MarkUsed finds the buffer without creating one. Marks only keep uploads in order, so nothing that was right before becomes wrong. Tested with syntax checks and strict warnings on the changed lines. The upload reorder check that follows (EDEN_NX2_UPLOAD_REORDER_CHECK) is the in-game proof that no upload runs ahead of a use. Co-Authored-By: Claude Opus 5.5 --- src/video_core/buffer_cache/buffer_cache.h | 25 ++++++++++++++++++- .../buffer_cache/buffer_cache_base.h | 4 +++ .../renderer_vulkan/vk_query_cache.cpp | 11 ++++++++ 3 files changed, 39 insertions(+), 1 deletion(-) diff --git a/src/video_core/buffer_cache/buffer_cache.h b/src/video_core/buffer_cache/buffer_cache.h index 0b4cdf8465..4da52d23e4 100644 --- a/src/video_core/buffer_cache/buffer_cache.h +++ b/src/video_core/buffer_cache/buffer_cache.h @@ -1502,6 +1502,22 @@ bool BufferCache

::DMAClear(GPUVAddr dst_address, u64 amount, u32 value) { return true; } +template +void BufferCache

::MarkUsed(DAddr device_addr, u64 size) { + const u64 page = device_addr >> CACHING_PAGEBITS; + if (device_addr == 0 || size == 0 || page >= page_table.size()) { + return; + } + const BufferId buffer_id = page_table[page]; + if (!buffer_id) { + return; + } + Buffer& buffer = slot_buffers[buffer_id]; + if (buffer.IsInBounds(device_addr, size)) { + buffer.MarkUsage(buffer.Offset(device_addr), size); + } +} + template std::pair BufferCache

::ObtainBuffer(GPUVAddr gpu_addr, u32 size, ObtainBufferSynchronize sync_info, @@ -1528,11 +1544,14 @@ std::pair BufferCache

::ObtainCPUBuffer( break; } + // Each caller records GPU work that reads or writes these bytes now (indirect dispatches, DMA + // copies between images and buffers, query results, conditional rendering): an upload of them + // later in this submission must run after it. + buffer.MarkUsage(buffer.Offset(device_addr), size); switch (post_op) { case ObtainBufferOperation::MarkAsWritten: MarkWrittenBuffer(buffer_id, device_addr, size, false, GpuWaitStats::WriteSite::ImageToBuffer); - buffer.MarkUsage(buffer.Offset(device_addr), size); break; case ObtainBufferOperation::DiscardWrite: { ClearDownload(device_addr, size); @@ -2319,6 +2338,10 @@ void BufferCache

::BindHostDrawIndirectBuffers() { Buffer& buffer = slot_buffers[binding.buffer_id]; TouchBuffer(buffer, binding.buffer_id); SynchronizeBuffer(buffer, binding.device_addr, binding.size); + // The draw reads the arguments (GetDrawIndirectBuffer, GetDrawIndirectCount). + if (binding.buffer_id != NULL_BUFFER_ID) { + buffer.MarkUsage(buffer.Offset(binding.device_addr), binding.size); + } }; if (current_draw_indirect->include_count) { bind_buffer(channel_state->count_buffer_binding); diff --git a/src/video_core/buffer_cache/buffer_cache_base.h b/src/video_core/buffer_cache/buffer_cache_base.h index 3dee3f4ec3..2b6b5eeb5c 100644 --- a/src/video_core/buffer_cache/buffer_cache_base.h +++ b/src/video_core/buffer_cache/buffer_cache_base.h @@ -353,6 +353,10 @@ class BufferCache : public VideoCommon::ChannelSetupCaches ObtainCPUBuffer(DAddr gpu_addr, u32 size, ObtainBufferSynchronize sync_info, ObtainBufferOperation post_op); + + /// GPU work recorded now reads or writes these bytes, of a buffer obtained before (work set + /// up once that keeps reading them, as conditional rendering does). Creates no buffer. + void MarkUsed(DAddr device_addr, u64 size); void FlushCachedWrites(); /// Where the CPU writes into GPU-written memory that were collected but not applied yet diff --git a/src/video_core/renderer_vulkan/vk_query_cache.cpp b/src/video_core/renderer_vulkan/vk_query_cache.cpp index 76d4f129e4..86484e4ffd 100644 --- a/src/video_core/renderer_vulkan/vk_query_cache.cpp +++ b/src/video_core/renderer_vulkan/vk_query_cache.cpp @@ -1419,6 +1419,9 @@ struct QueryCacheRuntimeImpl { VkConditionalRenderingBeginInfoEXT hcr_setup; VkBuffer hcr_buffer; size_t hcr_offset; + /// The guest bytes hcr_setup reads when they are a guest buffer's, 0 when it reads the + /// resolve buffer. Each resume reads them again, in the submission it is recorded in. + DAddr hcr_guest_address{}; bool hcr_is_set; bool is_hcr_running; @@ -1449,6 +1452,7 @@ void QueryCacheRuntime::EndHostConditionalRendering() { impl->is_hcr_running = false; impl->hcr_buffer = VkBuffer{}; impl->hcr_offset = 0; + impl->hcr_guest_address = 0; } void QueryCacheRuntime::PauseHostConditionalRendering() { @@ -1467,6 +1471,11 @@ void QueryCacheRuntime::ResumeHostConditionalRendering() { return; } if (!impl->is_hcr_running) { + if (impl->hcr_guest_address != 0) { + // An upload of the predicate later in this submission must run after this read. + std::scoped_lock lk(impl->buffer_cache.mutex); + impl->buffer_cache.MarkUsed(impl->hcr_guest_address, 8); + } impl->scheduler.Record([hcr_setup = impl->hcr_setup](vk::CommandBuffer cmdbuf) { cmdbuf.BeginConditionalRenderingEXT(hcr_setup); }); @@ -1497,6 +1506,7 @@ void QueryCacheRuntime::HostConditionalRenderingCompareValueImpl(VideoCommon::Lo } impl->hcr_setup.buffer = impl->hcr_buffer; impl->hcr_setup.offset = impl->hcr_offset; + impl->hcr_guest_address = object.address; impl->hcr_setup.flags = is_equal ? VK_CONDITIONAL_RENDERING_INVERTED_BIT_EXT : 0; impl->hcr_is_set = true; impl->is_hcr_running = false; @@ -1526,6 +1536,7 @@ void QueryCacheRuntime::HostConditionalRenderingCompareBCImpl(DAddr address, boo impl->conditional_resolve_pass->Resolve(*impl->hcr_resolve_buffer, to_resolve, to_resolve_offset, compare_to_zero); impl->hcr_setup.buffer = *impl->hcr_resolve_buffer; + impl->hcr_guest_address = 0; impl->hcr_setup.offset = 0; impl->hcr_setup.flags = is_equal ? 0 : VK_CONDITIONAL_RENDERING_INVERTED_BIT_EXT; impl->hcr_is_set = true; From 3406508a7540025ab1ae98d09b31f8cfbd7322dc Mon Sep 17 00:00:00 2001 From: jpkto <255363096+jpkto@users.noreply.github.com> Date: Sun, 8 Nov 2026 05:46:46 +0100 Subject: [PATCH 048/105] TEST: mark the read check's own download copy as a use of its bytes EDEN_NX2_READ_CHECK copies the bytes of a read a second time, from the same point of the GPU's work, and compares them with what the read's download brought. Unlike the read's own copies, that second copy marked nothing. Once uploads are reordered within a submission, an upload of those bytes recorded later in the submission could run before it, and the check would compare against bytes written after the read. The copy is marked now, as the read's copies are. Co-Authored-By: Claude Opus 5.5 --- src/video_core/buffer_cache/buffer_cache.h | 1 + 1 file changed, 1 insertion(+) diff --git a/src/video_core/buffer_cache/buffer_cache.h b/src/video_core/buffer_cache/buffer_cache.h index 4da52d23e4..3fb1dadcbb 100644 --- a/src/video_core/buffer_cache/buffer_cache.h +++ b/src/video_core/buffer_cache/buffer_cache.h @@ -726,6 +726,7 @@ void BufferCache

::RecordReadDownload(std::span> r for (Copy& entry : check_copies) { Buffer& buffer = slot_buffers[entry.buffer_id]; entry.copy.dst_offset += check->staging.offset; + buffer.MarkUsage(entry.copy.src_offset, entry.copy.size); runtime.CopyBuffer(check->staging.buffer, buffer, std::span{&entry.copy, 1}, true); check->copies.push_back(BufferCopy{ .src_offset = buffer.CpuAddr() + entry.copy.src_offset, From ddf01f582ae8ab60725790dd54fe77aa8b9499df Mon Sep 17 00:00:00 2001 From: jpkto <255363096+jpkto@users.noreply.github.com> Date: Sun, 8 Nov 2026 05:46:46 +0100 Subject: [PATCH 049/105] vulkan: put uploads ahead of a submission's work unless that work uses their bytes The GPU cost runs of Ocarina of Time count about 48 render pass breaks a frame at MappedUploadMemory. In nearly all of them a pass on the same framebuffer ends only so that an upload can run in order, and then begins again. Each break costs about 12 us of GPU time plus the attachments' store and load. An upload went in order whenever its buffer's usage bits (UsageTracker) were set over its bytes. TickFrame cleared those bits only for buffers whose last use had finished, and a buffer used in every frame never got cleared. The classifier run counted 17,900 in-order uploads a second: - 11,485 whose pages' last use had finished; - 1,669 last used in an earlier submission that was still running; - 4,762 used in the same command buffer. Because of this, the count depended on timing. Batching on and off both averaged about 48 breaks a frame, and single reports of each run ranged from 42 to 55. The 51.7 and 42.0 compared earlier were one report from each run. The slower classifier run, where more buffers were idle at frame ends, had 19. What changed: - Buffer keeps SubmissionUses (buffer_cache/upload_order.h). It holds the byte ranges that work recorded in the current submission uses, merged and sorted, plus the newest earlier submission whose work used the buffer. MarkUsage tracks into it, and it starts over when the tick changes. TickFrame no longer walks every buffer to reset bits. - DecideUploadOrder replaces CanReorderUpload. An upload stays in order when work recorded before it in its submission uses a byte it writes. Otherwise it goes ahead, together with the newest earlier submission that used the buffer. Either way its bytes are then tracked as used, so a second upload of them in the same submission stays in order behind it. - Scheduler::RecordReorderedUpload collects reordered uploads. SubmitExecution records them into the upload command buffer as one command with one copy per destination (ReorderedUploadBatch), before the submission takes its tick. - Before, a reordered upload's bytes had no marks left only when their buffer's last use had finished, so the upload command buffer needed nothing but its last barrier. Now an earlier submission may still be using those bytes. The batch therefore begins with a barrier (all commands, memory writes, to transfer reads and writes) unless the host GPU has finished the newest earlier submission that used one of its buffers. The timeline semaphore is read at most once a submission. - Uploads that stay in order keep their batching (RecordUpload). - OpenGL and the test runtime place every upload in order, as before. Why no upload runs before a use of its bytes: - A reordered upload runs before its submission's whole main command buffer. The decision keeps it in order when work recorded before it in that submission uses any byte it writes. SubmissionUses tracks those bytes exactly. Past 1024 disjoint ranges it merges neighbors, which counts more bytes as used, never fewer. - Earlier submissions: the barrier's first scope is all work submitted before it on the queue. It is left out only once the host has seen the newest earlier submission that used the buffer finish, so all earlier uses of those bytes are done. - No two copies in a batch write the same bytes, since the first upload is a use. So the copies need no order among themselves and can share one vkCmdCopyBuffer per destination. - The upload command buffer's last barrier (transfer to all commands) orders the copies before the main command buffer and everything after it. Tested: - Unit tests (src/tests/video_core/upload_order.cpp): SubmissionUses against a per-byte model over random marks and ticks, merging past the range limit, and the decisions. Mutating the merge, the previous tick, the recent-range cache or the coarsening makes them fail. - A lavapipe harness under synchronization validation, with submissions built as SubmitExecution builds them (upload command buffer, then main): - A reordered upload of bytes that the previous submission's compute shader read or wrote, or a transfer read, gives no report with the barrier and one without it. After a host wait on that submission's fence, it gives no report without the barrier. - Leaving out the upload command buffer's last barrier is reported. - A simulation over 288 submissions: compute reads and writes, transfer reads, and 2,146 uploads placed by DecideUploadOrder as the runtime places them (1,778 reordered; 250 batches behind the barrier, 37 without it). Its reads and the final buffers equal a CPU model that applies every operation in recording order, with no validation report. - Controls: placing every upload ahead gives 3,248 wrong words; leaving out the barrier gives 28 reports; not tracking uploads as uses puts 78 pairs of copies of the same bytes in one batch. - The upload batching, deferred clear pass and GPU cost harnesses pass as before. - Syntax checks of the changed sources, both template instantiations included, and strict warnings on the changed lines. - A benchmark of the tracking against the old bits, on draw-like marks into a 224 MiB buffer: 38 to 53 ns a mark against 29 to 49 ns. The new one is slower when a submission marks many distinct ranges and faster when the same ranges come back. Co-Authored-By: Claude Opus 5.5 --- src/tests/CMakeLists.txt | 1 + src/tests/video_core/buffer_cache_ports.cpp | 8 +- src/tests/video_core/upload_order.cpp | 192 ++++++++++++++++++ src/video_core/CMakeLists.txt | 1 + src/video_core/buffer_cache/buffer_cache.h | 8 +- src/video_core/buffer_cache/upload_order.h | 157 ++++++++++++++ .../renderer_opengl/gl_buffer_cache.cpp | 2 +- .../renderer_opengl/gl_buffer_cache.h | 8 +- src/video_core/renderer_vulkan/upload_batch.h | 84 ++++++++ .../renderer_vulkan/vk_buffer_cache.cpp | 68 ++++--- .../renderer_vulkan/vk_buffer_cache.h | 33 +-- .../renderer_vulkan/vk_scheduler.cpp | 40 +++- src/video_core/renderer_vulkan/vk_scheduler.h | 15 ++ 13 files changed, 557 insertions(+), 60 deletions(-) create mode 100644 src/tests/video_core/upload_order.cpp create mode 100644 src/video_core/buffer_cache/upload_order.h diff --git a/src/tests/CMakeLists.txt b/src/tests/CMakeLists.txt index 6d97c0c1e7..83f3a9bdcf 100644 --- a/src/tests/CMakeLists.txt +++ b/src/tests/CMakeLists.txt @@ -161,6 +161,7 @@ add_executable(tests video_core/turing_macro_effects.cpp video_core/turing_macro_ram.cpp video_core/uniform_stream_reuse.cpp + video_core/upload_order.cpp video_core/usage_tracker.cpp video_core/shader_invalidation.cpp video_core/descriptor_table.cpp diff --git a/src/tests/video_core/buffer_cache_ports.cpp b/src/tests/video_core/buffer_cache_ports.cpp index 4da78ed02f..40f15e96b8 100644 --- a/src/tests/video_core/buffer_cache_ports.cpp +++ b/src/tests/video_core/buffer_cache_ports.cpp @@ -22,6 +22,7 @@ #include "tests/core/system_test_context.h" #include "video_core/buffer_cache/buffer_cache.h" #include "video_core/buffer_cache/memory_tracker_base.h" +#include "video_core/buffer_cache/upload_order.h" #include "video_core/control/channel_state.h" #include "video_core/gpu.h" #include "video_core/nx2_list_stats.h" @@ -80,8 +81,9 @@ struct Runtime { return submitted_tick + 1; } void WaitSubmittedTick(u64) {} - bool CanReorderUpload(const Buffer&, std::span) const { - return false; + VideoCommon::UploadOrder DecideUploadOrder(const Buffer&, + std::span) const { + return {}; } Staging DownloadStagingBuffer(u64 size, bool = false) { // Kept until the end: read downloads hold theirs past the next request. @@ -123,7 +125,7 @@ struct Runtime { } } void CopyBuffer(Buffer& dst, Buffer* src, std::span copies, bool, - bool) { + VideoCommon::UploadOrder) { for (const auto& copy : copies) { REQUIRE(copy.src_offset + copy.size <= src->bytes.size()); REQUIRE(copy.dst_offset + copy.size <= dst.bytes.size()); diff --git a/src/tests/video_core/upload_order.cpp b/src/tests/video_core/upload_order.cpp new file mode 100644 index 0000000000..4e438eb11a --- /dev/null +++ b/src/tests/video_core/upload_order.cpp @@ -0,0 +1,192 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later + +#include +#include +#include +#include +#include + +#include "video_core/buffer_cache/upload_order.h" + +namespace { + +struct Copy { + u64 dst_offset; + u64 size; +}; + +/// The tick of each byte's last use, and every tick that used any byte. +struct ByteModel { + explicit ByteModel(size_t size) : last(size, 0) {} + + void Track(u64 offset, u64 size, u64 tick) { + for (u64 i = offset; i < offset + size; ++i) { + last[i] = tick; + } + if (size != 0) { + ticks.push_back(tick); + } + } + + bool IsUsed(u64 offset, u64 size, u64 tick) const { + for (u64 i = offset; i < offset + size; ++i) { + if (last[i] == tick) { + return true; + } + } + return false; + } + + /// The newest tick before `tick` that used a byte of [offset, offset + size). + u64 NewestByteUseBefore(u64 offset, u64 size, u64 tick) const { + u64 newest = 0; + for (u64 i = offset; i < offset + size; ++i) { + if (last[i] < tick) { + newest = (std::max)(newest, last[i]); + } + } + return newest; + } + + u64 NewestTickBefore(u64 tick) const { + u64 newest = 0; + for (const u64 used : ticks) { + if (used < tick) { + newest = (std::max)(newest, used); + } + } + return newest; + } + + std::vector last; + std::vector ticks; +}; + +} // Anonymous namespace + +TEST_CASE("Submission uses keep the newest submission's byte ranges", "[video_core]") { + VideoCommon::SubmissionUses uses; + REQUIRE_FALSE(uses.IsUsed(0, 16, 1)); + REQUIRE(uses.NewestTickBefore(1) == 0); + + uses.Track(256, 64, 5); + uses.Track(320, 64, 5); // touches the end of one: merged + uses.Track(1024, 16, 5); + uses.Track(1008, 16, 5); // touches the start of one: merged + REQUIRE(uses.RangeCount() == 2); + REQUIRE(uses.IsUsed(256, 1, 5)); + REQUIRE(uses.IsUsed(383, 1, 5)); + REQUIRE_FALSE(uses.IsUsed(384, 624, 5)); // the gap between the ranges + REQUIRE(uses.IsUsed(384, 625, 5)); + REQUIRE_FALSE(uses.IsUsed(192, 64, 5)); // ends where a range begins + REQUIRE(uses.IsUsed(192, 65, 5)); + REQUIRE(uses.IsUsed(1000, 100, 5)); + REQUIRE_FALSE(uses.IsUsed(992, 16, 5)); + REQUIRE_FALSE(uses.IsUsed(1040, 16, 5)); + REQUIRE_FALSE(uses.IsUsed(256, 64, 4)); // another submission's + REQUIRE_FALSE(uses.IsUsed(256, 0, 5)); + REQUIRE(uses.NewestTickBefore(5) == 0); + REQUIRE(uses.NewestTickBefore(6) == 5); + + // A range covering several merges them. + uses.Track(300, 800, 5); + REQUIRE(uses.RangeCount() == 1); + REQUIRE(uses.IsUsed(1050, 4, 5)); + + // The next submission starts empty; the one before is the newest earlier use. + uses.Track(4096, 4, 7); + REQUIRE(uses.RangeCount() == 1); + REQUIRE_FALSE(uses.IsUsed(256, 64, 7)); + REQUIRE(uses.IsUsed(4096, 4, 7)); + REQUIRE(uses.NewestTickBefore(7) == 5); + REQUIRE(uses.NewestTickBefore(8) == 7); +} + +TEST_CASE("Submission uses match a byte model", "[video_core]") { + constexpr u64 Size = 16384; + std::mt19937 random{0x0de7}; + for (u32 run = 0; run < 24; ++run) { + VideoCommon::SubmissionUses uses; + ByteModel model{Size}; + u64 tick = 1; + for (u32 step = 0; step < 3000; ++step) { + const u32 roll = random() % 100; + if (roll < 3) { + tick += 1 + random() % 3; + } + const u64 size = random() % 8 == 0 ? random() % 4096 : 1 + random() % 96; + const u64 offset = random() % (Size - size); + if (roll < 60) { + uses.Track(offset, size, tick); + model.Track(offset, size, tick); + continue; + } + // Ask about the submission being recorded, as an upload does. + REQUIRE(uses.IsUsed(offset, size, tick) == model.IsUsed(offset, size, tick)); + const u64 bound = uses.NewestTickBefore(tick); + REQUIRE(bound == model.NewestTickBefore(tick)); + REQUIRE(bound >= model.NewestByteUseBefore(offset, size, tick)); + const VideoCommon::UploadOrder order = + VideoCommon::DecideUploadOrder(uses, std::array{Copy{offset, size}}, tick); + REQUIRE(order.reorder == !model.IsUsed(offset, size, tick)); + if (order.reorder) { + REQUIRE(order.prior_tick == bound); + } + } + } +} + +TEST_CASE("Submission uses only grow past their range limit", "[video_core]") { + constexpr u64 Size = (VideoCommon::SubmissionUses::MaxRanges + 64) * 64; + VideoCommon::SubmissionUses uses; + ByteModel model{Size}; + std::mt19937 random{0x51de}; + // Disjoint 16-byte ranges in a random order, more of them than the limit. + std::vector slots(Size / 64); + for (u64 i = 0; i < slots.size(); ++i) { + slots[i] = i * 64; + } + std::shuffle(slots.begin(), slots.end(), random); + for (const u64 offset : slots) { + uses.Track(offset, 16, 3); + model.Track(offset, 16, 3); + REQUIRE(uses.RangeCount() <= VideoCommon::SubmissionUses::MaxRanges); + } + // Every used byte still is; unused ones may be counted as used, never the reverse. + for (u64 offset = 0; offset < Size; offset += 8) { + if (model.IsUsed(offset, 8, 3)) { + REQUIRE(uses.IsUsed(offset, 8, 3)); + } + } + REQUIRE_FALSE(uses.IsUsed(Size, 64, 3)); +} + +TEST_CASE("Uploads stay in order behind uses in their submission", "[video_core]") { + VideoCommon::SubmissionUses uses; + // Submission 4 drew from bytes 0..256. + uses.Track(0, 256, 4); + // Submission 5: the first upload goes ahead, after submission 4's work. + const auto first = VideoCommon::DecideUploadOrder(uses, std::array{Copy{128, 64}}, 5); + REQUIRE(first.reorder); + REQUIRE(first.prior_tick == 4); + uses.Track(128, 64, 5); + // A draw then reads other bytes; an upload of those must stay behind it. + uses.Track(512, 64, 5); + REQUIRE_FALSE(VideoCommon::DecideUploadOrder(uses, std::array{Copy{540, 4}}, 5).reorder); + // An upload of the first upload's bytes stays behind it too. + REQUIRE_FALSE(VideoCommon::DecideUploadOrder(uses, std::array{Copy{160, 8}}, 5).reorder); + // One copy of several is enough. + REQUIRE_FALSE(VideoCommon::DecideUploadOrder( + uses, std::array{Copy{1024, 8}, Copy{560, 8}, Copy{2048, 8}}, 5) + .reorder); + // Untouched bytes go ahead. + const auto other = VideoCommon::DecideUploadOrder(uses, std::array{Copy{1024, 8}}, 5); + REQUIRE(other.reorder); + REQUIRE(other.prior_tick == 4); + // A buffer no work used: ahead, after nothing. + const VideoCommon::SubmissionUses unused; + const auto fresh = VideoCommon::DecideUploadOrder(unused, std::array{Copy{0, 4}}, 5); + REQUIRE(fresh.reorder); + REQUIRE(fresh.prior_tick == 0); +} diff --git a/src/video_core/CMakeLists.txt b/src/video_core/CMakeLists.txt index 84c624d4f6..3321aedd43 100644 --- a/src/video_core/CMakeLists.txt +++ b/src/video_core/CMakeLists.txt @@ -19,6 +19,7 @@ add_library(video_core STATIC buffer_cache/buffer_cache.cpp buffer_cache/buffer_cache.h buffer_cache/memory_tracker_base.h + buffer_cache/upload_order.h buffer_cache/usage_tracker.h buffer_cache/virtual_range_cache.h buffer_cache/word_manager.h diff --git a/src/video_core/buffer_cache/buffer_cache.h b/src/video_core/buffer_cache/buffer_cache.h index 3fb1dadcbb..8885549677 100644 --- a/src/video_core/buffer_cache/buffer_cache.h +++ b/src/video_core/buffer_cache/buffer_cache.h @@ -3769,8 +3769,8 @@ void BufferCache

::MappedUploadMemory([[maybe_unused]] Buffer& buffer, // Apply the staging offset copy.src_offset += upload_staging.offset; } - const bool can_reorder = runtime.CanReorderUpload(buffer, copies); - runtime.CopyBuffer(buffer, upload_staging.buffer, copies, true, can_reorder); + const auto order = runtime.DecideUploadOrder(buffer, copies); + runtime.CopyBuffer(buffer, upload_staging.buffer, copies, true, order); } } @@ -3812,8 +3812,8 @@ void BufferCache

::InlineMemoryImplementation(DAddr dest_address, size_t copy_ }}; u8* const src_pointer = upload_staging.mapped_span.data(); std::memcpy(src_pointer, inlined_buffer.data(), copy_size); - const bool can_reorder = runtime.CanReorderUpload(buffer, copies); - runtime.CopyBuffer(buffer, upload_staging.buffer, copies, true, can_reorder); + const auto order = runtime.DecideUploadOrder(buffer, copies); + runtime.CopyBuffer(buffer, upload_staging.buffer, copies, true, order); } else { buffer.ImmediateUpload(buffer.Offset(dest_address), inlined_buffer.first(copy_size)); } diff --git a/src/video_core/buffer_cache/upload_order.h b/src/video_core/buffer_cache/upload_order.h new file mode 100644 index 0000000000..d5576cacfe --- /dev/null +++ b/src/video_core/buffer_cache/upload_order.h @@ -0,0 +1,157 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later + +#pragma once + +#include +#include +#include + +#include "common/common_types.h" + +namespace VideoCommon { + +/// Where an upload into a buffer runs. +struct UploadOrder { + /// In the submission's upload command buffer, ahead of all the work of its main one. If not, + /// at its place in the main command buffer, between barriers, after the work recorded before + /// it (a render pass in progress ends first). + bool reorder{}; + /// When reordered: the newest earlier submission whose work used the buffer, 0 if none. The + /// upload command buffer begins with a barrier on earlier work unless the host GPU finished + /// that submission by the time the upload command buffer is recorded. + u64 prior_tick{}; +}; + +/// The bytes of a buffer that GPU work recorded in a submission uses (reads or writes), by the +/// tick of the submission. Byte ranges are kept for the newest submission only: the ones before +/// it collapse into the tick of the newest of them, which no byte's last use is later than. +class SubmissionUses { +public: + /// Disjoint ranges are merged with their neighbors beyond this many: the uses then cover + /// more bytes than the work used, never fewer. + static constexpr size_t MaxRanges = 1024; + + /// Work recorded in submission `tick` uses [offset, offset + size). Ticks never go back. + void Track(u64 offset, u64 size, u64 tick) noexcept { + if (size == 0) { + return; + } + if (tick != current_tick) { + if (tick < current_tick) { + previous_tick = (std::max)(previous_tick, tick); + return; + } + if (!ranges.empty()) { + previous_tick = current_tick; + } + ranges.clear(); + recent.fill(Range{}); + current_tick = tick; + } + const u64 begin = offset; + const u64 end = offset + size; + // Draws bind the same ranges again and again. Ranges only grow until the tick changes, + // so one that covered these bytes still does. + for (const Range& range : recent) { + if (range.begin <= begin && end <= range.end) { + return; + } + } + recent[recent_cursor++ % recent.size()] = Range{begin, end}; + // The first range that ends at or after begin: it overlaps or touches [begin, end), or + // lies after it. + auto first = + std::lower_bound(ranges.begin(), ranges.end(), begin, + [](const Range& range, u64 value) { return range.end < value; }); + if (first != ranges.end() && first->begin <= begin && end <= first->end) { + return; + } + u64 merged_begin = begin; + u64 merged_end = end; + auto last = first; + while (last != ranges.end() && last->begin <= end) { + merged_begin = (std::min)(merged_begin, last->begin); + merged_end = (std::max)(merged_end, last->end); + ++last; + } + if (first == last) { + ranges.insert(first, Range{merged_begin, merged_end}); + } else { + *first = Range{merged_begin, merged_end}; + ranges.erase(first + 1, last); + } + if (ranges.size() > MaxRanges) { + Coarsen(); + } + } + + /// Whether work recorded in submission `tick` uses a byte of [offset, offset + size). + [[nodiscard]] bool IsUsed(u64 offset, u64 size, u64 tick) const noexcept { + if (size == 0 || tick != current_tick || ranges.empty()) { + return false; + } + const u64 begin = offset; + const u64 end = offset + size; + // The first range that ends after begin. + const auto it = + std::upper_bound(ranges.begin(), ranges.end(), begin, + [](u64 value, const Range& range) { return value < range.end; }); + return it != ranges.end() && it->begin < end; + } + + /// The newest submission before `tick` whose work used a byte of the buffer, 0 if none did. + [[nodiscard]] u64 NewestTickBefore(u64 tick) const noexcept { + // current_tick is 0 until the first use. Asked about the past (current_tick > tick), the + // newest use known is still a bound. + return current_tick == tick ? previous_tick : current_tick; + } + + /// How many disjoint ranges the newest submission's work uses (for tests). + [[nodiscard]] size_t RangeCount() const noexcept { + return ranges.size(); + } + +private: + struct Range { + u64 begin; + u64 end; + }; + + /// Merges each pair of neighbors, gap included. + void Coarsen() noexcept { + size_t out = 0; + for (size_t in = 0; in < ranges.size(); in += 2) { + const u64 merged_end = in + 1 < ranges.size() ? ranges[in + 1].end : ranges[in].end; + ranges[out++] = Range{ranges[in].begin, merged_end}; + } + ranges.resize(out); + recent.fill(Range{}); + } + + /// Sorted, disjoint, and never touching: neighbors merge. + std::vector ranges; + /// Ranges tracked lately in current_tick, as given. + std::array recent{}; + u32 recent_cursor{}; + u64 current_tick{}; + u64 previous_tick{}; +}; + +/// Where an upload that writes `copies` (dst_offset, size) of a buffer with these uses runs when +/// it is recorded in submission `tick`: in order when work recorded before it in the submission +/// uses a byte it writes, otherwise in the upload command buffer, after the work of the earlier +/// submissions that used the buffer. The caller then tracks the copies as uses: an upload of the +/// same bytes later in the submission stays in order behind this one. +template +[[nodiscard]] UploadOrder DecideUploadOrder(const SubmissionUses& uses, const Copies& copies, + u64 tick) { + for (const auto& copy : copies) { + if (uses.IsUsed(copy.dst_offset, copy.size, tick)) { + return UploadOrder{}; + } + } + return UploadOrder{.reorder = true, .prior_tick = uses.NewestTickBefore(tick)}; +} + +} // namespace VideoCommon diff --git a/src/video_core/renderer_opengl/gl_buffer_cache.cpp b/src/video_core/renderer_opengl/gl_buffer_cache.cpp index fb65a132d0..d5bef8fca4 100644 --- a/src/video_core/renderer_opengl/gl_buffer_cache.cpp +++ b/src/video_core/renderer_opengl/gl_buffer_cache.cpp @@ -188,7 +188,7 @@ void BufferCacheRuntime::CopyBuffer(GLuint dst_buffer, Buffer& src_buffer, void BufferCacheRuntime::CopyBuffer(Buffer& dst_buffer, GLuint src_buffer, std::span copies, bool barrier, - bool) { + VideoCommon::UploadOrder) { CopyBuffer(dst_buffer.Handle(), src_buffer, copies, barrier); } diff --git a/src/video_core/renderer_opengl/gl_buffer_cache.h b/src/video_core/renderer_opengl/gl_buffer_cache.h index 831ec09f46..b23e2e70ab 100644 --- a/src/video_core/renderer_opengl/gl_buffer_cache.h +++ b/src/video_core/renderer_opengl/gl_buffer_cache.h @@ -13,6 +13,7 @@ #include "common/common_types.h" #include "video_core/buffer_cache/buffer_cache_base.h" #include "video_core/buffer_cache/memory_tracker_base.h" +#include "video_core/buffer_cache/upload_order.h" #include "video_core/renderer_opengl/gl_device.h" #include "video_core/renderer_opengl/gl_resource_manager.h" #include "video_core/renderer_opengl/gl_staging_buffer_pool.h" @@ -73,8 +74,9 @@ class BufferCacheRuntime { void FreeDeferredStagingBuffer(StagingBufferMap& buffer); - bool CanReorderUpload(const Buffer&, std::span) { - return false; + VideoCommon::UploadOrder DecideUploadOrder(const Buffer&, + std::span) { + return {}; } void CopyBuffer(GLuint dst_buffer, GLuint src_buffer, @@ -85,7 +87,7 @@ class BufferCacheRuntime { void CopyBuffer(Buffer& dst_buffer, GLuint src_buffer, std::span copies, bool barrier, - bool can_reorder_upload = false); + VideoCommon::UploadOrder order = {}); void CopyBuffer(Buffer& dst_buffer, Buffer& src_buffer, std::span copies, bool); diff --git a/src/video_core/renderer_vulkan/upload_batch.h b/src/video_core/renderer_vulkan/upload_batch.h index 7c360c6b6a..3bb88b8a67 100644 --- a/src/video_core/renderer_vulkan/upload_batch.h +++ b/src/video_core/renderer_vulkan/upload_batch.h @@ -3,6 +3,8 @@ #pragma once +#include +#include #include #include @@ -107,4 +109,86 @@ class UploadBatch { std::vector groups; }; +/// Uploads that run in the upload command buffer, ahead of all the work of their submission's +/// main command buffer (Scheduler::RecordReorderedUpload). They are recorded as one command when +/// the submission is: a copy per source and destination. Nothing orders them against each other +/// and nothing has to: the buffer cache keeps an upload in order when work recorded before it in +/// the submission, an earlier upload included, uses a byte it writes (DecideUploadOrder). +class ReorderedUploadBatch { +public: + [[nodiscard]] bool Empty() const noexcept { + return entries.empty(); + } + + /// prior_tick: the newest earlier submission whose work uses bytes the copies write, or a + /// later one; 0 if none. + void Add(VkBuffer src, VkBuffer dst, std::span new_copies, u64 prior_tick) { + for (const VkBufferCopy& copy : new_copies) { + entries.push_back(Entry{src, dst, copy}); + } + newest_prior_tick = (std::max)(newest_prior_tick, prior_tick); + } + + /// Empties the batch into its copies, grouped by source and destination (in the order they + /// were added within a group), and the newest earlier submission they must come after. + void Take(std::vector& copies, std::vector& groups, + u64& prior_tick) { + std::stable_sort(entries.begin(), entries.end(), [](const Entry& lhs, const Entry& rhs) { + if (lhs.src != rhs.src) { + return std::less{}(lhs.src, rhs.src); + } + return std::less{}(lhs.dst, rhs.dst); + }); + copies.clear(); + groups.clear(); + for (const Entry& entry : entries) { + const u32 index = static_cast(copies.size()); + copies.push_back(entry.copy); + if (!groups.empty() && groups.back().src == entry.src && + groups.back().dst == entry.dst) { + ++groups.back().count; + } else { + groups.push_back(UploadBatch::Group{entry.src, entry.dst, index, 1}); + } + } + prior_tick = newest_prior_tick; + entries.clear(); + newest_prior_tick = 0; + } + + /// Records a batch into the upload command buffer. With after_prior_work it begins with a + /// barrier whose first scope is all the work submitted before it, earlier submissions' main + /// command buffers included: their reads and writes of the bytes the copies write come first. + /// The upload command buffer's last barrier (Scheduler::SubmitExecution) orders the copies + /// before the main command buffer's work and all later work. + static void Record(vk::CommandBuffer upload_cmdbuf, std::span batch_copies, + std::span batch_groups, bool after_prior_work) { + static constexpr VkMemoryBarrier PRIOR_WORK_BARRIER{ + .sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER, + .pNext = nullptr, + .srcAccessMask = VK_ACCESS_MEMORY_WRITE_BIT, + .dstAccessMask = VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT, + }; + if (after_prior_work) { + upload_cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, + VK_PIPELINE_STAGE_TRANSFER_BIT, 0, PRIOR_WORK_BARRIER); + } + for (const UploadBatch::Group& group : batch_groups) { + upload_cmdbuf.CopyBuffer( + group.src, group.dst, + vk::Span(batch_copies.data() + group.first, group.count)); + } + } + +private: + struct Entry { + VkBuffer src; + VkBuffer dst; + VkBufferCopy copy; + }; + + std::vector entries; + u64 newest_prior_tick = 0; +}; + } // namespace Vulkan diff --git a/src/video_core/renderer_vulkan/vk_buffer_cache.cpp b/src/video_core/renderer_vulkan/vk_buffer_cache.cpp index 5b9ede9e37..2b266dd9cf 100644 --- a/src/video_core/renderer_vulkan/vk_buffer_cache.cpp +++ b/src/video_core/renderer_vulkan/vk_buffer_cache.cpp @@ -34,7 +34,7 @@ namespace Vulkan { namespace { -/// TEST (EDEN_NX2_GPU_COST=2): Buffer::MarkUsage notes page ticks, CanReorderUpload classifies. +/// TEST (EDEN_NX2_GPU_COST=2): Buffer::MarkUsage notes page ticks, DecideUploadOrder classifies. const bool g_trace_uploads = GpuCost::TracesUploads(); void HostReadBarrier(vk::CommandBuffer cmdbuf) { @@ -116,7 +116,7 @@ vk::Buffer CreateBuffer(const Device& device, const MemoryAllocator& memory_allo } // Anonymous namespace Buffer::Buffer(BufferCacheRuntime& runtime, VideoCommon::NullBufferParams null_params) - : VideoCommon::BufferBase(null_params), scheduler{&runtime.scheduler}, tracker{4096} { + : VideoCommon::BufferBase(null_params), scheduler{&runtime.scheduler} { if (runtime.device.HasNullDescriptor()) { return; } @@ -133,8 +133,7 @@ Buffer::Buffer(BufferCacheRuntime& runtime, DAddr cpu_addr_, u64 size_bytes_, : VideoCommon::BufferBase(cpu_addr_, size_bytes_), device{&runtime.device}, scheduler{&runtime.scheduler}, buffer{CreateBuffer(*device, runtime.memory_allocator, SizeBytes(), - runtime.SparseAlignmentFor(sparse_compatible_))}, - tracker{SizeBytes()} { + runtime.SparseAlignmentFor(sparse_compatible_))} { sparse_compatible = sparse_compatible_; if (runtime.device.HasDebuggingToolAttached()) { buffer.SetObjectNameEXT(fmt::format("Buffer {:#x}", CpuAddr()).c_str()); @@ -145,10 +144,10 @@ Buffer::Buffer(BufferCacheRuntime& runtime, DAddr cpu_addr_, u64 size_bytes_, } void Buffer::MarkUsage(u64 offset, u64 size) noexcept { - tracker.Track(offset, size); - last_usage_tick = scheduler->CurrentTick(); + const u64 tick = scheduler->CurrentTick(); + uses.Track(offset, size, tick); if (g_trace_uploads) [[unlikely]] { - const std::array use{offset, size, last_usage_tick}; + const std::array use{offset, size, tick}; if (std::ranges::find(noted_uses, use) != noted_uses.end() || size == 0) { return; } @@ -159,11 +158,15 @@ void Buffer::MarkUsage(u64 offset, u64 size) noexcept { const size_t last = (std::min)(static_cast((offset + size - 1) >> 12), use_ticks.size() - 1); for (size_t page = static_cast(offset >> 12); page <= last; ++page) { - use_ticks[page] = last_usage_tick; + use_ticks[page] = tick; } } } +void Buffer::TrackUploadWrite(u64 offset, u64 size) noexcept { + uses.Track(offset, size, scheduler->CurrentTick()); +} + u64 Buffer::NewestUseTick(u64 offset, u64 size) const noexcept { u64 newest = 0; if (size == 0 || use_ticks.empty()) { @@ -500,13 +503,10 @@ u64 BufferCacheRuntime::GetMaxStorageBufferRange() const { return device.GetMaxStorageBufferRange(); } -void BufferCacheRuntime::TickFrame(Common::SlotVector& slot_buffers) noexcept { +void BufferCacheRuntime::TickFrame( + [[maybe_unused]] Common::SlotVector& slot_buffers) noexcept { + // Buffers' uses need no reset: they are kept by submission (VideoCommon::SubmissionUses). multi_range_buffers.TickFrame(scheduler); - for (auto it = slot_buffers.begin(); it != slot_buffers.end(); it++) { - if (scheduler.IsFree(it->LastUsageTick())) { - it->ResetUsageTracking(); - } - } } u64 BufferCacheRuntime::CurrentTick() { @@ -610,19 +610,20 @@ void BufferCacheRuntime::TraceCountWork(bool dispatch) { scheduler.TraceCountWork(dispatch); } -bool BufferCacheRuntime::CanReorderUpload(const Buffer& buffer, - std::span copies) { - if (Settings::values.disable_buffer_reorder) { - return false; +VideoCommon::UploadOrder BufferCacheRuntime::DecideUploadOrder( + Buffer& buffer, std::span copies) { + VideoCommon::UploadOrder order{}; + if (!Settings::values.disable_buffer_reorder) { + order = VideoCommon::DecideUploadOrder(buffer.Uses(), copies, scheduler.CurrentTick()); } - const bool can_use_upload_cmdbuf = - std::ranges::all_of(copies, [&](const VideoCommon::BufferCopy& copy) { - return !buffer.IsRegionUsed(copy.dst_offset, copy.size); - }); if (g_trace_uploads) [[unlikely]] { - TraceUpload(buffer, copies, can_use_upload_cmdbuf); + TraceUpload(buffer, copies, order.reorder); + } + // An upload of these bytes later in this submission must run after this one: in order. + for (const VideoCommon::BufferCopy& copy : copies) { + buffer.TrackUploadWrite(copy.dst_offset, copy.size); } - return can_use_upload_cmdbuf; + return order; } void BufferCacheRuntime::TraceUpload(const Buffer& buffer, @@ -681,10 +682,12 @@ void BufferCacheRuntime::TraceUpload(const Buffer& buffer, void BufferCacheRuntime::CopyBuffer(VkBuffer dst_buffer, VkBuffer src_buffer, std::span copies, bool barrier, - bool can_reorder_upload, std::source_location location) { + VideoCommon::UploadOrder order, std::source_location location) { if (dst_buffer == VK_NULL_HANDLE || src_buffer == VK_NULL_HANDLE) { return; } + const bool from_stream = src_buffer == staging_pool.StreamBuf(); + const bool reorder = from_stream && order.reorder; static constexpr VkMemoryBarrier READ_BARRIER{ .sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER, .pNext = nullptr, @@ -701,18 +704,17 @@ void BufferCacheRuntime::CopyBuffer(VkBuffer dst_buffer, VkBuffer src_buffer, // Measuring a popular game, this number never exceeds the specified size once data is warmed up boost::container::small_vector vk_copies(copies.size()); std::ranges::transform(copies, vk_copies.begin(), MakeBufferCopy); - if (src_buffer == staging_pool.StreamBuf() && can_reorder_upload) { - scheduler.RecordWithUploadBuffer( - [src_buffer, dst_buffer, vk_copies](vk::CommandBuffer, - vk::CommandBuffer upload_cmdbuf) { - upload_cmdbuf.CopyBuffer(src_buffer, dst_buffer, vk_copies); - }, - location); + if (reorder) { + // Ahead of this submission's work: no render pass ends for it. Recorded into the upload + // command buffer with the submission's other reordered uploads when it is submitted. + scheduler.RecordReorderedUpload( + src_buffer, dst_buffer, + std::span(vk_copies.data(), vk_copies.size()), order.prior_tick); return; } scheduler.RequestOutsideRenderPassOperationContext(location); - if (barrier && src_buffer == staging_pool.StreamBuf() && !vk_copies.empty()) { + if (barrier && from_stream && !vk_copies.empty()) { // An upload in order, with these barriers. Runs of them (Ocarina of Time's inline writes // into GPU-written pages: ~600 a frame) go in one command between one pair of them. scheduler.RecordUpload(src_buffer, dst_buffer, diff --git a/src/video_core/renderer_vulkan/vk_buffer_cache.h b/src/video_core/renderer_vulkan/vk_buffer_cache.h index f042a1a8b7..29f57762d9 100644 --- a/src/video_core/renderer_vulkan/vk_buffer_cache.h +++ b/src/video_core/renderer_vulkan/vk_buffer_cache.h @@ -15,7 +15,7 @@ #include "video_core/buffer_cache/buffer_cache_base.h" #include "video_core/buffer_cache/memory_tracker_base.h" -#include "video_core/buffer_cache/usage_tracker.h" +#include "video_core/buffer_cache/upload_order.h" #include "video_core/engines/maxwell_3d.h" #include "video_core/renderer_vulkan/vk_compute_pass.h" #include "video_core/renderer_vulkan/vk_multi_range_buffer.h" @@ -58,23 +58,22 @@ class Buffer : public VideoCommon::BufferBase { return buffer.Location(); } - [[nodiscard]] bool IsRegionUsed(u64 offset, u64 size) const noexcept { - return tracker.IsUsed(offset, size); + /// The bytes that work recorded in the current submission uses, and the newest earlier + /// submission whose work used the buffer (BufferCacheRuntime::DecideUploadOrder). + [[nodiscard]] const VideoCommon::SubmissionUses& Uses() const noexcept { + return uses; } /// TEST (EDEN_NX2_GPU_COST): the newest tick that used any 4 KiB page of the range, 0 when /// none did since the trace started. [[nodiscard]] u64 NewestUseTick(u64 offset, u64 size) const noexcept; + /// GPU work recorded now reads or writes [offset, offset + size). void MarkUsage(u64 offset, u64 size) noexcept; - void ResetUsageTracking() noexcept { - tracker.Reset(); - } - - [[nodiscard]] u64 LastUsageTick() const noexcept { - return last_usage_tick; - } + /// An upload recorded now writes [offset, offset + size): uploads of these bytes after it in + /// this submission run after it (DecideUploadOrder). + void TrackUploadWrite(u64 offset, u64 size) noexcept; [[nodiscard]] std::span Mapped() const noexcept { return buffer.IsHostCoherent() ? buffer.Mapped() : std::span{}; @@ -96,9 +95,8 @@ class Buffer : public VideoCommon::BufferBase { Scheduler* scheduler{}; vk::Buffer buffer; std::vector views; - VideoCommon::UsageTracker tracker; + VideoCommon::SubmissionUses uses; VkDeviceAddress device_address{}; - u64 last_usage_tick{}; bool is_null{}; bool sparse_compatible{}; /// TEST (EDEN_NX2_GPU_COST): the tick of each 4 KiB page's last use, and the last ranges @@ -182,16 +180,21 @@ class BufferCacheRuntime { [[nodiscard]] StagingBufferRef DownloadStagingBuffer(size_t size, bool deferred = false); - bool CanReorderUpload(const Buffer& buffer, std::span copies); + /// Where an upload of `copies` into buffer runs (VideoCommon::DecideUploadOrder): in order + /// when work recorded in this submission uses a byte it writes, otherwise in the upload + /// command buffer. Tracks the copies as uses of their bytes. + [[nodiscard]] VideoCommon::UploadOrder DecideUploadOrder( + Buffer& buffer, std::span copies); void FreeDeferredStagingBuffer(StagingBufferRef& ref); void PreCopyBarrier(); - /// location: the caller, which the render pass end and the copy are counted at. + /// order: where an upload from the stream buffer runs (DecideUploadOrder); other copies run + /// in order. location: the caller, which the render pass end and the copy are counted at. void CopyBuffer(VkBuffer dst_buffer, VkBuffer src_buffer, std::span copies, bool barrier, - bool can_reorder_upload = false, + VideoCommon::UploadOrder order = {}, std::source_location location = std::source_location::current()); void PostCopyBarrier(); diff --git a/src/video_core/renderer_vulkan/vk_scheduler.cpp b/src/video_core/renderer_vulkan/vk_scheduler.cpp index 5b7cd32f8e..cae72f2b4f 100644 --- a/src/video_core/renderer_vulkan/vk_scheduler.cpp +++ b/src/video_core/renderer_vulkan/vk_scheduler.cpp @@ -124,7 +124,8 @@ Scheduler::Scheduler(const Device& device_, StateTracker& state_tracker_) : device{device_}, state_tracker{state_tracker_}, master_semaphore{std::make_unique(device)}, command_pool{std::make_unique(*master_semaphore, device)}, - upload_batch{std::make_unique()} { + upload_batch{std::make_unique()}, + reordered_uploads{std::make_unique()} { if (Common::DkbFrameTrace::Enabled()) { const auto physical = device.GetPhysical(); @@ -575,6 +576,39 @@ void Scheduler::FlushUploads() { RecordAs(std::move(command), upload_batch_site, RecordSite::Own, 0); } +void Scheduler::RecordReorderedUpload(VkBuffer src, VkBuffer dst, + std::span copies, u64 prior_tick) { + if (VideoCommon::GpuSplit::check_owner) [[unlikely]] { + VideoCommon::GpuSplit::CheckOwner("Scheduler::RecordReorderedUpload"); + } + if (copies.empty()) { + return; + } + reordered_uploads->Add(src, dst, copies, prior_tick); + reordered_uploads_pending = true; +} + +void Scheduler::FlushReorderedUploads() { + std::vector batch_copies; + std::vector batch_groups; + u64 prior_tick = 0; + reordered_uploads->Take(batch_copies, batch_groups, prior_tick); + reordered_uploads_pending = false; + // The copies must come after earlier submissions' work on their bytes: finished already, or + // waited for by a barrier. The host GPU is asked once a submission, when it may still run it. + bool batch_after_prior = prior_tick != 0 && !master_semaphore->IsFree(prior_tick); + if (batch_after_prior) { + master_semaphore->Refresh(); + batch_after_prior = !master_semaphore->IsFree(prior_tick); + } + RecordAs( + [copies = std::move(batch_copies), groups = std::move(batch_groups), + after_prior_work = batch_after_prior](vk::CommandBuffer, vk::CommandBuffer upload_cmdbuf) { + ReorderedUploadBatch::Record(upload_cmdbuf, copies, groups, after_prior_work); + }, + std::source_location::current(), RecordSite::Own, 0); +} + void Scheduler::BeginRenderPassImpl(const Framebuffer* framebuffer, VkRenderPass renderpass, const VkClearValue* clear_values, u32 clear_value_count) { if (VideoCommon::Nx2Profile::Enabled()) { @@ -1096,6 +1130,10 @@ u64 Scheduler::SubmitExecution(VkSemaphore signal_semaphore, VkSemaphore wait_se } EndPendingOperations(cause); InvalidateState(); + if (reordered_uploads_pending) { + // Into the upload command buffer of the submission their uploads were decided for. + FlushReorderedUploads(); + } const u64 signal_value = master_semaphore->NextTick(); RecordAs([signal_semaphore, wait_semaphore, signal_value, diff --git a/src/video_core/renderer_vulkan/vk_scheduler.h b/src/video_core/renderer_vulkan/vk_scheduler.h index caaf61f3b7..206555d7d8 100644 --- a/src/video_core/renderer_vulkan/vk_scheduler.h +++ b/src/video_core/renderer_vulkan/vk_scheduler.h @@ -42,6 +42,7 @@ class Device; class Framebuffer; class GpuCost; class GraphicsPipeline; +class ReorderedUploadBatch; class StateTracker; class UploadBatch; @@ -218,6 +219,15 @@ class Scheduler { void RecordUpload(VkBuffer src, VkBuffer dst, std::span copies, std::source_location location = std::source_location::current()); + /// An upload ahead of all the work of this submission's main command buffer: it runs in the + /// upload command buffer, which the submission's reordered uploads are recorded into as one + /// command when it is submitted. The caller guarantees that no work recorded in this + /// submission uses the bytes it writes, nor work of an earlier submission after prior_tick + /// (0: of any). Unless the host GPU has finished submission prior_tick by then, the upload + /// command buffer begins with a barrier on the work of earlier submissions. + void RecordReorderedUpload(VkBuffer src, VkBuffer dst, std::span copies, + u64 prior_tick); + /// Send work to a separate thread. template requires std::is_invocable_v @@ -579,6 +589,11 @@ class Scheduler { bool uploads_pending{}; /// Records the pending uploads as one command. void FlushUploads(); + /// RecordReorderedUpload's copies, recorded when the submission is. + std::unique_ptr reordered_uploads; + bool reordered_uploads_pending{}; + /// Records the reordered uploads into the upload command buffer as one command. + void FlushReorderedUploads(); /// The line that ended the last render pass: the site of the next pass's begin. std::source_location last_pass_end{}; From a0ecdb040127415c444a10a952f26537f4a360be Mon Sep 17 00:00:00 2001 From: jpkto <255363096+jpkto@users.noreply.github.com> Date: Sun, 8 Nov 2026 05:46:51 +0100 Subject: [PATCH 050/105] TEST: check each upload's place against an exact model of buffer uses (EDEN_NX2_UPLOAD_REORDER_CHECK) Uploads now run ahead of their submission's work unless work recorded before them in it uses their bytes. Whether that is right depends on every GPU use of a guest buffer being marked, and on SubmissionUses and the barrier on earlier submissions doing what they claim. EDEN_NX2_UPLOAD_REORDER_CHECK=1 checks every upload from the stream buffer against a model kept apart from both. The model (vk_upload_reorder_check.cpp) keeps, per VkBuffer and submission, the exact byte ranges that work uses, in a map. It drops a submission once the host GPU has finished it. It hears of uses from two sides: - every Buffer::MarkUsage; - the runtime's own calls that put a buffer into GPU work: descriptor and texel buffer bindings, vertex, index and transform feedback bindings, multi-range sources, both sides of every copy (an upload's destination included), and clears. A use the buffer cache forgot to mark is therefore still in the model whenever the runtime records it. BufferCacheRuntime::CopyBuffer checks each upload as it places it, before the model hears of the upload's own copy. It counts as an error: - an upload put in the upload command buffer while work recorded before it in its submission uses a byte it writes; - one whose bytes are used by unfinished work of an earlier submission later than the one the batch waits for, so the barrier could be left out while that work still runs. The first 64 errors and every 1024th after are logged ("Upload reorder check: an upload into buffer ... ahead of of its bytes ..."). Every 10 s it logs: "Upload reorder check in 10.0 s: N uploads from the stream buffer: R reordered (U of them after unfinished work of earlier submissions on their bytes), I in order (J after a use of their bytes in the same submission); errors: 0 reordered past a use of their bytes, 0 reordered ahead of unfinished earlier work on their bytes without waiting for it (since start: 0 and 0 in T uploads)". Tested by driving the check with a stand-in scheduler. Correct placements log nothing: uploads that touch used ranges without overlapping them, uploads into other buffers, uploads after work the host saw finish, and uploads that wait for the right submission. Each wrong placement is logged and names what used the bytes (a descriptor, a copy writing, a vertex buffer over both). The report line gives the counts. Syntax checks and strict warnings on the changed lines. Co-Authored-By: Claude Opus 5.5 --- src/video_core/CMakeLists.txt | 2 + .../renderer_vulkan/vk_buffer_cache.cpp | 29 +- .../renderer_vulkan/vk_buffer_cache.h | 16 +- .../vk_upload_reorder_check.cpp | 292 ++++++++++++++++++ .../renderer_vulkan/vk_upload_reorder_check.h | 75 +++++ 5 files changed, 412 insertions(+), 2 deletions(-) create mode 100644 src/video_core/renderer_vulkan/vk_upload_reorder_check.cpp create mode 100644 src/video_core/renderer_vulkan/vk_upload_reorder_check.h diff --git a/src/video_core/CMakeLists.txt b/src/video_core/CMakeLists.txt index 3321aedd43..70cfd98f2e 100644 --- a/src/video_core/CMakeLists.txt +++ b/src/video_core/CMakeLists.txt @@ -252,6 +252,8 @@ add_library(video_core STATIC renderer_vulkan/vk_turbo_mode.h renderer_vulkan/vk_update_descriptor.cpp renderer_vulkan/vk_update_descriptor.h + renderer_vulkan/vk_upload_reorder_check.cpp + renderer_vulkan/vk_upload_reorder_check.h renderer_vulkan/descriptor_payload_growth.h vulkan_common/vulkan_debug_callback.cpp vulkan_common/vulkan_debug_callback.h diff --git a/src/video_core/renderer_vulkan/vk_buffer_cache.cpp b/src/video_core/renderer_vulkan/vk_buffer_cache.cpp index 2b266dd9cf..a8fab2165a 100644 --- a/src/video_core/renderer_vulkan/vk_buffer_cache.cpp +++ b/src/video_core/renderer_vulkan/vk_buffer_cache.cpp @@ -146,6 +146,9 @@ Buffer::Buffer(BufferCacheRuntime& runtime, DAddr cpu_addr_, u64 size_bytes_, void Buffer::MarkUsage(u64 offset, u64 size) noexcept { const u64 tick = scheduler->CurrentTick(); uses.Track(offset, size, tick); + if (UploadReorderCheck* const check = UploadReorderCheck::Active()) [[unlikely]] { + check->NoteUse(*buffer, offset, size, UploadReorderCheck::Source::Marked); + } if (g_trace_uploads) [[unlikely]] { const std::array use{offset, size, tick}; if (std::ranges::find(noted_uses, use) != noted_uses.end() || size == 0) { @@ -422,7 +425,7 @@ BufferCacheRuntime::BufferCacheRuntime(const Device& device_, MemoryAllocator& m staging_pool{staging_pool_}, guest_descriptor_queue{guest_descriptor_queue_}, quad_index_pass(device, scheduler, descriptor_pool, staging_pool, compute_pass_descriptor_queue), - multi_range_buffers(device_) { + multi_range_buffers(device_), reorder_check{UploadReorderCheck::Create(scheduler_)} { const VkDriverIdKHR driver_id = device.GetDriverID(); const bool enable_vertex_repack = VertexRepackEnabled(); LOG_INFO(Render_Vulkan, "69-byte vertex layout repacking {} ({})", @@ -701,6 +704,22 @@ void BufferCacheRuntime::CopyBuffer(VkBuffer dst_buffer, VkBuffer src_buffer, .dstAccessMask = VK_ACCESS_MEMORY_READ_BIT | VK_ACCESS_MEMORY_WRITE_BIT, }; + if (reorder_check) [[unlikely]] { + // The upload's place against the uses before it, then the copy's own uses. + if (from_stream) { + reorder_check->CheckUpload(dst_buffer, copies, + reorder ? order : VideoCommon::UploadOrder{}, location); + } + for (const VideoCommon::BufferCopy& copy : copies) { + if (!from_stream) { + reorder_check->NoteUse(src_buffer, copy.src_offset, copy.size, + UploadReorderCheck::Source::CopyRead); + } + reorder_check->NoteUse(dst_buffer, copy.dst_offset, copy.size, + UploadReorderCheck::Source::CopyWrite); + } + } + // Measuring a popular game, this number never exceeds the specified size once data is warmed up boost::container::small_vector vk_copies(copies.size()); std::ranges::transform(copies, vk_copies.begin(), MakeBufferCopy); @@ -772,6 +791,7 @@ void BufferCacheRuntime::ClearBuffer(VkBuffer dest_buffer, u32 offset, size_t si if (dest_buffer == VK_NULL_HANDLE) { return; } + NoteCheckUse(dest_buffer, offset, size, UploadReorderCheck::Source::Clear); static constexpr VkMemoryBarrier READ_BARRIER{ .sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER, .pNext = nullptr, @@ -829,6 +849,7 @@ bool BufferCacheRuntime::BindMultiRangeStorageBuffer(u64 key, bool is_written) { void BufferCacheRuntime::BindIndexBuffer(PrimitiveTopology topology, IndexFormat index_format, u32 base_vertex, u32 num_indices, VkBuffer buffer, u32 offset, [[maybe_unused]] u32 size) { + NoteCheckUse(buffer, offset, size, UploadReorderCheck::Source::Index); VkIndexType vk_index_type = MaxwellToVK::IndexFormat(index_format); VkDeviceSize vk_offset = offset; VkBuffer vk_buffer = buffer; @@ -904,6 +925,7 @@ void BufferCacheRuntime::BindVertexBuffer(u32 index, VkBuffer buffer, u32 offset if (index >= device.GetMaxVertexInputBindings()) { return; } + NoteCheckUse(buffer, offset, size, UploadReorderCheck::Source::Vertex); if (buffer != VK_NULL_HANDLE && size != 0 && IsVertexRepacked(index)) { const auto repacked = vertex_repack_pass->Assemble(buffer, offset, size, u64{offset} + size); @@ -938,6 +960,8 @@ void BufferCacheRuntime::BindVertexBuffers(VideoCommon::HostBindings& bi boost::container::static_vector buffer_handles(bindings.buffers.size()); for (u32 i = 0; i < bindings.buffers.size(); ++i) { auto handle = bindings.buffers[i]->Handle(); + NoteCheckUse(handle, bindings.offsets[i], bindings.sizes[i], + UploadReorderCheck::Source::Vertex); if (handle == VK_NULL_HANDLE) { bindings.offsets[i] = 0; bindings.sizes[i] = VK_WHOLE_SIZE; @@ -991,6 +1015,7 @@ void BufferCacheRuntime::BindTransformFeedbackBuffer(u32 index, VkBuffer buffer, // Already logged in the rasterizer return; } + NoteCheckUse(buffer, offset, size, UploadReorderCheck::Source::TransformFeedback); if (buffer == VK_NULL_HANDLE) { // Vulkan doesn't support null transform feedback buffers. // Replace it with our own null buffer. @@ -1014,6 +1039,8 @@ void BufferCacheRuntime::BindTransformFeedbackBuffers(VideoCommon::HostBindings< boost::container::static_vector buffer_handles(bindings.buffers.size()); for (u32 i = 0; i < bindings.buffers.size(); ++i) { auto handle = bindings.buffers[i]->Handle(); + NoteCheckUse(handle, bindings.offsets[i], bindings.sizes[i], + UploadReorderCheck::Source::TransformFeedback); if (handle == VK_NULL_HANDLE) { ReserveNullBuffer(); handle = *null_buffer; diff --git a/src/video_core/renderer_vulkan/vk_buffer_cache.h b/src/video_core/renderer_vulkan/vk_buffer_cache.h index 29f57762d9..d0a5d1a26f 100644 --- a/src/video_core/renderer_vulkan/vk_buffer_cache.h +++ b/src/video_core/renderer_vulkan/vk_buffer_cache.h @@ -21,6 +21,7 @@ #include "video_core/renderer_vulkan/vk_multi_range_buffer.h" #include "video_core/renderer_vulkan/vk_staging_buffer_pool.h" #include "video_core/renderer_vulkan/vk_update_descriptor.h" +#include "video_core/renderer_vulkan/vk_upload_reorder_check.h" #include "video_core/surface.h" #include "video_core/vulkan_common/vulkan_memory_allocator.h" #include "video_core/vulkan_common/vulkan_wrapper.h" @@ -72,7 +73,7 @@ class Buffer : public VideoCommon::BufferBase { void MarkUsage(u64 offset, u64 size) noexcept; /// An upload recorded now writes [offset, offset + size): uploads of these bytes after it in - /// this submission run after it (DecideUploadOrder). + /// this submission run after it (DecideUploadOrder). The check hears of it from the copy. void TrackUploadWrite(u64 offset, u64 size) noexcept; [[nodiscard]] std::span Mapped() const noexcept { @@ -271,6 +272,7 @@ class BufferCacheRuntime { } void PushMultiRangeSource(const Buffer& buffer, u32 offset, u32 size) { + NoteCheckUse(buffer.Handle(), offset, size, UploadReorderCheck::Source::MultiRange); const vk::MemoryLocation location = buffer.Location(); multi_range_sources.push_back(MultiRangeSource{ .handle = buffer.Handle(), @@ -309,6 +311,7 @@ class BufferCacheRuntime { if (format == VideoCore::Surface::PixelFormat::Invalid) { format = VideoCore::Surface::PixelFormat::R8_UNORM; } + NoteCheckUse(buffer.Handle(), offset, size, UploadReorderCheck::Source::Texel); guest_descriptor_queue.AddTexelBuffer(buffer.View(offset, size, format), buffer.DeviceAddress(), offset, size, TexelBufferFormat(format)); @@ -323,8 +326,16 @@ class BufferCacheRuntime { } private: + /// TEST (EDEN_NX2_UPLOAD_REORDER_CHECK): work recorded now uses these bytes. + void NoteCheckUse(VkBuffer handle, u64 offset, u64 size, UploadReorderCheck::Source source) { + if (reorder_check) [[unlikely]] { + reorder_check->NoteUse(handle, offset, size, source); + } + } + void BindBuffer(const Buffer& buffer, u32 offset, u32 size) { const VkBuffer handle = buffer.Handle(); + NoteCheckUse(handle, offset, size, UploadReorderCheck::Source::Descriptor); if (handle == VK_NULL_HANDLE) { guest_descriptor_queue.AddBuffer(handle, 0, 0, VK_WHOLE_SIZE); } else { @@ -364,6 +375,9 @@ class BufferCacheRuntime { StreamUniformRef last_stream_uniform{}; + /// TEST: EDEN_NX2_UPLOAD_REORDER_CHECK + std::unique_ptr reorder_check; + bool limit_dynamic_storage_buffers = false; u32 max_dynamic_storage_buffers = (std::numeric_limits::max)(); diff --git a/src/video_core/renderer_vulkan/vk_upload_reorder_check.cpp b/src/video_core/renderer_vulkan/vk_upload_reorder_check.cpp new file mode 100644 index 0000000000..0d3b8731ec --- /dev/null +++ b/src/video_core/renderer_vulkan/vk_upload_reorder_check.cpp @@ -0,0 +1,292 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "common/logging.h" +#include "video_core/renderer_vulkan/vk_scheduler.h" +#include "video_core/renderer_vulkan/vk_upload_reorder_check.h" +#include "video_core/texture_cache/types.h" + +namespace Vulkan { +namespace { + +constexpr size_t NUM_SOURCES = static_cast(UploadReorderCheck::Source::Count); +constexpr std::array SOURCE_NAMES{ + "a marked use", "a descriptor", + "a texel buffer", "a vertex buffer", + "an index buffer", "transform feedback", + "a multi-range source", "a copy reading", + "a copy writing", "a clear", +}; +static_assert(SOURCE_NAMES[NUM_SOURCES - 1] != nullptr); + +struct Range { + u64 end; + u32 sources; ///< bit per Source +}; + +/// Byte ranges by their begin, disjoint. Overlapping ranges merge, their sources too; touching +/// ones merge when they have the same sources, so an error names what used the bytes. +using RangeSet = std::map; + +void Insert(RangeSet& set, u64 begin, u64 end, u32 sources) { + auto it = set.upper_bound(begin); + if (it != set.begin()) { + const auto before = std::prev(it); + if (before->second.end >= end && (before->second.sources & sources) == sources) { + // Noted already (draws bind the same ranges again and again). + return; + } + if (before->second.end > begin || + (before->second.end == begin && before->second.sources == sources)) { + it = before; + } + } + u64 merged_begin = begin; + u64 merged_end = end; + u32 merged_sources = sources; + while (it != set.end() && (it->first < merged_end || + (it->first == merged_end && it->second.sources == merged_sources))) { + merged_begin = (std::min)(merged_begin, it->first); + merged_end = (std::max)(merged_end, it->second.end); + merged_sources |= it->second.sources; + it = set.erase(it); + } + set.emplace(merged_begin, Range{merged_end, merged_sources}); +} + +/// The sources of the ranges that share a byte with [begin, end), 0 when none does. +u32 Overlap(const RangeSet& set, u64 begin, u64 end) { + auto it = set.upper_bound(begin); + if (it != set.begin()) { + --it; + } + u32 sources = 0; + for (; it != set.end() && it->first < end; ++it) { + if (it->second.end > begin) { + sources |= it->second.sources; + } + } + return sources; +} + +/// A handle's value for the log: a pointer on 64-bit hosts, an integer on 32-bit ones. +template +u64 HandleValue(Handle handle) { + if constexpr (std::is_pointer_v) { + return static_cast(reinterpret_cast(handle)); + } else { + return static_cast(handle); + } +} + +std::string SourceList(u32 sources) { + std::string list; + for (size_t i = 0; i < NUM_SOURCES; ++i) { + if ((sources >> i) & 1) { + list += fmt::format("{}{}", list.empty() ? "" : ", ", SOURCE_NAMES[i]); + } + } + return list; +} + +struct Counts { + u64 uploads; + u64 reordered; + u64 reordered_after_unfinished; + u64 in_order; + u64 in_order_used; + u64 past_use; + u64 unordered; +}; + +} // Anonymous namespace + +struct UploadReorderCheck::Impl { + explicit Impl(Scheduler& scheduler_) : scheduler{scheduler_} {} + + struct TickUses { + u64 tick; + RangeSet ranges; + }; + struct BufferUses { + /// Oldest first, one entry a tick. + std::vector ticks; + }; + + /// Drops the uses of submissions the host GPU finished: nothing recorded now races them. + void Prune(BufferUses& uses) const { + const u64 current = scheduler.CurrentTick(); + auto it = uses.ticks.begin(); + while (it != uses.ticks.end() && it->tick < current && scheduler.IsFree(it->tick)) { + ++it; + } + uses.ticks.erase(uses.ticks.begin(), it); + } + + void Sweep() { + for (auto it = buffers.begin(); it != buffers.end();) { + Prune(it->second); + it = it->second.ticks.empty() ? buffers.erase(it) : std::next(it); + } + } + + void Report(std::chrono::steady_clock::time_point now) { + const double seconds = std::chrono::duration(now - window_start).count(); + LOG_INFO(Render_Vulkan, + "Upload reorder check in {:.1f} s: {} uploads from the stream buffer: {} " + "reordered ({} of them after unfinished work of earlier submissions on their " + "bytes), {} in order ({} after a use of their bytes in the same submission); " + "errors: {} reordered past a use of their bytes, {} reordered ahead of unfinished " + "earlier work on their bytes without waiting for it (since start: {} and {} in {} " + "uploads)", + seconds, window.uploads, window.reordered, window.reordered_after_unfinished, + window.in_order, window.in_order_used, window.past_use, window.unordered, + total.past_use, total.unordered, total.uploads); + window = {}; + window_start = now; + } + + Scheduler& scheduler; + std::mutex mutex; + std::unordered_map buffers; + u64 notes_since_sweep{}; + Counts window{}; + Counts total{}; + u64 errors_logged{}; + std::chrono::steady_clock::time_point window_start = std::chrono::steady_clock::now(); +}; + +std::unique_ptr UploadReorderCheck::Create(Scheduler& scheduler) { + const char* const value = std::getenv("EDEN_NX2_UPLOAD_REORDER_CHECK"); + if (value == nullptr || value[0] == '\0' || value[0] == '0') { + return nullptr; + } + std::unique_ptr check{new UploadReorderCheck(scheduler)}; + active = check.get(); + LOG_INFO(Render_Vulkan, + "Upload reorder check (EDEN_NX2_UPLOAD_REORDER_CHECK): checking the place of every " + "upload from the stream buffer against an exact model of the bytes each " + "submission's work uses"); + return check; +} + +UploadReorderCheck::UploadReorderCheck(Scheduler& scheduler) + : impl{std::make_unique(scheduler)} {} + +UploadReorderCheck::~UploadReorderCheck() { + if (active == this) { + active = nullptr; + } +} + +void UploadReorderCheck::NoteUse(VkBuffer buffer, u64 offset, u64 size, Source source) { + if (buffer == VK_NULL_HANDLE || size == 0) { + return; + } + Impl& self = *impl; + std::scoped_lock lock{self.mutex}; + const u64 tick = self.scheduler.CurrentTick(); + Impl::BufferUses& uses = self.buffers[buffer]; + self.Prune(uses); + if (uses.ticks.empty() || uses.ticks.back().tick < tick) { + uses.ticks.push_back(Impl::TickUses{tick, {}}); + } + // A whole-size binding ends at the end of the address space instead of wrapping around. + const u64 end = size > ~u64{0} - offset ? ~u64{0} : offset + size; + Insert(uses.ticks.back().ranges, offset, end, 1u << static_cast(source)); + if (++self.notes_since_sweep >= 1u << 16) { + self.notes_since_sweep = 0; + self.Sweep(); + } +} + +void UploadReorderCheck::CheckUpload(VkBuffer buffer, + std::span copies, + VideoCommon::UploadOrder order, std::source_location site) { + if (copies.empty()) { + return; + } + Impl& self = *impl; + std::scoped_lock lock{self.mutex}; + const u64 tick = self.scheduler.CurrentTick(); + // What work recorded before this upload in its submission uses of its bytes, and the newest + // earlier submission not known finished whose work does. + u32 used_now = 0; + u64 unfinished = 0; + const VideoCommon::BufferCopy* used_copy = nullptr; + if (const auto it = self.buffers.find(buffer); it != self.buffers.end()) { + self.Prune(it->second); + for (const Impl::TickUses& entry : it->second.ticks) { + for (const VideoCommon::BufferCopy& copy : copies) { + const u32 sources = + Overlap(entry.ranges, copy.dst_offset, copy.dst_offset + copy.size); + if (sources == 0) { + continue; + } + if (entry.tick == tick) { + used_now |= sources; + used_copy = used_copy ? used_copy : © + } else if (entry.tick < tick && !self.scheduler.IsFree(entry.tick)) { + unfinished = (std::max)(unfinished, entry.tick); + } + } + } + } + // A reordered upload waits for earlier work only through the submission's barrier, which is + // left out once the host GPU has finished order.prior_tick: no unfinished work after that + // submission may use its bytes. + const bool reordered = order.reorder; + const bool past_use = reordered && used_now != 0; + const bool unordered = reordered && unfinished > order.prior_tick; + for (Counts* const counts : {&self.window, &self.total}) { + ++counts->uploads; + if (reordered) { + ++counts->reordered; + counts->reordered_after_unfinished += unfinished != 0 ? 1 : 0; + } else { + ++counts->in_order; + counts->in_order_used += used_now != 0 ? 1 : 0; + } + counts->past_use += past_use ? 1 : 0; + counts->unordered += unordered ? 1 : 0; + } + if ((past_use || unordered) && + (self.errors_logged++ < 64 || (self.errors_logged & 1023) == 0)) { + const VideoCommon::BufferCopy& copy = used_copy ? *used_copy : copies.front(); + if (past_use) { + LOG_ERROR(Render_Vulkan, + "Upload reorder check: an upload into buffer {:#x} [{:#x}, {:#x}) at {}:{} " + "was put in the upload command buffer of submission {}, ahead of {} of its " + "bytes recorded before it in that submission", + HandleValue(buffer), copy.dst_offset, copy.dst_offset + copy.size, + site.file_name(), site.line(), tick, SourceList(used_now)); + } else { + LOG_ERROR(Render_Vulkan, + "Upload reorder check: an upload into buffer {:#x} at {}:{} was put in the " + "upload command buffer of submission {} after submission {} only, while " + "work of submission {} using its bytes is not known finished", + HandleValue(buffer), site.file_name(), site.line(), tick, order.prior_tick, + unfinished); + } + } + const auto now = std::chrono::steady_clock::now(); + if (now - self.window_start >= std::chrono::seconds{10}) { + self.Report(now); + } +} + +} // namespace Vulkan diff --git a/src/video_core/renderer_vulkan/vk_upload_reorder_check.h b/src/video_core/renderer_vulkan/vk_upload_reorder_check.h new file mode 100644 index 0000000000..f3e1079d3d --- /dev/null +++ b/src/video_core/renderer_vulkan/vk_upload_reorder_check.h @@ -0,0 +1,75 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later + +#pragma once + +#include +#include +#include + +#include "common/common_types.h" +#include "video_core/buffer_cache/upload_order.h" +#include "video_core/vulkan_common/vulkan_wrapper.h" + +namespace VideoCommon { +struct BufferCopy; +} + +namespace Vulkan { + +class Scheduler; + +/// TEST (EDEN_NX2_UPLOAD_REORDER_CHECK=1): checks the place of every upload from the stream +/// buffer against an exact model of which bytes of which buffers the work recorded in each +/// submission uses. The model is kept apart from the buffers' own tracking (SubmissionUses): it +/// hears of every use the runtime records (descriptor, texel, vertex, index and transform +/// feedback bindings, multi-range sources, copies, clears) and of every use the buffer cache +/// marks (Buffer::MarkUsage), byte for byte. It counts and logs: +/// - an upload placed in the upload command buffer although work recorded before it in the same +/// submission uses a byte it writes (it would run before that work); +/// - one placed there without the barrier on earlier submissions' work although such work, not +/// known finished, uses a byte it writes. +/// Both must stay 0. +class UploadReorderCheck { +public: + enum class Source : u8 { + Marked, + Descriptor, + Texel, + Vertex, + Index, + TransformFeedback, + MultiRange, + CopyRead, + CopyWrite, + Clear, + Count, + }; + + /// Null when the variable is unset. + [[nodiscard]] static std::unique_ptr Create(Scheduler& scheduler); + + ~UploadReorderCheck(); + + /// The check in use, null when off. + [[nodiscard]] static UploadReorderCheck* Active() noexcept { + return active; + } + + /// Work recorded now uses [offset, offset + size) of buffer. + void NoteUse(VkBuffer buffer, u64 offset, u64 size, Source source); + + /// An upload from the stream buffer writing `copies` of buffer is recorded with `order`. + void CheckUpload(VkBuffer buffer, std::span copies, + VideoCommon::UploadOrder order, std::source_location site); + +private: + explicit UploadReorderCheck(Scheduler& scheduler); + + struct Impl; + std::unique_ptr impl; + + static inline UploadReorderCheck* active = nullptr; +}; + +} // namespace Vulkan From d4e8d85d4d4f96f39a9541939edc3df723a84d3d Mon Sep 17 00:00:00 2001 From: jpkto <255363096+jpkto@users.noreply.github.com> Date: Sun, 8 Nov 2026 05:44:52 +0100 Subject: [PATCH 051/105] video_core: command entries don't wait for the writers before the last sync Before the split's front parses an entry on pages no GPU work has written yet, it waits until its back has run the compute and copy work sent before the last host sync (GpuSplit::WaitForSyncedWriters, the "list writers" site): a page learns it is GPU-written only when the back records the work, and that work could write the entry first. In Ocarina of Time's Hyrule Field (test49c) that wait takes 263 ms of every second, 1,104 waits a second, and each one leaves the back with nothing queued: 9,602 of 11,037 waits in 10 s ended with fewer than 64 ring words left (EDEN_NX2_LIST_DUMP, a diagnostic of the test builds that is not part of this series). The same dump shows what the wait protects. Every list the front read from GPU-written pages was data of a macro begun in the entry before: 3D 0xe62 took two words (24,697 lists in 10 s), compute 0xe74 three (7,059), the group counts compute culling writes. None held a method header, and no wait ever found its entry newly GPU-written. 6,743 of the waits (1,350 ms) had no GPU-written list within 16 entries after them. So an entry that begins between methods, a command buffer the CPU writes, no longer waits for compute work. Entries that begin inside a method's data, the way the parameters GPU work writes are fetched, wait as before. An entry resumed after an acquire resumes after a puller method, which no parameter list holds: it waits only if it resumes inside a method's data. The copy engine can copy GPU-written words into any memory: while copy work the back has not run is queued (DmaPusher notes its position), every entry waits as before. A command buffer that compute work writes for the first time while the entry waits behind it would now be read without that write: EDEN_NX2_DRAIN_CHECK checks every entry that skipped the wait (next commit). The waits before the GPU-written lists move to the lists themselves ("list copy writers", ReadGpuWrittenCommands); the others go away. Tested: the syntax check passes on dma_pusher.cpp and clang's strict warnings report nothing on the changed lines. Rebased: the dump's hook around the writer wait is not part of this series. Co-Authored-By: Claude Opus 5.5 --- src/video_core/dma_pusher.cpp | 30 +++++++++++++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/src/video_core/dma_pusher.cpp b/src/video_core/dma_pusher.cpp index 6cf0208122..7c791679b6 100644 --- a/src/video_core/dma_pusher.cpp +++ b/src/video_core/dma_pusher.cpp @@ -44,6 +44,18 @@ constexpr u32 ComputeInline = 0x6D; namespace { +/// The split's position after the last copy engine method that may write memory (the front's). +u64 g_last_copy_writer{}; + +/// Whether copy engine work sent so far may still be queued on the split's back. A new session +/// starts the split's positions over: a position past the current one is from an older session. +[[nodiscard]] bool CopyWritersPending(const VideoCommon::GpuSplit& split) { + if (g_last_copy_writer > split.Position()) { + g_last_copy_writer = 0; + } + return !split.Ran(g_last_copy_writer); +} + /// EDEN_NX2_LIST_STATS and _VERIFY: the front read `words` of a command list at `address` /// (`device`) on a GPU-written page, as `where` tells; see Nx2ListStats::NoteRead. void NoteListRead(MemoryManager& memory, s32 channel, GPUVAddr address, u64 device, @@ -269,7 +281,17 @@ bool DmaPusher::Step() { // queued before it. The GPU fetches the list after that work only past a host-level wait // (NoteSync): later work is not ordered before the fetch, and the list does not wait for // it (nor for draws, which write memory too; draw-written lists wait for the last sync). - if (!gpu_written && VideoCommon::GpuSplit::Get().Active() && + // Lists that begin between methods don't wait for compute work either: they are command + // buffers the CPU writes. The words GPU work writes for the front to fetch are the + // parameters of a method begun in the entry before (an indirect dispatch's or draw's, + // as in Ocarina of Time and title 1), which begin inside its data. An entry resumed after + // an acquire resumes after a puller method, which no parameter list holds. The copy + // engine can copy GPU-written words anywhere: while its work is queued, every list waits. + // The lists that did not wait are checked on the back (EDEN_NX2_DRAIN_CHECK). + const bool command_entry = dma_state.method_count == 0; + const bool waits_for_writers = + !command_entry || CopyWritersPending(VideoCommon::GpuSplit::Get()); + if (!gpu_written && VideoCommon::GpuSplit::Get().Active() && waits_for_writers && VideoCommon::GpuSplit::Get().WaitForSyncedWriters()) { gpu_written = IsGpuWrittenCommands(dma_state.dma_get, size_t{header.size} * sizeof(u32)); @@ -540,6 +562,9 @@ void DmaPusher::CallMethod(u32 argument) { SCOPE_EXIT { if (writer) { split.NoteWriter(); + if (engine_type == Engines::EngineTypes::MaxwellDMA) { + g_last_copy_writer = split.Position(); + } } }; if (split.Active()) { @@ -601,6 +626,9 @@ void DmaPusher::CallMultiMethod(const u32* base_start, u32 num_methods) { SCOPE_EXIT { if (writer) { split.NoteWriter(); + if (engine_type == Engines::EngineTypes::MaxwellDMA) { + g_last_copy_writer = split.Position(); + } } }; if (split.Active()) { From f7d4131e1e116bcbffddd5ebf0296b9a1c44f4cc Mon Sep 17 00:00:00 2001 From: jpkto <255363096+jpkto@users.noreply.github.com> Date: Sun, 8 Nov 2026 05:51:39 +0100 Subject: [PATCH 052/105] TEST: check the command entries parsed without waiting for GPU work (EDEN_NX2_DRAIN_CHECK, EDEN_NX2_ENTRY_CHECK) Command entries that begin between methods no longer wait for the compute work sent before the last host sync. That rests on such work never writing a command buffer for the first time while an entry on it waits behind the work. And entries that no page bit marks GPU-written run from the words the CPU thread copied at submit time, or from guest memory as the front reads it. Neither sees what GPU work writes to guest memory past the caches (inline uploads, semaphore releases, reports) after that copy or read: whether any title fetches command words written that way is unknown. EDEN_NX2_DRAIN_CHECK now also checks every entry that did not wait for the queued writers. At its place the back looks for pages GPU work has made GPU-written by then. For those it flushes the entry's bytes to guest memory and compares the words the front parsed: differing ones must be 0, and the first ones are logged with their words. The older check of entries that skipped only the writers after the last sync keeps its counts. It also counts the command entries that waited because copy work was queued. EDEN_NX2_ENTRY_CHECK compares every entry run from its submit-time copy or from guest memory, at its place on the back, with guest memory. Differences are words the CPU or GPU work wrote after the front took them; the first 16 are logged with both versions. Both report in one line every 10 s from the GPU thread (nx2_entry_check.h). With neither variable set, Step tests one constant. Tested: the syntax check passes on dma_pusher.cpp and clang's strict warnings report nothing on the changed lines. A harness runs the real split and back thread with a real DmaPusher and GPU memory manager. A rasterizer whose flushes can bring "GPU bytes" to guest memory checks both checks and the summary (17 checks pass, five runs on two CPUs and three pinned to one): a page left alone, a page queued work made GPU-written with and without changing the words, stale and current submit-time copies, and an inline upload queued before an entry the front read from memory. Rebased: the EDEN_NX2_LIST_DUMP hooks this commit was written next to are an investigation diagnostic that is not part of this series. EDEN_NX2_ENTRY_CHECK is a plain flag read again by VideoCore::CreateGPU, as upstream reads its diagnostic switches since #714, so it works from nx2_switches.txt too, not only from the environment. Co-Authored-By: Claude Opus 5.5 --- src/video_core/CMakeLists.txt | 1 + src/video_core/dma_pusher.cpp | 14 +- src/video_core/nx2_entry_check.h | 244 +++++++++++++++++++++++++++++++ src/video_core/video_core.cpp | 2 + 4 files changed, 260 insertions(+), 1 deletion(-) create mode 100644 src/video_core/nx2_entry_check.h diff --git a/src/video_core/CMakeLists.txt b/src/video_core/CMakeLists.txt index 70cfd98f2e..6e3306933f 100644 --- a/src/video_core/CMakeLists.txt +++ b/src/video_core/CMakeLists.txt @@ -121,6 +121,7 @@ add_library(video_core STATIC nx2_debug_control.cpp nx2_debug_control.h nx2_drain_check.h + nx2_entry_check.h nx2_list_stats.h nx2_profile.h nx2_perf.h diff --git a/src/video_core/dma_pusher.cpp b/src/video_core/dma_pusher.cpp index 7c791679b6..71aabd98be 100644 --- a/src/video_core/dma_pusher.cpp +++ b/src/video_core/dma_pusher.cpp @@ -34,6 +34,7 @@ #include "video_core/nx2.h" #include "video_core/nx2_cmdpage_diag.h" #include "video_core/nx2_drain_check.h" +#include "video_core/nx2_entry_check.h" #include "video_core/nx2_list_stats.h" #include "video_core/rasterizer_interface.h" @@ -296,7 +297,8 @@ bool DmaPusher::Step() { gpu_written = IsGpuWrittenCommands(dma_state.dma_get, size_t{header.size} * sizeof(u32)); } - if (VideoCommon::Nx2DrainCheck::g_enabled && !gpu_written) [[unlikely]] { + if (VideoCommon::Nx2DrainCheck::g_enabled && !gpu_written && waits_for_writers) + [[unlikely]] { CheckUnwaitedWriters(dma_state.dma_get, size_t{header.size} * sizeof(u32)); } // Resume and GPU-written entries need live words, even after a flush cleared dirty state. @@ -311,6 +313,16 @@ bool DmaPusher::Step() { gpu_written, use_safe && memory_manager.IsMemoryDirty(dma_state.dma_get, header.size * sizeof(u32))); + if (Nx2EntryCheck::Any() && !gpu_written && !use_safe) [[unlikely]] { + Nx2EntryCheck::NoteEntry( + *this, header, + submitted ? std::span{command_list.submitted_words.data() + + command_list.submitted_offsets + [dma_pushbuffer_subindex], + header.size} + : std::span{}, + !waits_for_writers, command_entry && waits_for_writers); + } if (submitted) { const std::span commands{ command_list.submitted_words.data() + diff --git a/src/video_core/nx2_entry_check.h b/src/video_core/nx2_entry_check.h new file mode 100644 index 0000000000..29004b2585 --- /dev/null +++ b/src/video_core/nx2_entry_check.h @@ -0,0 +1,244 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later + +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "common/common_types.h" +#include "common/logging.h" +#include "video_core/cache_types.h" +#include "video_core/dma_pusher.h" +#include "video_core/gpu_split.h" +#include "video_core/memory_manager.h" +#include "video_core/nx2.h" +#include "video_core/nx2_drain_check.h" +#include "video_core/rasterizer_interface.h" + +// TEST: checks of the command entries DmaPusher::Step parses without the GPU-written path, from +// their submit-time copy or from guest memory. +// +// EDEN_NX2_DRAIN_CHECK: the entries that began between methods and so did not wait for the +// compute work sent before the last sync while some was queued. At each one's place the back +// looks for pages that work has made GPU-written by then, brings the GPU's bytes of those to guest +// memory and compares the words the front parsed: differing ones must be 0. +// +// EDEN_NX2_ENTRY_CHECK: every such entry compared at its place on the back with guest memory: +// words written after the front took them, by the CPU or by GPU work that writes guest memory +// past the caches (inline uploads, semaphore releases, reports), which no page bit tells about. +// +// One line every 10 s from the GPU thread. +namespace Tegra::Nx2EntryCheck { + +// A plain flag: checked at every command entry. +inline bool g_entries = std::getenv("EDEN_NX2_ENTRY_CHECK") != nullptr; + +/// Called by VideoCore::CreateGPU: static initialization runs before the frontend loads +/// nx2_switches.txt, so a switch set only there takes effect from that point on. +inline void ReadSwitch() { + g_entries = std::getenv("EDEN_NX2_ENTRY_CHECK") != nullptr; +} + +[[nodiscard]] inline bool Any() { + return g_entries || VideoCommon::Nx2DrainCheck::g_enabled; +} + +/// What the front parsed: the first words and a hash of all. +struct Parsed { + std::array head{}; + u32 words{}; + u64 hash{}; +}; + +[[nodiscard]] inline Parsed Summarize(std::span words) { + Parsed parsed; + parsed.words = static_cast(words.size()); + parsed.hash = HashPushbufferWords(words); + for (size_t i = 0; i < parsed.head.size() && i < words.size(); ++i) { + parsed.head[i] = words[i]; + } + return parsed; +} + +/// The words of guest memory at [address, address + words * 4). +[[nodiscard]] inline Parsed SummarizeMemory(MemoryManager& memory, GPUVAddr address, u32 words) { + std::vector data(words); + if (words != 0) { + memory.ReadBlockUnsafe(address, data.data(), data.size() * sizeof(u32)); + } + return Summarize(data); +} + +enum Path : u32 { Copy, Memory, PathCount }; + +struct State { + // Front. + std::atomic skipped{}; ///< Entries that did not wait for queued writers + std::atomic kept_for_copies{}; ///< Command entries that waited for queued copy work + std::array, PathCount> compared{}; + // Back. + std::atomic skipped_checked{}; + std::atomic skipped_written{}; ///< On a page GPU work had made GPU-written at its place + std::atomic skipped_differ{}; ///< Of those, words differing after a flush + std::array, PathCount> differ{}; + std::atomic logged{}; + std::chrono::steady_clock::time_point last_log{std::chrono::steady_clock::now()}; +}; + +inline State& Get() { + static State state; + return state; +} + +[[nodiscard]] inline std::string Hex(std::span words, u32 count) { + std::string text; + for (u32 i = 0; i < count && i < words.size(); ++i) { + text += fmt::format("{}{:08x}", text.empty() ? "" : " ", words[i]); + } + return text.empty() ? std::string{"-"} : text; +} + +inline void LogDiffer(const char* what, GPUVAddr address, const Parsed& parsed, + const Parsed& memory) { + State& state = Get(); + if (state.logged.fetch_add(1, std::memory_order_relaxed) >= 16) { + return; + } + LOG_WARNING(HW_GPU, + "Command entry check: {} at {:#x} ({} words): parsed starts {}, guest memory " + "{} at the back's place", + what, address, parsed.words, Hex(parsed.head, parsed.words), + Hex(memory.head, memory.words)); +} + +/// Back, at the place of an entry that did not wait for the writers before the last sync: were +/// its words written by that work? +inline void CheckSkipped(MemoryManager* memory, VideoCore::RasterizerInterface* rasterizer, + GPUVAddr address, Parsed parsed) { + State& state = Get(); + state.skipped_checked.fetch_add(1, std::memory_order_relaxed); + const GPUVAddr end = address + u64{parsed.words} * sizeof(u32); + bool written = false; + for (GPUVAddr page = address & ~GPUVAddr{0xfff}; page < end; page += 0x1000) { + const auto device = memory->GpuToCpuAddress(page); + written = written || (device && Nx2IsGpuWritten(*device)); + } + if (!written) { + return; + } + state.skipped_written.fetch_add(1, std::memory_order_relaxed); + if (rasterizer != nullptr) { + // The GPU's bytes of the entry's pages, in guest memory for the comparison. + for (GPUVAddr piece = address; piece < end;) { + const GPUVAddr piece_end = (std::min)(end, (piece & ~GPUVAddr{0xfff}) + 0x1000); + if (const auto device = memory->GpuToCpuAddress(piece)) { + rasterizer->FlushRegion(*device, piece_end - piece, + VideoCommon::CacheType::BufferCache); + } + piece = piece_end; + } + } + const Parsed now = SummarizeMemory(*memory, address, parsed.words); + if (now.hash != parsed.hash) { + state.skipped_differ.fetch_add(1, std::memory_order_relaxed); + LogDiffer("an entry that did not wait for the writers differs from GPU-written memory", + address, parsed, now); + } +} + +/// Back, at an entry's place (EDEN_NX2_ENTRY_CHECK): does guest memory still hold what the front +/// parsed? +inline void CheckCompared(MemoryManager* memory, GPUVAddr address, Parsed parsed, Path path) { + const Parsed now = SummarizeMemory(*memory, address, parsed.words); + if (now.hash == parsed.hash) { + return; + } + Get().differ[path].fetch_add(1, std::memory_order_relaxed); + LogDiffer(path == Copy ? "an entry run from its submit-time copy differs from guest memory" + : "an entry run from guest memory differs from guest memory later", + address, parsed, now); +} + +inline void MaybeLog() { + State& state = Get(); + const auto now = std::chrono::steady_clock::now(); + const double seconds = std::chrono::duration(now - state.last_log).count(); + if (seconds < 10.0) { + return; + } + state.last_log = now; + const auto take = [](std::atomic& counter) { + return counter.exchange(0, std::memory_order_relaxed); + }; + const u64 skipped = take(state.skipped); + const u64 kept = take(state.kept_for_copies); + const u64 checked = take(state.skipped_checked); + const u64 written = take(state.skipped_written); + const u64 differ = take(state.skipped_differ); + const u64 copies = take(state.compared[Copy]); + const u64 copies_differ = take(state.differ[Copy]); + const u64 reads = take(state.compared[Memory]); + const u64 reads_differ = take(state.differ[Memory]); + LOG_INFO(HW_GPU, + "Command entry checks in {:.1f} s: entries that did not wait for queued writers {} " + "({} checked at their place, on pages GPU work had made GPU-written {}, their words " + "differing after a flush {}), command entries that waited for queued copy work {} | " + "entries compared with guest memory at their place: run from their submit-time copy " + "{} ({} differ), from guest memory {} ({} differ)", + seconds, skipped, checked, written, differ, kept, copies, copies_differ, reads, + reads_differ); +} + +/// Front, before it parses the entry `header` of `pusher` from its submit-time copy (`copy`, else +/// guest memory). `skipped_writers`: the entry did not wait for the writers before the last sync; +/// `kept_for_copies`: a command entry that waited for them because copy work was queued. +inline void NoteEntry(DmaPusher& pusher, const CommandListHeader& header, + std::span copy, bool skipped_writers, + bool kept_for_copies) { + MaybeLog(); + auto& split = VideoCommon::GpuSplit::Get(); + if (!split.Active()) { + return; + } + const GPUVAddr address = header.addr.Value(); + const auto words = static_cast(header.size.Value()); + Parsed parsed; + if (!copy.empty()) { + std::vector data(copy.size()); + for (size_t i = 0; i < copy.size(); ++i) { + data[i] = copy[i].argument; + } + parsed = Summarize(data); + } else { + parsed = SummarizeMemory(pusher.memory_manager, address, words); + } + State& state = Get(); + if (VideoCommon::Nx2DrainCheck::g_enabled) { + if (kept_for_copies && split.WritersPending()) { + state.kept_for_copies.fetch_add(1, std::memory_order_relaxed); + } + if (skipped_writers && split.WritersPending()) { + state.skipped.fetch_add(1, std::memory_order_relaxed); + split.Defer([memory = &pusher.memory_manager, rasterizer = pusher.rasterizer, address, + parsed] { CheckSkipped(memory, rasterizer, address, parsed); }, + VideoCommon::GpuSplit::Writes::Caches); + } + } + if (g_entries) { + const Path path = copy.empty() ? Memory : Copy; + state.compared[path].fetch_add(1, std::memory_order_relaxed); + split.Defer([memory = &pusher.memory_manager, address, parsed, + path] { CheckCompared(memory, address, parsed, path); }, + VideoCommon::GpuSplit::Writes::Noted); + } +} + +} // namespace Tegra::Nx2EntryCheck diff --git a/src/video_core/video_core.cpp b/src/video_core/video_core.cpp index 251b10e679..4753d685a7 100644 --- a/src/video_core/video_core.cpp +++ b/src/video_core/video_core.cpp @@ -17,6 +17,7 @@ #include "video_core/nx2_cmdpage_diag.h" #include "video_core/nx2_download_trace.h" #include "video_core/nx2_drain_check.h" +#include "video_core/nx2_entry_check.h" #include "video_core/nx2_list_stats.h" #include "video_core/nx2_perf.h" #include "video_core/nx2_profile.h" @@ -64,6 +65,7 @@ void CreateGPU(std::optional& gpu, Core::Frontend::EmuWindow& emu_wi Tegra::Nx2ListStats::ReadSwitches(); VideoCommon::Nx2AcquireStats::ReadSwitch(); VideoCommon::Nx2Perf::ReadSwitch(); + Tegra::Nx2EntryCheck::ReadSwitch(); VideoCommon::ReadUniformStreamReuseSwitch(); const auto nvdec_value = Settings::values.nvdec_emulation.GetValue(); From 4942eb8a398c857a0b214672d8cdb1f56c03b3f6 Mon Sep 17 00:00:00 2001 From: jpkto <255363096+jpkto@users.noreply.github.com> Date: Sun, 8 Nov 2026 06:01:00 +0100 Subject: [PATCH 053/105] core, video_core: let guest writes wait for a download landing under them The read download check of test49b found 11 bytes of one landed download that differed from its check copy: 0x2b648a20, three bytes of each of four words and two of the last, the top bytes equal. The download and its check copy are recorded back to back and read the same bytes, so guest memory changed between the landing and the check. The only writer that neither takes the buffer cache's lock first nor is left out of the check is a guest core. A guest CPU write into GPU-written memory is collected (HandleRasterizerWrite, GPUDirtyMemoryManager::Collect), then stored, without the buffer cache's lock. After a core's first write to such a page, its later writes there skip OnCPUWrite too: Ocarina of Time collects about 1.3 million writes a second this way, and the check sees about 11 KB a second of them over bytes that downloads landed. A download keeps the bytes of the writes collected when it looks at them (PendingCpuWriteRanges), then writes the rest. A write collected and stored in between was handled wrong: - after the landing's look and before its write, the download wrote the GPU's older bytes over the guest's. The guest's write was lost, in what the game reads. - after the check's look and before its read of guest memory, the check compared the guest's newer bytes with the GPU's copy: the test49b lines. The synchronous downloads (DownloadRangesMemory, DownloadBufferMemory) looked before they waited for the host GPU, so a write collected during that wait was overwritten as well. None of this is new in test49b; its verify run only lands more downloads a second. Each landing is now published (VideoCore::DownloadLanding: its range, and a sequence that is odd while it lasts). A download looks at the collected writes only inside its landing, and writes in it. HandleRasterizerWrite calls DownloadLanding::Await after Collect: a write in the range of a landing under way is stored once it ends. Each side fences between its store and its load, so either the landing's look finds the write and keeps its bytes, or the write finds the landing and comes after it. Downloads land under the buffer cache's lock, one at a time. A read download's landing also covers its check (EDEN_NX2_READ_CHECK), which then compares exactly what the download wrote. Every landing takes part: read downloads (WriteReadDownloads, early ones included), fence downloads (PopAsyncBuffers), and the synchronous ones, which now look once the host GPU has run their copies. A guest write pays a fence and a load. It waits only while a landing over its bytes is under way, a few microseconds. Tested in the read download harness (the real BufferCache, device memory manager and MemoryManager, with the fake runtime) with new tests in buffer_cache_ports.cpp. In them a second thread collects and stores a write as HandleRasterizerWrite does, right after a chosen look: - after a read download's look: the write was lost before this change; - after the check's look: before this change the check printed test49b's shape exactly (0x20a20+0x3 (0xcd for 0x68), 0x20a24+0x3, 0x20a28+0x3, 0x20a2c+0x2, 11 bytes differ); - an early download landed at a draw: the write was lost before; - during a synchronous download's wait for the host GPU, staged and mapped: the write was lost before; - 2000 writes racing read downloads at random delays: before, 2 of 3000 were lost and 326 checks reported differences. All 30 harness tests pass in 21 runs, 10 of them pinned to one CPU. Each of these breaks a test: the landing not published, the look before the landing, the check outside it, the synchronous look before the wait, an empty landing range. Co-Authored-By: Claude Opus 5.5 --- src/core/memory.cpp | 3 + src/tests/video_core/buffer_cache_ports.cpp | 282 ++++++++++++++++++ src/video_core/buffer_cache/buffer_cache.h | 96 ++++-- .../buffer_cache/buffer_cache_base.h | 4 +- src/video_core/rasterizer_download_area.h | 71 +++++ 5 files changed, 438 insertions(+), 18 deletions(-) diff --git a/src/core/memory.cpp b/src/core/memory.cpp index 5724301b12..7344e75c83 100644 --- a/src/core/memory.cpp +++ b/src/core/memory.cpp @@ -967,6 +967,9 @@ struct Memory::Impl { } } gpu_dirty_managers[core].Collect(address, size); + // A download writing to guest memory here may have looked at the collected writes + // before this one: its bytes would land over the ones stored after this returns. + VideoCore::DownloadLanding::Await(address, size); }); } diff --git a/src/tests/video_core/buffer_cache_ports.cpp b/src/tests/video_core/buffer_cache_ports.cpp index 40f15e96b8..fe6311603c 100644 --- a/src/tests/video_core/buffer_cache_ports.cpp +++ b/src/tests/video_core/buffer_cache_ports.cpp @@ -2,12 +2,16 @@ #include #include +#include #include +#include #include #include #include #include #include +#include +#include #include #include #include @@ -1549,3 +1553,281 @@ TEST_CASE("Read download check finds early downloaded bytes the GPU changed unre REQUIRE(ReadWord(f, 0x20010).preemtive); REQUIRE(Access::ReadChecks(*f.cache).compared == compared); } + +namespace { +/// Last frame's values of 16 bytes the guest CPU writes each frame (the GPU's copy), and this +/// frame's: three bytes of each word differ, the last word's top two don't. +constexpr std::array OldGuestValue{0x68, 0x34, 0x12, 0x3f, 0x8d, 0x11, 0x47, 0xbe, + 0x20, 0x55, 0x20, 0x41, 0x73, 0x0c, 0x40, 0x3f}; +constexpr std::array NewGuestValue{0xcd, 0x9a, 0x15, 0x3f, 0x95, 0x02, 0x49, 0xbe, + 0x4b, 0x13, 0x22, 0x41, 0x54, 0xe6, 0x40, 0x3f}; + +/// Guest CPU writes into GPU-written memory, as Core::Memory's HandleRasterizerWrite makes them +/// on any core: collected (GPUDirtyMemoryManager::Collect, in 64-byte blocks), then stored once +/// no download landing under them looked at the collected writes before. +class GuestCpuWrites { +public: + explicit GuestCpuWrites(Cache& cache_) : cache{cache_} { + cache.SetPendingCpuWrites([this](const std::function& callback) { + { + std::scoped_lock lock{mutex}; + for (const auto& [address, size] : collected) { + callback(address, size); + } + } + if (on_look) { + on_look(++looks); + } + }); + } + ~GuestCpuWrites() { + cache.SetPendingCpuWrites({}); + } + + void Write(u8* guest, DAddr address, std::span value) { + const DAddr first = Common::AlignDown(address, 64); + { + std::scoped_lock lock{mutex}; + collected.emplace_back(first, Common::AlignUp(address + value.size(), 64) - first); + } + VideoCore::DownloadLanding::Await(address, value.size()); + std::memcpy(guest, value.data(), value.size()); + } + + /// GPU::InvalidateGPUCache: the collected writes are applied. + void Apply() { + std::vector> applied; + { + std::scoped_lock lock{mutex}; + applied.swap(collected); + } + cache.ClaimCpuWrites(applied); + } + + /// Called after the buffer cache's n-th look at the collected writes. + std::function on_look; + int looks{}; + +private: + Cache& cache; + std::mutex mutex; + std::vector> collected; +}; + +/// Another core writes NewGuestValue at `address` right after the buffer cache's `at`-th look at +/// the collected writes: the look returns once the write is stored or waits for a landing. +class RacingGuestWrite { +public: + RacingGuestWrite(Fixture& f, GuestCpuWrites& writes_, DAddr address_, int at) + : writes{writes_}, guest{f.bytes + (address_ - 0x10000)}, address{address_} { + writes.looks = 0; + writes.on_look = [this, at](int look) { + if (look != at) { + return; + } + const u64 waits = VideoCore::DownloadLanding::Waits(); + go.store(true); + const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds{5}; + while (!stored.load() && VideoCore::DownloadLanding::Waits() == waits && + std::chrono::steady_clock::now() < deadline) { + std::this_thread::yield(); + } + }; + thread = std::thread([this] { + while (!go.load() && !quit.load()) { + std::this_thread::yield(); + } + if (go.load()) { + writes.Write(guest, address, NewGuestValue); + stored.store(true); + } + }); + } + ~RacingGuestWrite() { + quit.store(true); + thread.join(); + writes.on_look = {}; + } + + /// Whether the write was stored, waiting up to 5 s for it. + bool Stored() { + const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds{5}; + while (go.load() && !stored.load() && std::chrono::steady_clock::now() < deadline) { + std::this_thread::yield(); + } + return stored.load(); + } + +private: + GuestCpuWrites& writes; + u8* guest; + DAddr address; + std::atomic go{}; + std::atomic stored{}; + std::atomic quit{}; + std::thread thread; +}; + +/// The buffer at 0x20000, its first page written by a dispatch (the whole storage binding counts) +/// and read back, with OldGuestValue at 0x20a20 in the GPU's copy and in guest memory. +VideoCommon::BufferId GuestWrittenPage(Fixture& f) { + const auto id = Access::Add(*f.cache, 0x20000, 0x10000); + auto& buffer = Access::Get(*f.cache, id); + std::ranges::fill(buffer.bytes, 0x12); + std::ranges::copy(OldGuestValue, buffer.bytes.begin() + 0xa20); + std::ranges::copy(OldGuestValue, f.bytes + 0x10a20); + Access::MarkWritten(*f.cache, id, 0x20000, 0x1000); + return id; +} + +bool GuestHolds(const Fixture& f, DAddr address, std::span value) { + return std::equal(value.begin(), value.end(), f.bytes + (address - 0x10000)); +} +} // namespace + +TEST_CASE("A guest write stored while a read download lands keeps its bytes", "[buffer_ports]") { + Fixture f; + GuestWrittenPage(f); + Access::CheckEveryRead(*f.cache); + GuestCpuWrites writes{*f.cache}; + f.cache->DownloadMemoryForRead(0x20000, 0x1000); + const u64 waits = VideoCore::DownloadLanding::Waits(); + { + // Collected right after the landing looked at the collected writes. + RacingGuestWrite race{f, writes, 0x20a20, 1}; + f.cache->FinishReadDownloads(0x20000, 0x1000); + REQUIRE(race.Stored()); + } + REQUIRE(GuestHolds(f, 0x20a20, NewGuestValue)); + REQUIRE(f.bytes[0x10a1f] == 0x12); + REQUIRE(f.bytes[0x10a30] == 0x12); + // It waited for the landing to end. + REQUIRE(VideoCore::DownloadLanding::Waits() > waits); + REQUIRE(Access::ReadChecks(*f.cache).checked == 1); + REQUIRE(Access::ReadChecks(*f.cache).mismatched == 0); +} + +TEST_CASE("Read download check leaves out a guest write stored while it compares", + "[buffer_ports]") { + Fixture f; + GuestWrittenPage(f); + Access::CheckEveryRead(*f.cache); + GuestCpuWrites writes{*f.cache}; + f.cache->DownloadMemoryForRead(0x20000, 0x1000); + { + // The download is in guest memory: collected right after the check looked at the + // collected writes, before it reads guest memory. + RacingGuestWrite race{f, writes, 0x20a20, 2}; + f.cache->FinishReadDownloads(0x20000, 0x1000); + REQUIRE(race.Stored()); + } + REQUIRE(GuestHolds(f, 0x20a20, NewGuestValue)); + REQUIRE(Access::ReadChecks(*f.cache).checked == 1); + REQUIRE(Access::ReadChecks(*f.cache).mismatched == 0); +} + +TEST_CASE("An early download landed at a draw keeps a guest write stored meanwhile", + "[buffer_ports]") { + Fixture f; + const Nx2TitleScope nx2_title; + const auto id = Access::Add(*f.cache, 0x20000, 0x10000); + auto& buffer = Access::Get(*f.cache, id); + Access::CheckEveryRead(*f.cache); + GuestCpuWrites writes{*f.cache}; + for (u8 frame = 1; frame <= 2; ++frame) { + WriteByDispatch(f, id, 0x20000, 0x1000, frame); + ReadWord(f, 0x20010); + } + // The next write gets its early download. The host GPU runs it, and the GPU thread lands it + // at a draw while another core writes into the page. + WriteByDispatch(f, id, 0x20000, 0x1000, 3); + std::ranges::copy(OldGuestValue, buffer.bytes.begin() + 0xa20); + f.runtime.Complete(f.runtime.submitted_tick); + REQUIRE(Access::ReadDownloads(*f.cache) == 1); + { + RacingGuestWrite race{f, writes, 0x20a20, 1}; + for (u32 draw = 0; draw < 32; ++draw) { + f.cache->NoteWorkRecorded(false); + } + REQUIRE(Access::ReadDownloads(*f.cache) == 0); + REQUIRE(race.Stored()); + } + REQUIRE(GuestHolds(f, 0x20a20, NewGuestValue)); + REQUIRE(f.bytes[0x10a1f] == 3); + REQUIRE(f.bytes[0x10a30] == 3); + REQUIRE(Access::ReadChecks(*f.cache).mismatched == 0); +} + +TEST_CASE("Synchronous downloads keep guest writes collected while they wait for the host GPU", + "[buffer_ports]") { + Fixture f; + const bool mapped = GENERATE(false, true); + const auto id = Access::Add(*f.cache, 0x20000, 0x10000, mapped); + std::ranges::fill(Access::Get(*f.cache, id).bytes, 0x12); + GuestCpuWrites writes{*f.cache}; + // Collected and stored while the GPU thread waits for the host GPU's copies. + f.runtime.before_completion = [&] { + f.runtime.before_completion = {}; + writes.Write(f.bytes + 0x10100, 0x20100, NewGuestValue); + }; + Access::Dirty(*f.cache, 0x20000, 0x2000); + const std::array, 1> ranges{{{0x20000, 0x2000}}}; + Access::Download(*f.cache, ranges); + REQUIRE(GuestHolds(f, 0x20100, NewGuestValue)); + REQUIRE(f.bytes[0x100ff] == 0x12); + REQUIRE(f.bytes[0x10140] == 0x12); + // The same through DownloadMemory, one buffer at a time. + writes.Apply(); + f.runtime.before_completion = [&] { + f.runtime.before_completion = {}; + writes.Write(f.bytes + 0x10200, 0x20200, NewGuestValue); + }; + Access::Dirty(*f.cache, 0x20000, 0x2000); + f.cache->DownloadMemory(0x20000, 0x2000); + REQUIRE(GuestHolds(f, 0x20200, NewGuestValue)); + REQUIRE(f.bytes[0x101ff] == 0x12); + REQUIRE(f.bytes[0x10240] == 0x12); +} + +TEST_CASE("Guest writes racing read downloads keep their bytes", "[buffer_ports]") { + Fixture f; + const auto id = Access::Add(*f.cache, 0x20000, 0x10000); + auto& buffer = Access::Get(*f.cache, id); + Access::CheckEveryRead(*f.cache); + GuestCpuWrites writes{*f.cache}; + u32 seed = 1234; + const auto next_delay = [&seed] { + seed = seed * 1103515245 + 12345; + return (seed >> 8) % 4000; + }; + const auto spin = [](u32 iterations) { + for (volatile u32 i = 0; i < iterations; i = i + 1) { + } + }; + int lost = 0; + for (int iteration = 0; iteration < 2000; ++iteration) { + // The guest's earlier writes are applied, then a dispatch writes the page: 16 of its bytes + // hold the GPU's value until the guest writes this round's from another core. + writes.Apply(); + std::array gpu_value; + gpu_value.fill(static_cast(iteration * 2 + 1)); + std::array cpu_value; + cpu_value.fill(static_cast(iteration * 2 + 2)); + std::ranges::copy(gpu_value, buffer.bytes.begin() + 0xa20); + Access::MarkWritten(*f.cache, id, 0x20000, 0x1000); + f.cache->DownloadMemoryForRead(0x20000, 0x1000); + const u32 writer_delay = next_delay(); + const u32 reader_delay = next_delay(); + std::thread writer{[&] { + spin(writer_delay); + writes.Write(f.bytes + 0x10a20, 0x20a20, cpu_value); + }}; + spin(reader_delay); + f.cache->FinishReadDownloads(0x20000, 0x1000); + writer.join(); + lost += GuestHolds(f, 0x20a20, cpu_value) ? 0 : 1; + } + REQUIRE(lost == 0); + REQUIRE(Access::ReadChecks(*f.cache).mismatched == 0); + REQUIRE(Access::ReadChecks(*f.cache).uncovered == 0); +} diff --git a/src/video_core/buffer_cache/buffer_cache.h b/src/video_core/buffer_cache/buffer_cache.h index 8885549677..5e4163a6f1 100644 --- a/src/video_core/buffer_cache/buffer_cache.h +++ b/src/video_core/buffer_cache/buffer_cache.h @@ -460,11 +460,44 @@ void BufferCache

::DownloadRangesMemory(std::span> for (const Download& download : mapped) { GpuWaitStats::AddDownloadBytes(download.copy.size); } - const Common::RangeSet cpu_written = PendingCpuWriteRanges(); + // Once the host GPU ran the copies: the CPU writes collected meanwhile keep their bytes + // too. + const auto land = [&](const Async_Buffer* staging) { + DAddr landing_begin = ~DAddr{0}; + DAddr landing_end = 0; + const auto cover = [&](const Download& download) { + const DAddr address = + slot_buffers[download.buffer_id].CpuAddr() + download.copy.src_offset; + landing_begin = (std::min)(landing_begin, address); + landing_end = (std::max)(landing_end, address + download.copy.size); + }; + std::ranges::for_each(staged, cover); + std::ranges::for_each(mapped, cover); + const VideoCore::DownloadLanding landing{landing_begin, landing_end}; + const Common::RangeSet cpu_written = PendingCpuWriteRanges(); + if (staging != nullptr) { + for (const Download& download : staged) { + WriteDownloadedMemory( + cpu_written, + slot_buffers[download.buffer_id].CpuAddr() + download.copy.src_offset, + staging->mapped_span.data() + (download.copy.dst_offset - staging->offset), + download.copy.size); + } + } + if constexpr (has_mapped) { + for (const Download& download : mapped) { + Buffer& buffer = slot_buffers[download.buffer_id]; + WriteDownloadedMemory(cpu_written, buffer.CpuAddr() + download.copy.src_offset, + buffer.Mapped().data() + download.copy.src_offset, + download.copy.size); + } + } + }; if (staged.empty()) { if constexpr (has_mapped) { runtime.FinishForHostRead(); } + land(nullptr); } else { auto staging = runtime.DownloadStagingBuffer(total_size_bytes); for (Download& download : staged) { @@ -478,21 +511,7 @@ void BufferCache

::DownloadRangesMemory(std::span> } else { runtime.Finish(); } - for (const Download& download : staged) { - WriteDownloadedMemory( - cpu_written, - slot_buffers[download.buffer_id].CpuAddr() + download.copy.src_offset, - staging.mapped_span.data() + (download.copy.dst_offset - staging.offset), - download.copy.size); - } - } - if constexpr (has_mapped) { - for (const Download& download : mapped) { - Buffer& buffer = slot_buffers[download.buffer_id]; - WriteDownloadedMemory(cpu_written, buffer.CpuAddr() + download.copy.src_offset, - buffer.Mapped().data() + download.copy.src_offset, - download.copy.size); - } + land(&staging); } } } @@ -1125,6 +1144,20 @@ void BufferCache

::WriteReadDownloads(u64 tick) { if constexpr (!IS_OPENGL) { while (!read_downloads.empty() && read_downloads.front().tick <= tick) { ReadDownload& download = read_downloads.front(); + // Guest memory this writes, and TEST: what its check reads. The guest CPU writes + // collected from now on store their bytes once this download is in guest memory. + DAddr landing_begin = ~DAddr{0}; + DAddr landing_end = 0; + const auto cover = [&](const BufferCopy& copy) { + landing_begin = (std::min)(landing_begin, static_cast(copy.src_offset)); + landing_end = + (std::max)(landing_end, static_cast(copy.src_offset + copy.size)); + }; + std::ranges::for_each(download.copies, cover); + if (download.check) [[unlikely]] { + std::ranges::for_each(download.check->copies, cover); + } + const VideoCore::DownloadLanding landing{landing_begin, landing_end}; // CPU writes are collected before they reach guest memory, and stay pending until // applied, which takes them out of write_ranges (ClaimCpuWrites): either way they // keep their bytes. @@ -2066,6 +2099,13 @@ void BufferCache

::PopAsyncBuffers() { auto& async_buffer = async_buffers.front(); u8* base = async_buffer->mapped_span.data(); const size_t base_offset = async_buffer->offset; + DAddr landing_begin = ~DAddr{0}; + DAddr landing_end = 0; + for (const auto& copy : downloads) { + landing_begin = (std::min)(landing_begin, static_cast(copy.src_offset)); + landing_end = (std::max)(landing_end, static_cast(copy.src_offset + copy.size)); + } + const VideoCore::DownloadLanding landing{landing_begin, landing_end}; // This runs on the fencing thread while the CPU keeps writing: CPU writes collected since the // copy was taken are newer than it, and the GPU thread hasn't dropped them from `ranges` yet. const Common::RangeSet cpu_written = PendingCpuWriteRanges(); @@ -3863,6 +3903,14 @@ void BufferCache

::DownloadBuffersMemory(std::span buffer_ids) } runtime.Finish(); const u8* const mapped_memory = download_staging.mapped_span.data(); + DAddr landing_begin = ~DAddr{0}; + DAddr landing_end = 0; + for (const Download& download : downloads) { + const DAddr address = slot_buffers[download.buffer_id].CpuAddr() + download.copy.src_offset; + landing_begin = (std::min)(landing_begin, address); + landing_end = (std::max)(landing_end, address + download.copy.size); + } + const VideoCore::DownloadLanding landing{landing_begin, landing_end}; const Common::RangeSet cpu_written = PendingCpuWriteRanges(); for (const Download& download : downloads) { const DAddr device_addr = slot_buffers[download.buffer_id].CpuAddr() + download.copy.src_offset; @@ -3904,11 +3952,20 @@ void BufferCache

::DownloadBufferMemory(Buffer& buffer, DAddr device_addr, u64 return; } GpuWaitStats::AddDownloadBytes(total_size_bytes); - const Common::RangeSet cpu_written = PendingCpuWriteRanges(); + // Guest memory is written once the host GPU ran the copies: the CPU writes collected meanwhile + // keep their bytes too. + DAddr landing_begin = ~DAddr{0}; + DAddr landing_end = 0; + for (const BufferCopy& copy : copies) { + landing_begin = (std::min)(landing_begin, buffer.CpuAddr() + copy.src_offset); + landing_end = (std::max)(landing_end, buffer.CpuAddr() + copy.src_offset + copy.size); + } if constexpr (requires { buffer.Mapped(); }) { if (const auto mapped = buffer.Mapped(); !mapped.empty()) { runtime.FinishForHostRead(); + const VideoCore::DownloadLanding landing{landing_begin, landing_end}; + const Common::RangeSet cpu_written = PendingCpuWriteRanges(); for (const BufferCopy& copy : copies) { WriteDownloadedMemory(cpu_written, buffer.CpuAddr() + copy.src_offset, mapped.data() + copy.src_offset, copy.size); @@ -3927,6 +3984,8 @@ void BufferCache

::DownloadBufferMemory(Buffer& buffer, DAddr device_addr, u64 } runtime.CopyBuffer(download_staging.buffer, buffer, copies_span, true); runtime.Finish(); + const VideoCore::DownloadLanding landing{landing_begin, landing_end}; + const Common::RangeSet cpu_written = PendingCpuWriteRanges(); for (const BufferCopy& copy : copies) { const DAddr copy_device_addr = buffer.CpuAddr() + copy.src_offset; // Undo the modified offset @@ -3939,6 +3998,9 @@ void BufferCache

::DownloadBufferMemory(Buffer& buffer, DAddr device_addr, u64 for (const BufferCopy& copy : copies) { buffer.ImmediateDownload(copy.src_offset, immediate_buffer.subspan(0, copy.size)); const DAddr copy_device_addr = buffer.CpuAddr() + copy.src_offset; + const VideoCore::DownloadLanding landing{copy_device_addr, + copy_device_addr + copy.size}; + const Common::RangeSet cpu_written = PendingCpuWriteRanges(); WriteDownloadedMemory(cpu_written, copy_device_addr, immediate_buffer.data(), copy.size); } } diff --git a/src/video_core/buffer_cache/buffer_cache_base.h b/src/video_core/buffer_cache/buffer_cache_base.h index 2b6b5eeb5c..d8e63aa732 100644 --- a/src/video_core/buffer_cache/buffer_cache_base.h +++ b/src/video_core/buffer_cache/buffer_cache_base.h @@ -267,7 +267,9 @@ class BufferCache : public VideoCommon::ChannelSetupCaches PendingCpuWriteRanges(); - /// Writes downloaded bytes to guest memory, except over `cpu_written`. + /// Writes downloaded bytes to guest memory, except over `cpu_written`. Callers take + /// `cpu_written` inside a VideoCore::DownloadLanding over the bytes and write in it: the CPU + /// writes collected after it store their bytes once the landing has ended. void WriteDownloadedMemory(const Common::RangeSet& cpu_written, DAddr device_addr, const u8* data, u64 size); diff --git a/src/video_core/rasterizer_download_area.h b/src/video_core/rasterizer_download_area.h index e817e32e16..b4693a34f3 100644 --- a/src/video_core/rasterizer_download_area.h +++ b/src/video_core/rasterizer_download_area.h @@ -8,6 +8,7 @@ #include #include #include +#include #include "common/common_types.h" #include "video_core/nx2.h" @@ -47,6 +48,76 @@ inline std::atomic g_read_downloads{}; Tegra::Nx2MayBeGpuWritten(addr, size)); } +/// What DownloadLanding publishes. +struct DownloadLandingState { + /// Odd while a landing is published. + std::atomic sequence{}; + std::atomic begin{}; + std::atomic end{}; + std::atomic waits{}; +}; + +/// The buffer cache writing downloaded GPU bytes to guest memory [begin, end). A download keeps the +/// bytes of the guest CPU writes collected and not yet applied when it looks at them +/// (BufferCache::PendingCpuWriteRanges). A guest core collects a write +/// (Core::GPUDirtyMemoryManager::Collect) and stores its bytes right after, without the buffer +/// cache's lock: a write collected after the download looked had its bytes written over with the +/// GPU's older ones. Downloads now look only inside a landing, and a write collected in the +/// landing's range stores its bytes once the landing has ended (Await, from Core::Memory's +/// HandleRasterizerWrite). +/// +/// Each side fences between its store (the landing, the collected write) and its load (the +/// collected writes, the landing): either the download finds the write, or the write finds the +/// landing. Downloads land under the buffer cache's lock, so one landing is published at a time. +class DownloadLanding { +public: + explicit DownloadLanding(DAddr begin, DAddr end) noexcept { + state.begin.store(begin, std::memory_order_relaxed); + state.end.store(end, std::memory_order_relaxed); + state.sequence.fetch_add(1, std::memory_order_seq_cst); + std::atomic_thread_fence(std::memory_order_seq_cst); + } + ~DownloadLanding() { + state.sequence.fetch_add(1, std::memory_order_release); + } + DownloadLanding(const DownloadLanding&) = delete; + DownloadLanding& operator=(const DownloadLanding&) = delete; + + /// A guest CPU write of [addr, addr + size) was collected: returns once its bytes may be + /// stored. + static void Await(DAddr addr, u64 size) noexcept { + std::atomic_thread_fence(std::memory_order_seq_cst); + const u64 sequence = state.sequence.load(std::memory_order_acquire); + if ((sequence & 1) != 0) [[unlikely]] { + Wait(sequence, addr, size); + } + } + + /// Writes that waited for a landing so far. + [[nodiscard]] static u64 Waits() noexcept { + return state.waits.load(std::memory_order_relaxed); + } + +private: + static void Wait(u64 sequence, DAddr addr, u64 size) noexcept { + const DAddr begin = state.begin.load(std::memory_order_relaxed); + const DAddr end = state.end.load(std::memory_order_relaxed); + std::atomic_thread_fence(std::memory_order_acquire); + // Ended (its writes come first, and the landings after it found this write), or it + // writes elsewhere. + if (state.sequence.load(std::memory_order_acquire) != sequence || addr >= end || + addr + size <= begin) { + return; + } + state.waits.fetch_add(1, std::memory_order_relaxed); + while (state.sequence.load(std::memory_order_acquire) == sequence) { + std::this_thread::yield(); + } + } + + static inline DownloadLandingState state; +}; + /// TEST (EDEN_NX2_READ_CHECK): the latest writes from the GPU's side that ReadDownloadsMayCover /// let through, to name the writer when the check finds newer bytes where an earlier read's /// download landed. From cf3c7f00998d20dc26d9ab777e54e1e8e1d1d3a2 Mon Sep 17 00:00:00 2001 From: jpkto <255363096+jpkto@users.noreply.github.com> Date: Sun, 8 Nov 2026 06:07:53 +0100 Subject: [PATCH 054/105] TEST: count guest writes held by download landings, explain check differences With EDEN_NX2_READ_CHECK, the check's line every 10 s now ends with how many guest CPU writes waited for a download landing under them, and for how long in total, then what the checked downloads were (totals since the start, as the rest of the line): ...; N guest CPU writes waited for a download landing under them, T ms; recorded: R read downloads, A check copies alone, E early downloads (L left out at their bounds); F reads an early download served flushed for the check Before the previous commit, each of those N writes raced a landing. The second part splits the downloads the check adds. Each read an early download serves flushes in check mode (F), as a read download (R) or a check copy alone (A), so the verify run lands more downloads than a normal run. test49b's verify run landed ~2,000 a second against ~1,430 in test49, while its normal run flushed as often as test49's. The split shows which kind grew. If the check still finds bytes that differ from its copy, each logged difference gets a second line: Read download check: A+S is in a read's|an early download of tick K (now C), buffer B+Z at offset O: the download brought X, as the check copy: guest memory changed after it landed | as guest memory holds: the GPU wrote between the download's copy and the check's | neither; a guest CPU write is collected there now | no guest CPU write is collected there; newest GPU-side guest memory write noted there: W The download's copy and the check's are recorded back to back. If the download brought the check's byte, guest memory changed after the landing: a writer that neither takes the buffer cache's lock nor waits for landings. If it brought guest memory's byte, the GPU wrote between the two copies. That second case is an ordering problem on the host GPU, which the upload batching would be the first suspect for. Tested in the read download harness: a write stored without the landing protocol prints the first kind, a download staging changed after its copy prints the second, and all tests pass in 11 runs, 5 pinned to one CPU. Co-Authored-By: Claude Opus 5.5 --- src/video_core/buffer_cache/buffer_cache.h | 68 +++++++++++++++++-- .../buffer_cache/buffer_cache_base.h | 13 +++- src/video_core/rasterizer_download_area.h | 13 ++++ 3 files changed, 88 insertions(+), 6 deletions(-) diff --git a/src/video_core/buffer_cache/buffer_cache.h b/src/video_core/buffer_cache/buffer_cache.h index 5e4163a6f1..8b471c3339 100644 --- a/src/video_core/buffer_cache/buffer_cache.h +++ b/src/video_core/buffer_cache/buffer_cache.h @@ -347,6 +347,7 @@ std::optional BufferCache

::GetFlushArea(DA } if (check_early) [[unlikely]] { area->preemtive = false; + ++read_check.early_served; } // A preflushable mark persists and downloads later GPU writes at every fence. Native titles // instead keep read traps and flush on demand. @@ -762,6 +763,8 @@ void BufferCache

::RecordReadDownload(std::span> r for (const LandedCopy& landed : landed_copies) { std::ranges::copy(landed.as_landed, check->landed_bytes.data() + landed.offset); } + ++(early ? read_check.early_recorded + : (check_only ? read_check.alone_recorded : read_check.reads_recorded)); download.check = std::move(check); } // A read waits for its copies; an early download's go with the next submission. @@ -976,6 +979,7 @@ void BufferCache

::RecordEarlyDownloads() { // Bounded: past it, the reads of these pages flush on demand. if (pending_early_downloads >= MaxPendingEarlyDownloads || early_bytes_this_frame >= EarlyBytesPerFrame) { + ++read_check.early_left_out; // TEST return; } RecordReadDownload(pages, pages, true); @@ -1195,7 +1199,7 @@ void BufferCache

::WriteReadDownloads(u64 tick) { } } if (download.check) [[unlikely]] { - CheckReadDownload(*download.check, cpu_written); + CheckReadDownload(download, cpu_written); async_buffers_death_ring.push_back(download.check->staging); } download.clean_ranges.ForEach([&](DAddr start, DAddr end) { @@ -1250,8 +1254,9 @@ bool BufferCache

::ReadDownloadsCover(DAddr device_addr, u64 size) const { } template -void BufferCache

::CheckReadDownload(const ReadCheck& check, +void BufferCache

::CheckReadDownload(const ReadDownload& download, const Common::RangeSet& cpu_written) { + const ReadCheck& check = *download.check; // Guest memory now holds this download and the earlier ones: compare it with the synchronous // copy, except where a newer value replaced it since (CPU writes, ClearDownload). Writes // collected since the download was written count as newer too. Where an earlier download @@ -1272,7 +1277,8 @@ void BufferCache

::CheckReadDownload(const ReadCheck& check, Outdated, ///< An earlier download's bytes differ: the GPU wrote them since NotLanded, ///< No earlier download wrote it }; - const auto report = [this](Result result, DAddr address, u64 size, u8 found, u8 wanted) { + const auto report = [this, &download](Result result, DAddr address, u64 size, u8 found, + u8 wanted) { if (result == Result::Rewritten) { read_check.rewritten += size; if (read_check.logged_rewrites++ < 16) { @@ -1297,6 +1303,9 @@ void BufferCache

::CheckReadDownload(const ReadCheck& check, "Read download check: guest memory {:#x}+{:#x} differs from a synchronous " "flush ({:#04x} for {:#04x} at its start){}", address, size, found, wanted, why); + if (result == Result::Differs) { + ExplainReadCheckDifference(download, address, size, found, wanted); + } }; std::vector guest; std::vector sources; @@ -1371,9 +1380,58 @@ void BufferCache

::CheckReadDownload(const ReadCheck& check, "Read download check (EDEN_NX2_READ_CHECK): {} downloads compared with a " "synchronous flush, {} KiB; {} bytes differ, {} GPU-modified bytes no download " "brought (both must stay 0); {} bytes got newer values after an earlier read's " - "download brought them (not compared)", + "download brought them (not compared); {} guest CPU writes waited for a " + "download landing under them, {:.3f} ms; recorded: {} read downloads, {} check " + "copies alone, {} early downloads ({} left out at their bounds); {} reads an " + "early download served flushed for the check", read_check.checked, read_check.compared / 1024, read_check.mismatched, - read_check.uncovered, read_check.rewritten); + read_check.uncovered, read_check.rewritten, VideoCore::DownloadLanding::Waits(), + static_cast(VideoCore::DownloadLanding::WaitNs()) / 1e6, + read_check.reads_recorded, read_check.alone_recorded, read_check.early_recorded, + read_check.early_left_out, read_check.early_served); + } +} + +template +void BufferCache

::ExplainReadCheckDifference(const ReadDownload& download, DAddr address, + u64 size, u8 found, u8 wanted) { + if constexpr (!IS_OPENGL) { + // What the download's own copy holds there: the check's copy was recorded right after it, + // so they differ only if the GPU wrote between the two. Equal, guest memory changed after + // the download wrote it. + std::optional brought; + for (const BufferCopy& copy : download.copies) { + if (address >= copy.src_offset && address < copy.src_offset + copy.size) { + brought = download.staging.mapped_span[copy.dst_offset - download.staging.offset + + (address - copy.src_offset)]; + } + } + std::string source = "the download has no copy there"; + if (brought) { + source = fmt::format( + "the download brought {:#04x}{}", *brought, + *brought == wanted ? ", as the check copy: guest memory changed after it landed" + : *brought == found + ? ", as guest memory holds: the GPU wrote between the download's copy and the " + "check's" + : ", neither"); + } + DAddr buffer_addr = 0; + u64 buffer_size = 0; + ForEachBufferInRange(address, 1, [&](BufferId, Buffer& buffer) { + buffer_addr = buffer.CpuAddr(); + buffer_size = buffer.SizeBytes(); + }); + const bool collected = PendingCpuWriteRanges().Intersects(address, size); + LOG_ERROR(HW_GPU, + "Read download check: {:#x}+{:#x} is in {} download of tick {} (now {}), buffer " + "{:#x}+{:#x} at offset {:#x}: {}; {}; newest GPU-side guest memory write noted " + "there: {}", + address, size, download.early ? "an early" : "a read's", download.tick, + runtime.CurrentTick(), buffer_addr, buffer_size, address - buffer_addr, source, + collected ? "a guest CPU write is collected there now" + : "no guest CPU write is collected there", + VideoCore::ReadCheckWrites::Find(address, size)); } } diff --git a/src/video_core/buffer_cache/buffer_cache_base.h b/src/video_core/buffer_cache/buffer_cache_base.h index d8e63aa732..4bc3bf6c1a 100644 --- a/src/video_core/buffer_cache/buffer_cache_base.h +++ b/src/video_core/buffer_cache/buffer_cache_base.h @@ -857,6 +857,11 @@ class BufferCache : public VideoCommon::ChannelSetupCaches& cpu_written); [[nodiscard]] bool ReadChecks() const { return read_check.every != 0 || ReadCheckEvery() != 0; } @@ -895,6 +899,13 @@ class BufferCache : public VideoCommon::ChannelSetupCaches& cpu_written); + /// TEST (EDEN_NX2_READ_CHECK): where a difference the check found comes from. + void ExplainReadCheckDifference(const ReadDownload& download, DAddr address, u64 size, u8 found, + u8 wanted); /// Oldest first, and written to guest memory in that order: none lands over newer bytes. std::deque read_downloads; /// All the read downloads' clean_ranges (they don't overlap): GPU-modified bytes already on diff --git a/src/video_core/rasterizer_download_area.h b/src/video_core/rasterizer_download_area.h index b4693a34f3..387daa0cf6 100644 --- a/src/video_core/rasterizer_download_area.h +++ b/src/video_core/rasterizer_download_area.h @@ -6,6 +6,7 @@ #include #include #include +#include #include #include #include @@ -55,6 +56,7 @@ struct DownloadLandingState { std::atomic begin{}; std::atomic end{}; std::atomic waits{}; + std::atomic wait_ns{}; ///< TEST (EDEN_NX2_READ_CHECK) }; /// The buffer cache writing downloaded GPU bytes to guest memory [begin, end). A download keeps the @@ -98,6 +100,11 @@ class DownloadLanding { return state.waits.load(std::memory_order_relaxed); } + /// TEST (EDEN_NX2_READ_CHECK): how long they waited, in total. + [[nodiscard]] static u64 WaitNs() noexcept { + return state.wait_ns.load(std::memory_order_relaxed); + } + private: static void Wait(u64 sequence, DAddr addr, u64 size) noexcept { const DAddr begin = state.begin.load(std::memory_order_relaxed); @@ -110,9 +117,15 @@ class DownloadLanding { return; } state.waits.fetch_add(1, std::memory_order_relaxed); + const auto start = std::chrono::steady_clock::now(); while (state.sequence.load(std::memory_order_acquire) == sequence) { std::this_thread::yield(); } + state.wait_ns.fetch_add( + static_cast(std::chrono::duration_cast( + std::chrono::steady_clock::now() - start) + .count()), + std::memory_order_relaxed); } static inline DownloadLandingState state; From 758b5aae3a08cd2a00242f56d970826326b99f4a Mon Sep 17 00:00:00 2001 From: jpkto <255363096+jpkto@users.noreply.github.com> Date: Sun, 8 Nov 2026 06:45:58 +0100 Subject: [PATCH 055/105] video_core: release threads waiting for sync operations once the GPU threads stop The test49g verify run hung at shutdown. ForceStop came at 117.684 and the emulation thread was still running 10 s later, when the frontend terminated it. In the other shutdowns, ~TimeWorker logs SetUserSystemClockContext within a few milliseconds of ForceStop. It runs during service teardown, after KernelCore::SuspendEmulation. That line never came, so System::ShutdownMainProcess stopped at or before the kernel's suspension. A guest thread that reads GPU-written memory (GPU::OnCPURead) queues a sync operation, asks the GPU threads to run it, and waits in WaitForSyncOperation. ShutdownMainProcess first stops the GPU threads (NotifyShutdown) and only then the guest threads: - The split's back leaves its loop when asked to stop, without running a tick asked for after its last one. - Once it has stopped, RequestTick does nothing. TickGPU queues a command for the front thread, which has already been joined. - NotifyShutdown woke sync_cv, but not sync_request_cv, and WaitForSyncOperation had no other way out. So a read made around the stop never returned. KernelCore::SuspendEmulation waits until no core runs a thread of the process, and that thread stays current on its core in host code. The wait never ended. In the verify runs, reads waited 38 to 47% of the time (750 to 1,000 reads a second), and they went on until ForceStop. The race is older than the split: the single GPU thread ran these requests before and stopped the same way. Nothing in the upload reordering waits. What changed: once the GPU threads have stopped, NotifyShutdown drops the queued requests and releases their waiters. The requests are dropped because they can refer to their waiters' frames (OnCPURead's timing), and nothing runs them after that point. WaitForSyncOperation returns when its request ran or when the requests were dropped. A read released this way goes on without its flush, which no longer matters. The release comes after the threads stop, so no request runs once its waiter has left. Tested with a harness on the real gpu.cpp and gpu_split.cpp (agents/upload-reorder/hshutdown): a Tegra::GPU, the split's back thread with GPU::TickWork as its tick hook (as GPU::Impl::Start sets it), and a guest thread reading as OnCPURead does. The back served 15,000 to 32,000 reads, then NotifyShutdown ran and the guest read again. Without this change the guest was still waiting 3 s later: 5 of 5 runs on this branch, and 5 of 5 on the oot-amd-test49g sources. With it, the reads return and the request queued before the stop never runs: 30 of 30 runs here, and 30 of 30 with this change applied to the oot-amd-test49g sources (it applies with line offsets only). Also a syntax check and strict warnings on the changed lines. Co-Authored-By: Claude Opus 5.5 --- src/video_core/gpu.cpp | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/src/video_core/gpu.cpp b/src/video_core/gpu.cpp index 8efd8bc750..66881a1fa1 100644 --- a/src/video_core/gpu.cpp +++ b/src/video_core/gpu.cpp @@ -164,9 +164,13 @@ struct GPU::Impl { return current_sync_fence.load(std::memory_order_relaxed); } + /// Returns once the request of `fence` ran, or once the GPU threads stopped for good + /// (NotifyShutdown): then it never runs. void WaitForSyncOperation(const u64 fence) { std::unique_lock lck{sync_request_mutex}; - sync_request_cv.wait(lck, [this, fence] { return CurrentSyncRequestFence() >= fence; }); + sync_request_cv.wait(lck, [this, fence] { + return CurrentSyncRequestFence() >= fence || sync_requests_dropped; + }); } void WaitForIdle() { @@ -227,6 +231,17 @@ struct GPU::Impl { void NotifyShutdown() { gpu_thread.NotifyShutdown(); VideoCommon::GpuSplit::Get().SetTickHook(nullptr, nullptr); + { + // The GPU threads stopped: no sync operation runs anymore. Guest threads still run + // until the kernel suspends them, and one waiting for a read's flush (OnCPURead) + // would wait forever, in host code where the kernel cannot suspend it + // (KernelCore::SuspendEmulation then waits for it forever). Drop the requests, which + // may refer to their waiters' frames, and release the waiters. + std::scoped_lock requests_lock{sync_request_mutex}; + sync_requests.clear(); + sync_requests_dropped = true; + } + sync_request_cv.notify_all(); std::unique_lock lk{sync_mutex}; shutting_down.store(true, std::memory_order::relaxed); sync_cv.notify_all(); @@ -475,6 +490,8 @@ struct GPU::Impl { VideoCommon::PendingMaps pending_maps; std::mutex sync_request_mutex; std::condition_variable sync_request_cv; + /// The GPU threads stopped and the requests were dropped (NotifyShutdown, under the mutex). + bool sync_requests_dropped{}; /// InvalidateGPUCache: the writes it gathered and is applying, on each thread doing so. std::vector>*> applying_writes; From 15efb1116cb629631c2c324b160d70e4eb46cd2b Mon Sep 17 00:00:00 2001 From: jpkto <255363096+jpkto@users.noreply.github.com> Date: Sun, 8 Nov 2026 06:38:57 +0100 Subject: [PATCH 056/105] video_core: keep a QMD entry from the split's back until the front noted it test49d's verify run logged one wrong note from the QMD view in 120 s: QMD view check: the back released 0x86000=0x398b | noted 0x86000=0x398c The front values a QMD launch right after it sends the PCAS or inline QMD entry. For a reduction or a maximum it needs the word the back finds when it runs that entry. When no release still queued writes the word, it reads the word from memory. But Commit may have published the entry already: to a back that waits for work it publishes after 8 words. That back can run the entry, and its increment, before the front reads the word, and the front then applies the increment a second time. The back is idle right after a takeover, which is when the view values; a front thread preempted between the publish and the read is enough. A maximum gives the same value twice, so only reductions show it. The compute report's reduction (0x6c3) reads its base from memory the same way. GpuSplit::PublishHold now keeps what the front sends unpublished while it notes what that entry writes. Commit publishes nothing during the hold and publishes what is due when it ends. Publish itself, for drains, waits and a full ring, still goes out. KeplerCompute's front holds its QMD methods (PCAS and LOAD_INLINE_QMD_DATA's last word) and 0x6c3, sent directly, as a run or by a macro, from the send to the end of FrontNoteWrites. An idle back waits the few microseconds the front takes to value the entry. Tested: - A new QMD harness mode lets the back run whatever is published at once, also right before each memory read the view makes. Without the hold, a scripted case logs "the back released 0x108014=0x398b | noted 0x108014=0x398c", the shape of the test49d line, for an increment, an add of 1 and an increment launched by a decrement, and 20 fuzz runs fail 1,360 checks. With the hold, the scripted case and 200 fuzz runs pass: 524,028 check lists, nothing released or read otherwise, 995 tables equal. The four older modes pass as before. - Split harness with the real back thread: an idle back gets nothing while the front holds, then what Commit would have published; a drain inside a hold still publishes. 68 checks; 20 runs on 2 CPUs and 10 pinned to one passed. The older split harness passed 20 and 10 too. - The compile check passes on every changed file and on the users of gpu_split.h. clang's strict warnings report nothing on the changed lines. Co-Authored-By: Claude Opus 5.5 --- src/video_core/engines/kepler_compute.cpp | 40 ++++++++++++----- src/video_core/front_qmd.h | 13 ++++-- src/video_core/gpu_split.cpp | 1 + src/video_core/gpu_split.h | 54 ++++++++++++++++++++--- 4 files changed, 85 insertions(+), 23 deletions(-) diff --git a/src/video_core/engines/kepler_compute.cpp b/src/video_core/engines/kepler_compute.cpp index 17f2ac4b32..67a368c428 100644 --- a/src/video_core/engines/kepler_compute.cpp +++ b/src/video_core/engines/kepler_compute.cpp @@ -1481,6 +1481,13 @@ struct FrontQmdMemory { return method == 0xae || method == 0xaf || method == 0xb0 || method == 0x107; } +// Methods whose notes the front takes from guest memory as the back finds it running them: the +// QMD methods (descriptors, and the words maxima and reductions combine with) and the compute +// report (reductions). The front holds them unpublished until it noted them. +[[nodiscard]] bool ValuedFromMemory(u32 method) { + return IsQmdMethod(method) || method == 0x6c3; +} + // Whether a PCAS method signals the QMD scheduler, as ProcessNativeMethod filters them. [[nodiscard]] bool SignalsQmd(u32 method, u32 argument) { if (method == 0xaf) { @@ -1630,8 +1637,11 @@ void KeplerCompute::FrontCallMethod(Core::System& system, VideoCommon::GpuSplit& if (IsQmdMethod(method)) { GetFrontQmd().MaybeTakeOver(split, native_qmd, QmdTakeOver::QmdMethod); } - split.Method(this, method, argument, is_last_call, segment, FrontWrites(method)); - FrontNoteWrites(split, fs, method, argument, false); + { + const VideoCommon::GpuSplit::PublishHold hold{split, ValuedFromMemory(method)}; + split.Method(this, method, argument, is_last_call, segment, FrontWrites(method)); + FrontNoteWrites(split, fs, method, argument, false); + } FrontQmdCheck(split); } @@ -1794,14 +1804,17 @@ void KeplerCompute::FrontCallMultiMethod(Core::System& system, VideoCommon::GpuS if (IsQmdMethod(method)) { GetFrontQmd().MaybeTakeOver(split, native_qmd, QmdTakeOver::QmdMethod); } - split.MultiMethod(this, method, base_start, amount, methods_pending, segment, - FrontWrites(method)); - // The back runs them one by one; what they write is the back's once it ran them all. Each - // upload data word may write the upload's range: once covers them. - const u32 notes = - method == KEPLER_COMPUTE_REG_INDEX(data_upload) ? (std::min)(amount, 1U) : amount; - for (u32 i = 0; i < notes; ++i) { - FrontNoteWrites(split, fs, method, base_start[i], false); + { + const VideoCommon::GpuSplit::PublishHold hold{split, ValuedFromMemory(method)}; + split.MultiMethod(this, method, base_start, amount, methods_pending, segment, + FrontWrites(method)); + // The back runs them one by one; what they write is the back's once it ran them all. + // Each upload data word may write the upload's range: once covers them. + const u32 notes = + method == KEPLER_COMPUTE_REG_INDEX(data_upload) ? (std::min)(amount, 1U) : amount; + for (u32 i = 0; i < notes; ++i) { + FrontNoteWrites(split, fs, method, base_start[i], false); + } } FrontQmdCheck(split); } @@ -1827,8 +1840,11 @@ void KeplerCompute::FrontRunMacro(Core::System& system, VideoCommon::GpuSplit& s if (IsQmdMethod(address)) { GetFrontQmd().MaybeTakeOver(split, native_qmd, QmdTakeOver::QmdMethod); } - split.MacroWrite(this, address, value, FrontWrites(address)); - FrontNoteWrites(split, fs, address, value, true); + { + const VideoCommon::GpuSplit::PublishHold hold{split, ValuedFromMemory(address)}; + split.MacroWrite(this, address, value, FrontWrites(address)); + FrontNoteWrites(split, fs, address, value, true); + } FrontQmdCheck(split); return true; }; diff --git a/src/video_core/front_qmd.h b/src/video_core/front_qmd.h index af7c471d86..88b94a49db 100644 --- a/src/video_core/front_qmd.h +++ b/src/video_core/front_qmd.h @@ -31,8 +31,10 @@ namespace VideoCommon { /// (Nx2MayBeGpuWritten); /// - from the inline QMD a LOAD_INLINE_QMD_DATA still queued writes there, when that is the last /// write sent there (NoteInlineWrite). -/// No change of the GPU's mappings may be queued either (GpuSplit::LastMappingChange). Where the -/// front can't tell what the back reads, its table may no longer be the back's: it stops valuing, +/// No change of the GPU's mappings may be queued either (GpuSplit::LastMappingChange). Memory holds +/// what the back finds there only while the back has not run the signal's entry yet: the front +/// sends and values it under a GpuSplit::PublishHold. Where the front can't tell what the back +/// reads, its table may no longer be the back's: it stops valuing, /// the launches count as writing guest memory anywhere, and it takes the back's table over again /// the next time the back has run everything sent, between commands or at a QMD method /// (MaybeTakeOver). @@ -123,7 +125,8 @@ class FrontQmdView { /// The scheduling of PCAS action `argument` on the QMD at `address` that the back runs at the /// entry just sent (KeplerCompute::ProcessNativeMethod); `inline_qmd`: the QMD it writes there - /// right before. Notes what the launches it runs release. + /// right before. Notes what the launches it runs release. The entry must still be unpublished + /// (GpuSplit::PublishHold): the back must not have run it. template void Signal(GpuSplit& split, const Memory& memory, GPUVAddr address, u32 argument, const Descriptor* inline_qmd) { @@ -319,7 +322,9 @@ class FrontQmdView { /// The word the back finds at `address` (`device`) when it runs the entry just sent, where /// the front knows it: the last release sent there, if nothing sent since may have written - /// over it, else memory, once the back ran every write sent there. + /// over it, else memory, once the back ran every write sent there before the entry. The entry + /// itself is held unpublished (PublishHold): an idle back that ran it before this read would + /// leave its own increment in memory, and the front would count it twice. template [[nodiscard]] std::optional WordAt(const GpuSplit& split, const Memory& memory, GPUVAddr address, u64 device) const { diff --git a/src/video_core/gpu_split.cpp b/src/video_core/gpu_split.cpp index f35c5db263..2da4fb6764 100644 --- a/src/video_core/gpu_split.cpp +++ b/src/video_core/gpu_split.cpp @@ -154,6 +154,7 @@ void GpuSplit::Stop() { write_positions.fill(0); coarse_positions.fill(0); published_local = 0; + publish_held = false; consumed_cache = 0; publish_every = PublishEvery; front_engine = nullptr; diff --git a/src/video_core/gpu_split.h b/src/video_core/gpu_split.h index 92a7cefff9..7ddd429197 100644 --- a/src/video_core/gpu_split.h +++ b/src/video_core/gpu_split.h @@ -248,6 +248,31 @@ class GpuSplit { void PublishSoon() { publish_every = PublishAfterWait; } + /// Keeps the entries the front sends unpublished while it notes what they write from guest + /// memory as the back will find it running them (the QMD view: descriptors, and the words + /// maxima and reductions combine with). An idle back gets a published entry at once and may + /// run it before the front reads, which then reads what the entry wrote. While `hold`, Commit + /// publishes nothing; Publish (drains, waits, a full ring) still does. + class PublishHold { + public: + PublishHold(GpuSplit& split_, bool hold) : split{split_}, was_held{split_.publish_held} { + if (hold) { + split.publish_held = true; + } + } + ~PublishHold() { + if (split.publish_held != was_held) { + split.publish_held = was_held; + split.PublishIfDue(); + } + } + PublishHold(const PublishHold&) = delete; + PublishHold& operator=(const PublishHold&) = delete; + + private: + GpuSplit& split; + bool was_held; + }; /// Waits until the back executed everything sent so far. void Drain(DrainSite site = DrainSite::Other); @@ -457,16 +482,29 @@ class GpuSplit { NoteFirstUnpublished(); // TEST } write_position += words; + if (PublishDue() && !publish_held) { + PublishNow(); + } + } + /// Whether Commit publishes now. + [[nodiscard]] bool PublishDue() const { const u64 unpublished = write_position - published_local; // The back waits for work at the last position published (it stored it in back_idle_at // before it spun): it gets these words after a few, not after publish_every. - if (unpublished >= publish_every || - (unpublished >= PublishAfterWait && - back_idle_at.load(std::memory_order_relaxed) == published_local)) { - if (profile_publishes && unpublished < publish_every) [[unlikely]] { - NoteEarlyPublish(); // TEST - } - Publish(); + return unpublished >= publish_every || + (unpublished >= PublishAfterWait && + back_idle_at.load(std::memory_order_relaxed) == published_local); + } + void PublishNow() { + if (profile_publishes && write_position - published_local < publish_every) [[unlikely]] { + NoteEarlyPublish(); // TEST + } + Publish(); + } + /// What Commit would have published during a PublishHold. + void PublishIfDue() { + if (PublishDue()) { + PublishNow(); } } /// TEST (EDEN_NX2_GPU_PROFILE): how long the back waited for work while the front held words @@ -541,6 +579,8 @@ class GpuSplit { std::array write_positions{}; std::array coarse_positions{}; u64 published_local{}; + /// PublishHold: Commit publishes nothing. + bool publish_held{}; u64 consumed_cache{}; u64 first_unpublished_ticks{}; ///< TEST: when the first word not published yet was sent /// Words between publishes: small right after the front waited for the back (Drain, From 3cee6b61a0ae5936ee5625b48e6a4c6f3c166b43 Mon Sep 17 00:00:00 2001 From: jpkto <255363096+jpkto@users.noreply.github.com> Date: Sun, 8 Nov 2026 06:54:31 +0100 Subject: [PATCH 057/105] video_core: keep the split's placed writes apart where their granules share a slot In test49d's verify run the QMD view lost its table about 55 times a second, each time to "a write into the QMD queued". Once lost, it waits for an idle back to take the table over: later in the run it valued about 590 signals a second and left 18,500 a second unvalued. The view reads a descriptor from memory only once the back ran the last write placed in the QMD's 256-byte granule (MemoryWritesTarget). NoteWrite kept that position in 16,384 slots hashed by granule, and the granules hashed to one slot shared its position. Every queued draw places its constant buffer words. When one shared a QMD's slot, the QMD read as written by that draw, and the view lost its table although nothing had written the QMD. The slots now form 4,096 sets of four. A slot holds a granule (its low 32 bits) and the position of its last write. A write to a granule the set lacks takes the slot written longest ago, and the set keeps the latest position of the writes it pushed out that way. A read waits for its granule's own last write while the set holds it, else for the ones pushed out. A write elsewhere can still delay a read but never hide a write from it. The draws' words delay a QMD read only once four other granules pushed the QMD's out within the back's lag. Coarse writes (16 KiB to 16 MiB, by 64 KiB granule) get 1,024 such sets. Macro DMA reads (WaitForWrites) wait on the same positions, so they wait less too. Each set is one cache line: 320 KiB in all, against 160 KiB before. Tested: - QMD harness: a new fuzz write lands in unmapped memory whose granule shares a set with a QMD's. With the back always behind and only writes the view can place, it must value every QMD entry. Before this change it valued 29,018 of 899,640 in 200 runs; now it values all of them. All six modes pass with and without EDEN_NX2_DRAIN_CHECK. - Split harness: six granules hashed to one set, written and read in turn. A read finds its granule's own last write while the set holds it, else the latest write pushed out. 80 checks; 20 runs on 2 CPUs and 10 pinned to one passed, and the older split harness too. - The compile check passes on gpu_split.cpp, its users and the unit tests. clang's strict warnings report nothing on the changed lines. Co-Authored-By: Claude Opus 5.5 --- src/video_core/gpu_split.cpp | 41 ++++++++++++++++++++++++++++++------ src/video_core/gpu_split.h | 38 ++++++++++++++++++++++++--------- 2 files changed, 63 insertions(+), 16 deletions(-) diff --git a/src/video_core/gpu_split.cpp b/src/video_core/gpu_split.cpp index 2da4fb6764..c3e1dbdf6b 100644 --- a/src/video_core/gpu_split.cpp +++ b/src/video_core/gpu_split.cpp @@ -151,8 +151,8 @@ void GpuSplit::Stop() { last_unplaced_kind = UnplacedKind::Other; last_mapping_kind = UnplacedKind::Other; last_noted_write = 0; - write_positions.fill(0); - coarse_positions.fill(0); + write_sets.fill({}); + coarse_sets.fill({}); published_local = 0; publish_held = false; consumed_cache = 0; @@ -337,10 +337,39 @@ void GpuSplit::NoteWrite(const Tegra::MemoryManager& memory, GPUVAddr address, u [this](DAddr device, u64 bytes) { NoteDeviceWrite(device, bytes); }); } +void GpuSplit::NoteSetWrite(WriteSet& set, u64 granule, u64 position) { + const u32 tag = static_cast(granule); + size_t oldest = 0; + for (size_t way = 0; way < WriteWays; ++way) { + if (set.granules[way] == tag) { + set.positions[way] = std::max(set.positions[way], position); + return; + } + if (set.positions[way] < set.positions[oldest]) { + oldest = way; + } + } + // The granule's earlier writes, if any, are among the evicted ones. + const u64 granule_before = set.evicted; + set.evicted = std::max(set.evicted, set.positions[oldest]); + set.granules[oldest] = tag; + set.positions[oldest] = std::max(position, granule_before); +} + +u64 GpuSplit::SetTarget(const WriteSet& set, u64 granule) { + const u32 tag = static_cast(granule); + for (size_t way = 0; way < WriteWays; ++way) { + if (set.granules[way] == tag) { + return set.positions[way]; + } + } + return set.evicted; +} + void GpuSplit::NoteDeviceWrite(DAddr address, u64 size) { const u64 last = (address + size - 1) >> WriteGranuleBits; for (u64 granule = address >> WriteGranuleBits; granule <= last; ++granule) { - write_positions[WriteSlot(granule)] = write_position; + NoteSetWrite(write_sets[WriteSlot(granule)], granule, write_position); } last_noted_write = write_position; } @@ -348,7 +377,7 @@ void GpuSplit::NoteDeviceWrite(DAddr address, u64 size) { void GpuSplit::NoteCoarseDeviceWrite(DAddr address, u64 size) { const u64 last = (address + size - 1) >> CoarseGranuleBits; for (u64 granule = address >> CoarseGranuleBits; granule <= last; ++granule) { - coarse_positions[CoarseSlot(granule)] = write_position; + NoteSetWrite(coarse_sets[CoarseSlot(granule)], granule, write_position); } last_noted_write = write_position; } @@ -381,11 +410,11 @@ u64 GpuSplit::DeviceWritesTarget(DAddr address, u64 size) const { u64 target = 0; const u64 last = (address + size - 1) >> WriteGranuleBits; for (u64 granule = address >> WriteGranuleBits; granule <= last; ++granule) { - target = std::max(target, write_positions[WriteSlot(granule)]); + target = std::max(target, SetTarget(write_sets[WriteSlot(granule)], granule)); } const u64 last_coarse = (address + size - 1) >> CoarseGranuleBits; for (u64 granule = address >> CoarseGranuleBits; granule <= last_coarse; ++granule) { - target = std::max(target, coarse_positions[CoarseSlot(granule)]); + target = std::max(target, SetTarget(coarse_sets[CoarseSlot(granule)], granule)); } return target; } diff --git a/src/video_core/gpu_split.h b/src/video_core/gpu_split.h index 7ddd429197..f888d6188d 100644 --- a/src/video_core/gpu_split.h +++ b/src/video_core/gpu_split.h @@ -430,24 +430,42 @@ class GpuSplit { static constexpr size_t PublishAfterWait = 8; /// NoteWrite places writes by 256-byte granule of device memory, up to MaxNotedBytes long, - /// in write_positions slots hashed by granule: granules that share a slot share its position, - /// so a read may wait for a write elsewhere, never miss one. 16384 slots: the constant buffer - /// words of a few hundred queued draws share few with a QMD. Writes up to MaxCoarseNotedBytes - /// (copies, uploads) go by 64 KiB granule into coarse_positions, longer ones count as - /// unplaced. + /// in sets of WriteWays slots hashed by granule. Writes up to MaxCoarseNotedBytes (copies, + /// uploads) go by 64 KiB granule into coarse sets, longer ones count as unplaced. A slot holds + /// a granule and the position of its last write; a write to a granule its set lacks takes the + /// slot written longest ago, and the set keeps the latest position of the writes it so + /// dropped (`evicted`). A read waits for its granule's own last write while the set holds + /// it, else for the evicted ones: it may wait for a write elsewhere, never miss one, and the + /// writes to other granules hashed there (the constant buffer words of queued draws) hold up + /// a read of a QMD only once they pushed it out of its set. static constexpr u32 WriteGranuleBits = 8; static constexpr u64 MaxNotedBytes = 0x4000; - static constexpr u32 WriteSlotBits = 14; + static constexpr u32 WriteSlotBits = 12; ///< Sets static constexpr u32 CoarseGranuleBits = 16; static constexpr u64 MaxCoarseNotedBytes = u64{16} << 20; - static constexpr u32 CoarseSlotBits = 12; - + static constexpr u32 CoarseSlotBits = 10; + static constexpr size_t WriteWays = 4; + + /// One cache line. + struct WriteSet { + std::array positions; + /// The granules' low 32 bits: granules that agree in them share a slot. + std::array granules; + u64 evicted; + u64 unused; + }; + static_assert(sizeof(WriteSet) == 64); + /// The set a granule is hashed to. static size_t WriteSlot(u64 granule) { return static_cast((granule * 0x9E3779B97F4A7C15ULL) >> (64 - WriteSlotBits)); } static size_t CoarseSlot(u64 granule) { return static_cast((granule * 0x9E3779B97F4A7C15ULL) >> (64 - CoarseSlotBits)); } + /// A write to `granule` at `position` (the newest, or an earlier one placed late). + static void NoteSetWrite(WriteSet& set, u64 granule, u64 position); + /// The latest position of the writes to `granule` in its set, at most. + [[nodiscard]] static u64 SetTarget(const WriteSet& set, u64 granule); void NoteDeviceWrite(DAddr address, u64 size); void NoteCoarseDeviceWrite(DAddr address, u64 size); /// The position WaitForWrites waits for. @@ -576,8 +594,8 @@ class GpuSplit { UnplacedKind last_unplaced_kind{}; ///< TEST: see TagUnplaced UnplacedKind last_mapping_kind{}; ///< TEST u64 last_noted_write{}; - std::array write_positions{}; - std::array coarse_positions{}; + alignas(64) std::array write_sets{}; + alignas(64) std::array coarse_sets{}; u64 published_local{}; /// PublishHold: Commit publishes nothing. bool publish_held{}; From 3c7cfea4d10d5bd802175a89497b35df77e4903f Mon Sep 17 00:00:00 2001 From: jpkto <255363096+jpkto@users.noreply.github.com> Date: Sun, 8 Nov 2026 07:25:29 +0100 Subject: [PATCH 058/105] video_core: have a QMD signal the front can't read yet wait, not lose the view When the front's QMD view could not read a descriptor as the back will (a write into it still queued, work that may write anywhere, a change of the mappings), it dropped its table and waited until the back had run everything sent. In test49d that took the rest of the frame: about 330 signals unvalued after each of 55 losses a second. The table is fine up to that signal. The signal now waits instead, with what it changed in the table taken back: NativeQmdScheduler records the states an attempt changes (BeginAttempt) and restores them (RollbackAttempt), also the finished ones it drops at its 4,096 limit. Every signal sent after it waits behind it, so the table stays the back's as of the signals valued. Before the next signal of the same address space, the front values the waiting ones in order once the back ran what the first waits for, with the same scheduling, and then the new one as before. Nothing about the back changes. Late, the back may have run the signal and what came after it. The front still reads what the back read, as long as: - nothing sent after the signal writes the QMD (GpuSplit's write positions, the unplaced writes included); - no change of the mappings was sent after it; - no release of a waiting signal writes a QMD read late, before or after the read in the same signal. Otherwise it loses the table as before, as when too many signals wait (4,096). Their releases are not noted for the semaphore view. They are placed at their signals' positions (GpuSplit::NoteWriteAt), so the signals valued after them wait for those writes as for any other. A takeover drops what waits: the back ran it. A signal the view could not value no longer counts as a write anywhere (NoteUnplacedWrite). The view was the only reader of the position: while signals wait it values nothing past them, and with its table lost it takes the back's over only once the back ran everything. Tested: - QMD harness, a new mode as the game builds QMDs: a ring of 512 written inline, each patched once, then launched by a decrement, with the back 4 to 200 entries behind and never idle. Before, the view valued 626 launches in 200 runs, lost its table at the first patch still queued and never took it over again. Now it values 2,387 at once and 473,270 late, and never loses it. In a scripted case the waiting ones are valued late and the next one at once; a QMD written after a waiting launch, and a release of a waiting launch into a QMD read late, lose the table until the next takeover. All eight modes pass with and without EDEN_NX2_DRAIN_CHECK: 2.04 million check lists, nothing released or read otherwise, every table compared equal. - Unit tests, run with a small Catch2 stand-in: an attempt that stops at a QMD it can't read restores the table, and the history it dropped at the limit; one committed matches a table that never tried. 7 passed. - Split harness: 20 runs on 2 CPUs and 10 pinned to one passed. - The compile check passes on the changed files, the users of the changed headers and the unit tests. clang's strict warnings report nothing on the changed lines. Co-Authored-By: Claude Opus 5.5 --- src/tests/video_core/maxwell_fast_method.cpp | 9 +- src/tests/video_core/native_qmd.cpp | 98 ++++++ src/video_core/engines/kepler_compute.cpp | 6 + src/video_core/front_qmd.h | 305 +++++++++++++++---- src/video_core/gpu_split.cpp | 26 +- src/video_core/gpu_split.h | 13 +- src/video_core/native_qmd.h | 43 +++ src/video_core/nx2_drain_check.h | 18 +- 8 files changed, 432 insertions(+), 86 deletions(-) diff --git a/src/tests/video_core/maxwell_fast_method.cpp b/src/tests/video_core/maxwell_fast_method.cpp index 561f6a4f13..b9816c04d4 100644 --- a/src/tests/video_core/maxwell_fast_method.cpp +++ b/src/tests/video_core/maxwell_fast_method.cpp @@ -1619,20 +1619,21 @@ TEST_CASE("Split front values the releases of the QMD launches it sends as the b Access::RunBack(split); CHECK(word(0x20) == 7); - // Work that may write guest memory anywhere, still queued: the front can't tell what the - // back reads, its table is no longer the back's. + // Work that may write guest memory anywhere and change the mappings, still queued: the front + // can't tell what the back reads yet, the launch waits for it. split.Defer([] {}); inline_qmd(qmds + 0x500, make_qmd(0, 0x30, Enable | OneWord, 1), false); CHECK_FALSE(queued(0x30).has_value()); Access::RunBack(split); - // Taken over again once the back ran everything. + // The back ran everything: the front takes its table over, the waiting launch with it. inline_qmd(qmds + 0x600, make_qmd(0, 0x30, Enable | OneWord, 2), true); release = queued(0x30); REQUIRE(release.has_value()); CHECK(release->payload == 2); Access::RunBack(split); - // A write to a waiting QMD still queued when its decrement is sent: not valued. + // A write to a waiting QMD still queued when its decrement is sent: the launch waits, not + // valued then. inline_qmd(qmds + 0x700, make_qmd(1, 0x40, Enable | OneWord | Maximum, 3), false); Access::RunBack(split); split.Defer([] {}, Writes::Noted); diff --git a/src/tests/video_core/native_qmd.cpp b/src/tests/video_core/native_qmd.cpp index 2d00eb8267..2a7943a018 100644 --- a/src/tests/video_core/native_qmd.cpp +++ b/src/tests/video_core/native_qmd.cpp @@ -4,7 +4,9 @@ #include #include #include +#include #include +#include #include #include @@ -191,3 +193,99 @@ TEST_CASE("A QMD scheduler with a table of its own schedules apart from the shar REQUIRE(signal(other, 7).empty()); REQUIRE(signal(other, 3) == run); } + +TEST_CASE("A QMD scheduler takes back what an attempt changed and keeps what one committed", + "[video_core][native_qmd]") { + constexpr Address first = 0x40000; + constexpr Address second = 0x40100; + auto waiting = MakeQmd(); + waiting[4] = 1U << 14; // One dependency + SetRelease(waiting, 24, 0x5000, Enable | OneWord, 1); + auto chained = MakeQmd(); // Decrements `first` when it runs + SetRelease(chained, 24, 0x5004, Enable | OneWord, 2); + chained[15] = static_cast(first >> 8); + chained[16] = 1 | (4U << 1); + std::map memory{{first, waiting}, {second, chained}}; + bool unreadable{}; + const auto signal = [&](NativeQmdScheduler& scheduler, Address address, Word action) { + std::vector released; + const auto error = scheduler.Signal( + address, action, + [&](Address qmd_address, Descriptor& descriptor) { + const auto it = memory.find(qmd_address); + if (it == memory.end() || (unreadable && qmd_address == first)) { + return false; + } + descriptor = it->second; + return true; + }, + [&](Address destination, Word payload, Word flags) { + released.push_back({destination, payload, flags}); + return true; + }); + return std::make_pair(std::string{error}, released); + }; + NativeQmdScheduler own{NativeQmdScheduler::OwnTable{}}; + NativeQmdScheduler reference{NativeQmdScheduler::OwnTable{}}; + REQUIRE(signal(own, first, 3).second.empty()); // Loaded, waits for its dependency + REQUIRE(signal(reference, first, 3).second.empty()); + // The chained launch runs and decrements `first`, which can't be read: taken back, the table + // is as before, `second` never ran. + unreadable = true; + own.BeginAttempt(); + REQUIRE_FALSE(signal(own, second, 3).first.empty()); + own.RollbackAttempt(); + REQUIRE(own.SameStates(reference)); + REQUIRE(own.StateCount() == 1); + // Tried again once it can be read: both run, as on a table that never tried. + unreadable = false; + own.BeginAttempt(); + const auto ran = signal(own, second, 3); + own.CommitAttempt(); + const std::vector both{{0x5004, 2, Enable | OneWord}, {0x5000, 1, Enable | OneWord}}; + REQUIRE(ran.first.empty()); + REQUIRE(ran.second == both); + REQUIRE(signal(reference, second, 3).second == both); + REQUIRE(own.SameStates(reference)); + // A rollback after the commit changes nothing. + own.RollbackAttempt(); + REQUIRE(own.SameStates(reference)); +} + +TEST_CASE("A QMD scheduler takes back the history an attempt dropped at its limit", + "[video_core][native_qmd]") { + // Any address holds a QMD that runs at once; one of them can't be read. + const Descriptor qmd = MakeQmd(); + Address unreadable = 0; + const auto signal = [&](NativeQmdScheduler& scheduler, Address address) { + return std::string{scheduler.Signal( + address, 3, + [&](Address qmd_address, Descriptor& descriptor) { + descriptor = qmd; + return qmd_address != unreadable; + }, + [](Address, Word, Word) { return true; })}; + }; + NativeQmdScheduler own{NativeQmdScheduler::OwnTable{}}; + NativeQmdScheduler reference{NativeQmdScheduler::OwnTable{}}; + for (Address i = 0; i < 4096; ++i) { + REQUIRE(signal(own, 0x100000 + i * 0x100).empty()); + REQUIRE(signal(reference, 0x100000 + i * 0x100).empty()); + } + REQUIRE(own.StateCount() == 4096); + // A new QMD past the limit drops the finished ones first; it can't be read: all taken back. + unreadable = 0x900000; + own.BeginAttempt(); + REQUIRE_FALSE(signal(own, unreadable).empty()); + own.RollbackAttempt(); + REQUIRE(own.StateCount() == 4096); + REQUIRE(own.SameStates(reference)); + // Kept, it drops them as the reference does. + unreadable = 0; + own.BeginAttempt(); + REQUIRE(signal(own, 0x900000).empty()); + own.CommitAttempt(); + REQUIRE(signal(reference, 0x900000).empty()); + REQUIRE(own.StateCount() == 1); + REQUIRE(own.SameStates(reference)); +} diff --git a/src/video_core/engines/kepler_compute.cpp b/src/video_core/engines/kepler_compute.cpp index 67a368c428..b1fac746df 100644 --- a/src/video_core/engines/kepler_compute.cpp +++ b/src/video_core/engines/kepler_compute.cpp @@ -1474,6 +1474,12 @@ struct FrontQmdMemory { void NoteWrite(VideoCommon::GpuSplit& split, GPUVAddr address, u64 size) const { split.NoteWrite(memory, address, size); } + void NoteWriteAt(VideoCommon::GpuSplit& split, GPUVAddr address, u64 size, u64 position) const { + split.NoteWriteAt(memory, address, size, position); + } + [[nodiscard]] const void* Id() const { + return &memory; + } }; // The QMD methods: PCAS (0xae-0xb0) and the inline QMD's last word (0x107). diff --git a/src/video_core/front_qmd.h b/src/video_core/front_qmd.h index 88b94a49db..4ea6abc732 100644 --- a/src/video_core/front_qmd.h +++ b/src/video_core/front_qmd.h @@ -6,8 +6,10 @@ #include #include #include +#include #include #include +#include #include #include "common/common_types.h" @@ -33,17 +35,28 @@ namespace VideoCommon { /// write sent there (NoteInlineWrite). /// No change of the GPU's mappings may be queued either (GpuSplit::LastMappingChange). Memory holds /// what the back finds there only while the back has not run the signal's entry yet: the front -/// sends and values it under a GpuSplit::PublishHold. Where the front can't tell what the back -/// reads, its table may no longer be the back's: it stops valuing, -/// the launches count as writing guest memory anywhere, and it takes the back's table over again -/// the next time the back has run everything sent, between commands or at a QMD method -/// (MaybeTakeOver). +/// sends and values it under a GpuSplit::PublishHold. +/// +/// Where the back has yet to run a write the front can't read past (into a QMD the signal reads, +/// or anywhere, or to the mappings), the signal waits, with what it changed in the table taken +/// back (NativeQmdScheduler::BeginAttempt), and every signal sent after it waits behind it: the +/// table stays the back's as of the signals valued. The front values the waiting ones in order +/// once the back ran what the first waits for (CatchUp). Late, the back may have run the signal +/// and what followed it, so memory holds what the back read only where nothing sent after the +/// signal writes the QMD, and no release of the waiting signals writes a QMD read late. Their +/// releases are placed at their signals' positions, not noted. Where the front can't read what +/// the back read at all (a QMD on a GPU-written page, written after the signal, released into, +/// or the mappings changed after it), it loses the table: it stops valuing, and takes the back's +/// table over the next time the back has run everything sent, between commands or at a QMD +/// method (MaybeTakeOver). /// /// `Memory`, the channel's GPU address space as the back reads it, has: /// - bool IsMapped(GPUVAddr, u64), void Read(GPUVAddr, void*, u64), u32 ReadWord(GPUVAddr) (as /// MemoryManager::Read), bool InSyncpointShim(GPUVAddr, u64); /// - std::optional Device(GPUVAddr), bool GpuWritten(u64 device, u64 size); -/// - u64 WritesTarget(const GpuSplit&, GPUVAddr, u64), void NoteWrite(GpuSplit&, GPUVAddr, u64). +/// - u64 WritesTarget(const GpuSplit&, GPUVAddr, u64), void NoteWrite(GpuSplit&, GPUVAddr, u64), +/// void NoteWriteAt(GpuSplit&, GPUVAddr, u64 size, u64 position); +/// - const void* Id(): the address space's, the same for every Memory of it. class FrontQmdView { public: using Descriptor = Tegra::NativeQmdScheduler::Descriptor; @@ -60,14 +73,15 @@ class FrontQmdView { const auto start = std::chrono::steady_clock::now(); // TEST scheduler.CopyStates(back); synced = true; + ClearWaiting(); Nx2AcquireStats::NoteQmdSync(where, std::chrono::steady_clock::now() - start); } /// The front, between commands and before QMD methods: takes the back's table over if the - /// view lost it and the back has run everything sent. + /// view lost it or signals wait, and the back has run everything sent. void MaybeTakeOver(const GpuSplit& split, const Tegra::NativeQmdScheduler& back, Nx2AcquireStats::QmdTakeOver where) { - if (!synced && split.Ran(split.Position())) [[unlikely]] { + if (!Current() && split.Ran(split.Position())) [[unlikely]] { TakeOver(back, where); } } @@ -90,7 +104,7 @@ class FrontQmdView { } checks_pending = false; Checks taken{std::move(checks), std::move(check_reads), nullptr, !checks_unvalued}; - if (taken.valued && synced && ++table_checks % 512 == 0) { + if (taken.valued && Current() && ++table_checks % 512 == 0) { auto copy = std::make_shared(Tegra::NativeQmdScheduler::OwnTable{}); copy->CopyStates(scheduler); @@ -106,11 +120,16 @@ class FrontQmdView { void Lose() { synced = false; inline_writes_count = 0; + ClearWaiting(); } [[nodiscard]] bool Synced() const { return synced; } + /// Whether the table is the back's as of everything sent: synced, and no signal waits. + [[nodiscard]] bool Current() const { + return synced && waiting.empty(); + } /// LOAD_INLINE_QMD_DATA was just sent: running it, the back writes `qmd` to the QMD at /// `device`. Until then the front reads it from here (ReadDescriptor). @@ -125,46 +144,192 @@ class FrontQmdView { /// The scheduling of PCAS action `argument` on the QMD at `address` that the back runs at the /// entry just sent (KeplerCompute::ProcessNativeMethod); `inline_qmd`: the QMD it writes there - /// right before. Notes what the launches it runs release. The entry must still be unpublished - /// (GpuSplit::PublishHold): the back must not have run it. + /// right before. Notes what the launches it runs release, or has the signal wait. The entry + /// must still be unpublished (GpuSplit::PublishHold): the back must not have run it. template void Signal(GpuSplit& split, const Memory& memory, GPUVAddr address, u32 argument, const Descriptor* inline_qmd) { split.NoteCacheWrite(); // The dispatches' shader stores using Lost = Nx2AcquireStats::QmdLost; checks_pending = Nx2DrainCheck::g_enabled; // TEST + const u64 position = split.Position(); + if (synced) { + CatchUp(split, memory); + } if (!synced) { Unvalued(split, Lost::NotSynced); return; } - // A change of the mappings still queued may unmap what the back reads or releases to. - if (!split.Ran(split.LastMappingChange())) { + if (!waiting.empty()) { + // Behind a signal still waiting. + Wait(split, position, memory.Id(), address, argument, inline_qmd, std::nullopt); + return; + } + const Outcome outcome = + Attempt(split, memory, position, address, argument, inline_qmd, false); + if (outcome.valued) { + Nx2AcquireStats::NoteQmdSignal(std::nullopt); + return; + } + if (outcome.retry_after) { + retry_after = *outcome.retry_after; + Wait(split, position, memory.Id(), address, argument, inline_qmd, outcome.lost); + return; + } + synced = false; + Unvalued(split, outcome.lost); + } + +private: + struct Release { + GPUVAddr address; + u32 payload; + u32 flags; + }; + + /// An inline QMD a LOAD_INLINE_QMD_DATA sent writes (NoteInlineWrite). + struct InlineWrite { + u64 device{}; + u64 position{}; + Descriptor qmd{}; + bool overwritten{}; ///< A release of the same entry writes into it after + }; + + /// A signal sent at `position` that waits to be valued (`inline_qmd` if `has_inline`). + struct WaitingSignal { + u64 position; + const void* memory; ///< Memory::Id of the address space it reads + GPUVAddr address; + u32 argument; + bool has_inline; + Descriptor inline_qmd; + }; + + /// How an attempt to value a signal ended: valued, or taken back, to try again once the back + /// ran `retry_after`, or for good (the view loses its table). + struct Outcome { + bool valued; + std::optional retry_after; + Nx2AcquireStats::QmdLost lost; + }; + + /// Signals that wait, at most: a back that far behind leaves the view to its next takeover. + static constexpr size_t MaxWaiting = 4096; + + /// A signal the front can't value, its table lost: no release of it is noted, and the view + /// takes the back's table over once the back ran it. + void Unvalued(const GpuSplit& split, Nx2AcquireStats::QmdLost lost) { + checks_unvalued = true; // TEST + Nx2AcquireStats::NoteQmdSignal(lost, WhatSent(split, lost)); + } + + /// TEST: what sent the last write the view could not read past, for `lost`. + [[nodiscard]] static UnplacedKind WhatSent(const GpuSplit& split, + Nx2AcquireStats::QmdLost lost) { + return lost == Nx2AcquireStats::QmdLost::Mappings ? split.LastMappingKind() + : split.LastUnplacedKind(); + } + + /// The signal sent at `position` waits, first (`why`: what it waits for) or behind the others. + /// None of its releases is noted: they are placed once it is valued. + void Wait(const GpuSplit& split, u64 position, const void* memory_id, GPUVAddr address, + u32 argument, const Descriptor* inline_qmd, + std::optional why) { + checks_unvalued = true; // TEST + if (waiting.size() >= MaxWaiting) { synced = false; - Unvalued(split, Lost::Mappings); + ClearWaiting(); + Nx2AcquireStats::NoteQmdSignal(Nx2AcquireStats::QmdLost::NotSynced); // TEST return; } + const auto lost = why.value_or(Nx2AcquireStats::QmdLost::NotSynced); // TEST + Nx2AcquireStats::NoteQmdSignal(lost, WhatSent(split, lost)); + WaitingSignal& signal = waiting.emplace_back(); + signal.position = position; + signal.memory = memory_id; + signal.address = address; + signal.argument = argument; + signal.has_inline = inline_qmd != nullptr; + if (inline_qmd != nullptr) { + signal.inline_qmd = *inline_qmd; + } + } + + void ClearWaiting() { + waiting.clear(); + late_reads.clear(); + retry_after = 0; + } + + /// Values the waiting signals that read `memory` in order, from the first, as long as the back + /// ran what the first waits for. + template + void CatchUp(GpuSplit& split, const Memory& memory) { + while (!waiting.empty()) { + const WaitingSignal& first = waiting.front(); + if (first.memory != memory.Id() || !split.Ran(retry_after)) { + return; + } + const Outcome outcome = + Attempt(split, memory, first.position, first.address, first.argument, + first.has_inline ? &first.inline_qmd : nullptr, true); + if (outcome.retry_after) { + retry_after = *outcome.retry_after; + return; + } + if (!outcome.valued) { + synced = false; + ClearWaiting(); + return; + } + waiting.pop_front(); + retry_after = 0; + } + // Every signal sent before the reads made late is valued: none of their releases waits. + late_reads.clear(); + } + + /// Runs the signal sent at `position` on the table as the back runs it, and notes what its + /// launches release, or places it if `late`. Takes it back where the front can't read a QMD + /// as the back reads it. + template + Outcome Attempt(GpuSplit& split, const Memory& memory, u64 position, GPUVAddr address, + u32 argument, const Descriptor* inline_qmd, bool late) { + using Lost = Nx2AcquireStats::QmdLost; + // The back finds the mappings every change sent before the signal made, and none after it. + const u64 mapping_change = split.LastMappingChange(); + if (mapping_change > position) { + return {false, std::nullopt, Lost::MappedAfter}; + } + if (!split.Ran(mapping_change)) { + return {false, mapping_change, Lost::Mappings}; + } releases.clear(); - std::optional lost; + attempt_reads.clear(); + std::optional failure; const auto read = [&](u64 qmd_address, Descriptor& descriptor) { if (!memory.IsMapped(qmd_address, sizeof(descriptor))) { return false; } if (inline_qmd != nullptr && qmd_address == address) { descriptor = *inline_qmd; - } else if (const auto unknown = ReadDescriptor(split, memory, qmd_address, descriptor); - unknown && !lost) { - lost = unknown; - } - if (checks_pending) { - check_reads.push_back({qmd_address, descriptor}); // TEST + } else if (const auto unread = + ReadDescriptor(split, memory, qmd_address, position, descriptor)) { + failure = unread; + return false; // Stops the scheduling, which is taken back } // A release this signal ran before writes into the QMD. for (const Release& earlier : releases) { - if (!lost && earlier.address + 16 > qmd_address && + if (earlier.address + 16 > qmd_address && earlier.address < qmd_address + sizeof(descriptor)) { - lost = Lost::ReleasedInto; + failure = Outcome{false, std::nullopt, Lost::ReleasedInto}; + return false; } } + attempt_reads.push_back(qmd_address); + if (checks_pending && !late) { + check_reads.push_back({qmd_address, descriptor}); // TEST + } return true; }; const auto release = [&](u64 destination, u32 payload, u32 flags) { @@ -175,60 +340,68 @@ class FrontQmdView { releases.push_back({destination, payload, flags}); return true; }; + scheduler.BeginAttempt(); scheduler.Signal(address, argument, read, release, [](const Descriptor&) { return true; }); - if (lost) { - synced = false; - Unvalued(split, *lost); - return; + if (!failure && late && ReleasesIntoLateReads()) { + failure = Outcome{false, std::nullopt, Lost::LateReleasedInto}; } - Nx2AcquireStats::NoteQmdSignal(std::nullopt); - for (const Release& noted : releases) { - NoteRelease(split, memory, noted.address, noted.payload, noted.flags); + if (failure) { + scheduler.RollbackAttempt(); + return *failure; } + scheduler.CommitAttempt(); + if (late) { + for (const u64 read_address : attempt_reads) { + late_reads.insert(read_address >> 8); + } + for (const Release& done : releases) { + memory.NoteWriteAt(split, done.address, (done.flags >> 30) == 0 ? 16 : 4, position); + } + } else { + for (const Release& noted : releases) { + NoteRelease(split, memory, noted.address, noted.payload, noted.flags); + } + } + return {true, std::nullopt, Lost::NotSynced}; } -private: - struct Release { - GPUVAddr address; - u32 payload; - u32 flags; - }; - - /// An inline QMD a LOAD_INLINE_QMD_DATA sent writes (NoteInlineWrite). - struct InlineWrite { - u64 device{}; - u64 position{}; - Descriptor qmd{}; - bool overwritten{}; ///< A release of the same entry writes into it after - }; - - /// A signal the front can't value: its releases may write guest memory anywhere (they change - /// no mappings). - void Unvalued(GpuSplit& split, Nx2AcquireStats::QmdLost lost) { - // TEST: what sent the write the view could not read past. - const UnplacedKind kind = lost == Nx2AcquireStats::QmdLost::Mappings - ? split.LastMappingKind() - : split.LastUnplacedKind(); - split.NoteUnplacedWrite(); - split.TagUnplaced(UnplacedKind::QmdLaunch); // TEST - checks_unvalued = true; // TEST - Nx2AcquireStats::NoteQmdSignal(lost, kind); + /// Late: whether a release of the signal writes a QMD it read, or one a signal valued late + /// before read. The back may have run the release before the front read that QMD. + [[nodiscard]] bool ReleasesIntoLateReads() const { + for (const Release& done : releases) { + const u64 first = done.address >> 8; + const u64 last = (done.address + 15) >> 8; + for (const u64 read_address : attempt_reads) { + if ((read_address >> 8) == first || (read_address >> 8) == last) { + return true; + } + } + if (late_reads.contains(first) || late_reads.contains(last)) { + return true; + } + } + return false; } - /// Reads into `descriptor` the QMD at `qmd_address` as the back will read it running the entry - /// just sent, or tells why the front can't. + /// Reads into `descriptor` the QMD at `qmd_address` as the back read it, or will, running the + /// signal sent at `position`, or tells why the front can't. template - [[nodiscard]] std::optional ReadDescriptor( - const GpuSplit& split, const Memory& memory, GPUVAddr qmd_address, - Descriptor& descriptor) const { + [[nodiscard]] std::optional ReadDescriptor(const GpuSplit& split, const Memory& memory, + GPUVAddr qmd_address, u64 position, + Descriptor& descriptor) const { using Lost = Nx2AcquireStats::QmdLost; memory.Read(qmd_address, descriptor.data(), sizeof(descriptor)); const auto device = memory.Device(qmd_address); if (!device || memory.GpuWritten(*device, sizeof(descriptor))) { - return Lost::GpuWritten; + return Outcome{false, std::nullopt, Lost::GpuWritten}; } // The last write sent there, placed or not (the inline write covers the whole QMD). const u64 target = memory.WritesTarget(split, qmd_address, sizeof(descriptor)); + const Lost lost = target == split.LastUnplacedWrite() ? Lost::Unplaced : Lost::Pending; + if (target > position) { + // Sent after the signal (it waited): memory may hold it, the back read what was before. + return Outcome{false, std::nullopt, Lost::WrittenAfter}; + } if (split.Ran(target)) { return std::nullopt; } @@ -238,7 +411,7 @@ class FrontQmdView { Nx2AcquireStats::NoteQmdInlineRead(); // TEST return std::nullopt; } - return target == split.LastUnplacedWrite() ? Lost::Unplaced : Lost::Pending; + return Outcome{false, target, lost}; } /// The last inline QMD sent to the QMD at `device` that the back has not written yet. @@ -350,7 +523,13 @@ class FrontQmdView { /// The table the engines of every channel share, which the back's scheduling uses. Tegra::NativeQmdScheduler back_table; std::vector releases; + std::vector attempt_reads; ///< The QMDs the attempt read bool synced{}; + /// Signals that wait to be valued, oldest first; the first may be once the back ran + /// retry_after. late_reads: the granules of the QMDs read late since the waiting began. + std::deque waiting; + u64 retry_after{}; + std::unordered_set late_reads; /// The last inline QMDs sent, oldest first from inline_writes_count on: a back a millisecond /// behind has some ten to write, a few times that in long frames. std::array inline_writes{}; diff --git a/src/video_core/gpu_split.cpp b/src/video_core/gpu_split.cpp index c3e1dbdf6b..35e8abe626 100644 --- a/src/video_core/gpu_split.cpp +++ b/src/video_core/gpu_split.cpp @@ -319,6 +319,11 @@ void ForEachDeviceRange(const Tegra::MemoryManager& memory, GPUVAddr address, u6 } // Anonymous namespace void GpuSplit::NoteWrite(const Tegra::MemoryManager& memory, GPUVAddr address, u64 size) { + NoteWriteAt(memory, address, size, write_position); +} + +void GpuSplit::NoteWriteAt(const Tegra::MemoryManager& memory, GPUVAddr address, u64 size, + u64 position) { if (size == 0) { return; } @@ -328,13 +333,14 @@ void GpuSplit::NoteWrite(const Tegra::MemoryManager& memory, GPUVAddr address, u return; } if (size > MaxNotedBytes) { - ForEachDeviceRange(memory, address, size, [this](DAddr device, u64 bytes) { - NoteCoarseDeviceWrite(device, bytes); + ForEachDeviceRange(memory, address, size, [this, position](DAddr device, u64 bytes) { + NoteCoarseDeviceWriteAt(device, bytes, position); }); return; } - ForEachDeviceRange(memory, address, size, - [this](DAddr device, u64 bytes) { NoteDeviceWrite(device, bytes); }); + ForEachDeviceRange(memory, address, size, [this, position](DAddr device, u64 bytes) { + NoteDeviceWriteAt(device, bytes, position); + }); } void GpuSplit::NoteSetWrite(WriteSet& set, u64 granule, u64 position) { @@ -366,20 +372,20 @@ u64 GpuSplit::SetTarget(const WriteSet& set, u64 granule) { return set.evicted; } -void GpuSplit::NoteDeviceWrite(DAddr address, u64 size) { +void GpuSplit::NoteDeviceWriteAt(DAddr address, u64 size, u64 position) { const u64 last = (address + size - 1) >> WriteGranuleBits; for (u64 granule = address >> WriteGranuleBits; granule <= last; ++granule) { - NoteSetWrite(write_sets[WriteSlot(granule)], granule, write_position); + NoteSetWrite(write_sets[WriteSlot(granule)], granule, position); } - last_noted_write = write_position; + last_noted_write = std::max(last_noted_write, position); } -void GpuSplit::NoteCoarseDeviceWrite(DAddr address, u64 size) { +void GpuSplit::NoteCoarseDeviceWriteAt(DAddr address, u64 size, u64 position) { const u64 last = (address + size - 1) >> CoarseGranuleBits; for (u64 granule = address >> CoarseGranuleBits; granule <= last; ++granule) { - NoteSetWrite(coarse_sets[CoarseSlot(granule)], granule, write_position); + NoteSetWrite(coarse_sets[CoarseSlot(granule)], granule, position); } - last_noted_write = write_position; + last_noted_write = std::max(last_noted_write, position); } u64 GpuSplit::WritesTarget(const Tegra::MemoryManager& memory, GPUVAddr address, u64 size) const { diff --git a/src/video_core/gpu_split.h b/src/video_core/gpu_split.h index f888d6188d..97f14c5150 100644 --- a/src/video_core/gpu_split.h +++ b/src/video_core/gpu_split.h @@ -344,6 +344,9 @@ class GpuSplit { /// mapped now. Up to MaxNotedBytes by 256-byte granule, up to MaxCoarseNotedBytes by 64 KiB /// granule, longer writes count as unplaced. void NoteWrite(const Tegra::MemoryManager& memory, GPUVAddr address, u64 size); + /// NoteWrite for the entry sent at `position` (the one before it ends there), placed late: the + /// front's QMD view values a launch once the back ran what it waited for. + void NoteWriteAt(const Tegra::MemoryManager& memory, GPUVAddr address, u64 size, u64 position); /// The entry just sent may write guest memory anywhere and change the GPU's mappings /// (Writes::Anywhere). void NoteAnyWrite() { @@ -466,8 +469,14 @@ class GpuSplit { static void NoteSetWrite(WriteSet& set, u64 granule, u64 position); /// The latest position of the writes to `granule` in its set, at most. [[nodiscard]] static u64 SetTarget(const WriteSet& set, u64 granule); - void NoteDeviceWrite(DAddr address, u64 size); - void NoteCoarseDeviceWrite(DAddr address, u64 size); + void NoteDeviceWrite(DAddr address, u64 size) { + NoteDeviceWriteAt(address, size, write_position); + } + void NoteDeviceWriteAt(DAddr address, u64 size, u64 position); + void NoteCoarseDeviceWrite(DAddr address, u64 size) { + NoteCoarseDeviceWriteAt(address, size, write_position); + } + void NoteCoarseDeviceWriteAt(DAddr address, u64 size, u64 position); /// The position WaitForWrites waits for. [[nodiscard]] u64 WritesTarget(const Tegra::MemoryManager& memory, GPUVAddr address, u64 size) const; diff --git a/src/video_core/native_qmd.h b/src/video_core/native_qmd.h index 42dd1ed305..ed16358774 100644 --- a/src/video_core/native_qmd.h +++ b/src/video_core/native_qmd.h @@ -5,10 +5,13 @@ #include #include #include +#include #include #include #include #include +#include +#include namespace Tegra { @@ -59,6 +62,30 @@ class NativeQmdScheduler { [[nodiscard]] std::size_t StateCount() const { return table->size(); } + /// The split GPU thread's front runs signals it may have to take back, where it can't read a + /// QMD as the back will yet (FrontQmdView): from BeginAttempt on, Signal records what it + /// changes in the table, until CommitAttempt keeps it or RollbackAttempt restores the table. + void BeginAttempt() { + undo.clear(); + attempting = true; + } + void CommitAttempt() { + undo.clear(); + attempting = false; + } + void RollbackAttempt() { + auto& states = *table; + // Newest first: each QMD ends with the state it had before the attempt. + for (auto it = undo.rbegin(); it != undo.rend(); ++it) { + if (it->second) { + states.insert_or_assign(it->first, *it->second); + } else { + states.erase(it->first); + } + } + undo.clear(); + attempting = false; + } // Optional diagnostics: called with "load"/"count"/"defer"/"run" events. std::function trace; // Optional diagnostics: called with the QMD, slot and raw words of a release it skips. @@ -90,11 +117,24 @@ class NativeQmdScheduler { const auto request = pending[i]; if (states.size() >= 4096 && !states.contains(request.address)) { // Finished QMDs with nothing deferred are only history, drop them instead of stalling. + if (attempting) { + for (const auto& [kept_address, kept] : states) { + if (kept.completed && kept.deferred == 0) { + undo.emplace_back(kept_address, kept); + } + } + } std::erase_if(states, [](const auto& entry) { return entry.second.completed && entry.second.deferred == 0; }); if (states.size() >= 4096) return "QMD cache limit"; } + if (attempting) { + const auto found = states.find(request.address); + undo.emplace_back(request.address, found != states.end() + ? std::optional{found->second} + : std::nullopt); + } auto& state = states[request.address]; if (request.action == 3 || request.action == 2) { if (request.action == 3 || !state.loaded) { @@ -183,5 +223,8 @@ class NativeQmdScheduler { static inline std::unordered_map shared_states; std::unordered_map own_states; std::unordered_map* table{&shared_states}; + /// BeginAttempt: the states the attempt changed, as they were (none: the QMD had no state). + bool attempting{}; + std::vector>> undo; }; } // namespace Tegra diff --git a/src/video_core/nx2_drain_check.h b/src/video_core/nx2_drain_check.h index c900adc238..c4df8edd76 100644 --- a/src/video_core/nx2_drain_check.h +++ b/src/video_core/nx2_drain_check.h @@ -390,14 +390,18 @@ enum class Releaser : u32 { Count, }; -/// Why the front's QMD view could not value a signal (FrontQmdView::Signal). +/// Why the front's QMD view could not value a signal (FrontQmdView::Signal), or one that waited. enum class QmdLost : u32 { - NotSynced, ///< Its table is not the back's yet (the back has not caught up since) - Unplaced, ///< Work that may write guest memory anywhere was still queued - GpuWritten, ///< A QMD it read is on a page GPU work wrote - Pending, ///< A write into a QMD it read was still queued - ReleasedInto, ///< A release of the same signal writes into a QMD it read - Mappings, ///< Work that may change the GPU's mappings was still queued + NotSynced, ///< Its table is not the back's yet (the back has not caught up since) + Unplaced, ///< Work that may write guest memory anywhere was still queued + GpuWritten, ///< A QMD it read is on a page GPU work wrote + Pending, ///< A write into a QMD it read was still queued + ReleasedInto, ///< A release of the same signal writes into a QMD it read + Mappings, ///< Work that may change the GPU's mappings was still queued + WrittenAfter, ///< Waiting, a QMD it reads was written after it + MappedAfter, ///< Waiting, the mappings may have changed after it + LateReleasedInto, ///< Waiting, a release of it writes a QMD read late + TooMany, ///< Too many signals waited Count, }; From b57e486f2c2ba323575accb275426917f93324b3 Mon Sep 17 00:00:00 2001 From: jpkto <255363096+jpkto@users.noreply.github.com> Date: Sun, 8 Nov 2026 07:44:54 +0100 Subject: [PATCH 059/105] TEST: check the QMD view's late valuing and publish hold, name what its signals wait for EDEN_NX2_DRAIN_CHECK checks the new paths of the front's QMD view; the counts below must stay 0. The QMD part of the "Drain checks" line is now: QMD view: methods valued N with R releases, released otherwise than noted W, with D descriptors read, read otherwise X; valued after the back could run them P; signals valued late L, read otherwise Y, not compared M; tables compared T (after valuing late A), differing Z; not valued U - P: QMD methods the front valued with their entry already published, so the back could have run them before it read. It checks the publish hold at every signal and logs "QMD view check: the front valued a QMD method the back could run". - L, Y, M: the back marks where each signal's reads end. For a method whose signals wait, it keeps their reads by the position of the entry and their place in it, up to 16,384. Once the front values them late, it sends what it read, and the back compares. A difference logs "QMD view check: the back read other descriptors for a signal the front valued late". M counts the signals whose reads were no longer kept. - A: after every catch-up the next check carries a copy of the front's table, not only every 512th one. A difference logs "...differs from the back's after it valued signals late". EDEN_NX2_ACQUIRE_STATS, "QMD view" line: signals valued at once and late (how many catch-ups); signals that waited first, by why, and behind an earlier one; waiting signals lost (a QMD written after it, the mappings changed after it, a release into a QMD read late, too many waiting) and how many they dropped; waiting signals takeovers dropped (the back had run them); not valued with the table lost. A wait for a write into a QMD names its writer: a release into one of its words the front noted, else the engine and method whose entry ends at the write's position, else "another entry" (a deferred call, a CPU function). With the variable set, GpuSplit keeps the last 65,536 engine method entries for that. The first 16 waits are logged with the QMD, the writer, how long before the signal it was sent and how far behind the back was. Tested: - QMD harness: every mode passes with the checks on. 480,163 signals valued late were compared with what the back read: none read otherwise, none missing. All 2,288 tables compared after a catch-up were the back's. The harness's back is reused from run to run, so a position from an earlier run starts over at 0, as after a split restart: kept reads past it are dropped. - With the "written after the signal" rule taken out of the late reads, 7,316 checks fail, 1,838 of them on tables. Without the hold, the race mode counts 53,128 methods valued after the back could run them in 20 runs, and logs the error line. - Split harness: the method log finds the engine and method of a method, a multi-method and a macro entry by the position after them, nothing for a deferred call, and nothing once 70,000 newer methods pushed an entry out. 87 checks; 20 runs on 2 CPUs and 10 pinned to one passed with EDEN_NX2_ACQUIRE_STATS, 10 without. - A small program printed both report lines with sample counters. - The compile check passes on the changed files and the users of the changed headers. clang's strict warnings report nothing on the changed lines. The method log names an engine by the class DmaPusher::BindSubchannel bound it as (EngineInterface::bound_type, taken when the method is sent), not by typeid: Lichii builds without RTTI (/GR- on MSVC, -fno-rtti on GCC and Clang). With typeid this commit did not compile with GCC or Clang, and MSVC built it, with warning C4541 ("unpredictable behavior"), into code that reads RTTI data the engines don't have; it runs at the first QMD signal that waits for a write by an engine method, which Ocarina of Time sends in its first frames. The MSVC test build (test49l) ended at its first frames with EDEN_NX2_ACQUIRE_STATS set, with no error line. Naming a method later also never reads an engine that may be gone by then. EDEN_NX2_ACQUIRE_STATS turns the log on when the split starts (GpuSplit::MaybeStart), after VideoCore::CreateGPU read the switches (#714), so it works from nx2_switches.txt too. Rebase tests: the split harness checks the names by bound class ("KeplerCompute 0x6d"; "an unbound engine 0x44" before a bind, "MaxwellDMA 0x300" after one): 89 checks pass in 20 runs on 2 CPUs and 10 pinned to one with EDEN_NX2_ACQUIRE_STATS, 80 in 10 without. The QMD harness, every mode with EDEN_NX2_DRAIN_CHECK=1 and EDEN_NX2_ACQUIRE_STATS=1, gives the counts above; without the hold it fails as before (53,128). The split and QMD harnesses and gpu_split.cpp build with -fno-rtti. Co-Authored-By: Claude Opus 5.5 --- src/video_core/dma_pusher.h | 1 + src/video_core/engines/engine_interface.h | 5 + src/video_core/engines/kepler_compute.cpp | 14 +- src/video_core/front_qmd.h | 119 ++++++++- src/video_core/gpu_split.cpp | 65 ++++- src/video_core/gpu_split.h | 23 ++ src/video_core/nx2_drain_check.h | 281 ++++++++++++++++++++-- 7 files changed, 470 insertions(+), 38 deletions(-) diff --git a/src/video_core/dma_pusher.h b/src/video_core/dma_pusher.h index 48f66b8716..a4416486f9 100644 --- a/src/video_core/dma_pusher.h +++ b/src/video_core/dma_pusher.h @@ -183,6 +183,7 @@ class DmaPusher final { void BindSubchannel(Engines::EngineInterface* engine, u32 subchannel_id, Engines::EngineTypes engine_type) { subchannels[subchannel_id] = engine; subchannel_type[subchannel_id] = engine_type; + engine->bound_type = engine_type; } void BindRasterizer(VideoCore::RasterizerInterface* rasterizer); diff --git a/src/video_core/engines/engine_interface.h b/src/video_core/engines/engine_interface.h index 1c5fad7ad8..0a975b3e10 100644 --- a/src/video_core/engines/engine_interface.h +++ b/src/video_core/engines/engine_interface.h @@ -8,6 +8,7 @@ #include #include +#include #include #include "common/common_types.h" @@ -50,6 +51,10 @@ class EngineInterface { std::vector> method_sink{}; GPUVAddr current_dma_segment; bool current_dirty{}; + /// The class DmaPusher::BindSubchannel bound the engine as, if it did: logs name engines by + /// it, as Lichii builds without RTTI (/GR-, -fno-rtti). + std::optional bound_type; + protected: virtual void ConsumeSinkImpl(Core::System& system) { for (auto [method, value] : method_sink) { diff --git a/src/video_core/engines/kepler_compute.cpp b/src/video_core/engines/kepler_compute.cpp index b1fac746df..99291b344d 100644 --- a/src/video_core/engines/kepler_compute.cpp +++ b/src/video_core/engines/kepler_compute.cpp @@ -437,6 +437,10 @@ bool KeplerCompute::ProcessNativeMethod(Core::System& system, u32 method, u32 ar } return true; }, [&](const auto& descriptor) { return DispatchNativeCompute(descriptor); }); + if (VideoCommon::Nx2DrainCheck::g_enabled) [[unlikely]] { + // TEST: where this signal's reads end, for the front's QMD view check. + VideoCommon::Nx2DrainCheck::NoteQmdSignalEnd(VideoCommon::GpuSplit::OnBackThread()); + } if (error.empty()) { LOG_DEBUG(HW_GPU, "Native QMD control action={:#x}, QMD={:#x}", argument, qmd_address); return true; @@ -1664,12 +1668,16 @@ void KeplerCompute::FrontQmdCheck(VideoCommon::GpuSplit& split) { } split.DeferFunction( [predicted = std::move(checks->releases), reads = std::move(checks->reads), - table = std::move(checks->table), valued = checks->valued && QmdReleasesLandAtOnce()] { - VideoCommon::Nx2DrainCheck::CheckQmdReleases(predicted, reads, valued); + table = std::move(checks->table), valued = checks->valued && QmdReleasesLandAtOnce(), + after_late = checks->table_after_late, wait_position = checks->wait_position, + first_waiting = checks->first_waiting, late = std::move(checks->late)] { + VideoCommon::Nx2DrainCheck::CheckQmdReleases(predicted, reads, valued, wait_position, + first_waiting); + VideoCommon::Nx2DrainCheck::CheckLateQmdReads(late); if (table && valued) { const NativeQmdScheduler shared; // The table the engines share VideoCommon::Nx2DrainCheck::NoteQmdTables(table->SameStates(shared), - table->StateCount()); + table->StateCount(), after_late); } }, VideoCommon::GpuSplit::Writes::Noted); diff --git a/src/video_core/front_qmd.h b/src/video_core/front_qmd.h index 4ea6abc732..e4ddd0fe96 100644 --- a/src/video_core/front_qmd.h +++ b/src/video_core/front_qmd.h @@ -9,6 +9,7 @@ #include #include #include +#include #include #include @@ -73,6 +74,7 @@ class FrontQmdView { const auto start = std::chrono::steady_clock::now(); // TEST scheduler.CopyStates(back); synced = true; + Nx2AcquireStats::NoteQmdTakeOverDropped(waiting.size()); // TEST ClearWaiting(); Nx2AcquireStats::NoteQmdSync(where, std::chrono::steady_clock::now() - start); } @@ -91,28 +93,46 @@ class FrontQmdView { /// TEST (EDEN_NX2_DRAIN_CHECK): the releases noted since the last call, the descriptors read /// for them, and whether every signal since was valued; every 512th time while the view is - /// synced, a copy of its table; nothing if no signal was sent since. + /// current, and after it valued signals late, a copy of its table; the position of the entry + /// whose signals wait, from the `first_waiting`-th on; the signals valued late since, with + /// what they read; nothing if no signal was sent since. struct Checks { std::vector releases; std::vector reads; std::shared_ptr table; bool valued; + bool table_after_late; + u64 wait_position; + u32 first_waiting; + std::vector late; }; [[nodiscard]] std::optional TakeChecks() { if (!checks_pending) { return std::nullopt; } checks_pending = false; - Checks taken{std::move(checks), std::move(check_reads), nullptr, !checks_unvalued}; - if (taken.valued && Current() && ++table_checks % 512 == 0) { + Checks taken{std::move(checks), + std::move(check_reads), + nullptr, + !checks_unvalued, + false, + check_wait_position, + check_first_waiting, + std::move(late_checks)}; + if (taken.valued && Current() && (table_check_due || ++table_checks % 512 == 0)) { auto copy = std::make_shared(Tegra::NativeQmdScheduler::OwnTable{}); copy->CopyStates(scheduler); taken.table = std::move(copy); + taken.table_after_late = table_check_due; + table_check_due = false; } checks.clear(); check_reads.clear(); + late_checks.clear(); checks_unvalued = false; + check_wait_position = 0; + check_first_waiting = 0; return taken; } @@ -130,6 +150,10 @@ class FrontQmdView { [[nodiscard]] bool Current() const { return synced && waiting.empty(); } + /// TEST: whether the view's table holds the QMD states `other` does. + [[nodiscard]] bool SameTable(const Tegra::NativeQmdScheduler& other) const { + return scheduler.SameStates(other); + } /// LOAD_INLINE_QMD_DATA was just sent: running it, the back writes `qmd` to the QMD at /// `device`. Until then the front reads it from here (ReadDescriptor). @@ -153,6 +177,14 @@ class FrontQmdView { using Lost = Nx2AcquireStats::QmdLost; checks_pending = Nx2DrainCheck::g_enabled; // TEST const u64 position = split.Position(); + if (checks_pending) { + // TEST: the back must not have run the entry yet, the n-th signal of its entry. + if (split.Unpublished() == 0) [[unlikely]] { + Nx2DrainCheck::NotePublishedEarly(); + } + entry_signals = position != entry_position ? 0 : entry_signals + 1; + entry_position = position; + } if (synced) { CatchUp(split, memory); } @@ -173,6 +205,9 @@ class FrontQmdView { } if (outcome.retry_after) { retry_after = *outcome.retry_after; + if (Nx2AcquireStats::g_enabled && outcome.lost == Lost::Pending) [[unlikely]] { + NoteWaitWriter(split, memory, position, outcome); // TEST + } Wait(split, position, memory.Id(), address, argument, inline_qmd, outcome.lost); return; } @@ -203,6 +238,7 @@ class FrontQmdView { u32 argument; bool has_inline; Descriptor inline_qmd; + u32 index; ///< TEST: of the signals of its entry }; /// How an attempt to value a signal ended: valued, or taken back, to try again once the back @@ -211,6 +247,7 @@ class FrontQmdView { bool valued; std::optional retry_after; Nx2AcquireStats::QmdLost lost; + u64 qmd{}; ///< TEST: the QMD it could not read }; /// Signals that wait, at most: a back that far behind leaves the view to its next takeover. @@ -230,6 +267,39 @@ class FrontQmdView { : split.LastUnplacedKind(); } + /// TEST (EDEN_NX2_ACQUIRE_STATS): what wrote the QMD a signal sent at `position` waits for, as + /// it was sent: a release into it the front noted, else the engine method whose entry ends at + /// the write's position, else another entry (a CPU function, a deferred call). Logs the first. + template + void NoteWaitWriter(const GpuSplit& split, const Memory& memory, u64 position, + const Outcome& outcome) const { + const u64 target = *outcome.retry_after; + std::string writer; + if (const auto device = memory.Device(outcome.qmd)) { + for (u64 offset = 0; offset < sizeof(Descriptor) && writer.empty(); offset += 4) { + const auto queued = split.QueuedSemaphoreWriteAt(*device + offset); + if (queued && queued->position == target) { + writer = fmt::format("a release into word {}", offset / 4); + } + } + } + if (writer.empty()) { + writer = split.MethodEndingAt(target); + } + if (writer.empty()) { + writer = "another entry"; + } + static u32 logged{}; + if (logged < 16) { + ++logged; + LOG_INFO(HW_GPU, + "QMD view: a signal waits for a write into the QMD at {:#x} by {}, sent {} " + "ring words before it; the back is {} words behind", + outcome.qmd, writer, position - target, split.Backlog()); + } + Nx2AcquireStats::NoteQmdWaitWriter(writer); + } + /// The signal sent at `position` waits, first (`why`: what it waits for) or behind the others. /// None of its releases is noted: they are placed once it is valued. void Wait(const GpuSplit& split, u64 position, const void* memory_id, GPUVAddr address, @@ -237,13 +307,14 @@ class FrontQmdView { std::optional why) { checks_unvalued = true; // TEST if (waiting.size() >= MaxWaiting) { + Nx2AcquireStats::NoteQmdLateLost(Nx2AcquireStats::QmdLost::TooMany, + waiting.size()); // TEST synced = false; ClearWaiting(); Nx2AcquireStats::NoteQmdSignal(Nx2AcquireStats::QmdLost::NotSynced); // TEST return; } - const auto lost = why.value_or(Nx2AcquireStats::QmdLost::NotSynced); // TEST - Nx2AcquireStats::NoteQmdSignal(lost, WhatSent(split, lost)); + Nx2AcquireStats::NoteQmdWait(why, why ? WhatSent(split, *why) : UnplacedKind::Other); WaitingSignal& signal = waiting.emplace_back(); signal.position = position; signal.memory = memory_id; @@ -253,6 +324,14 @@ class FrontQmdView { if (inline_qmd != nullptr) { signal.inline_qmd = *inline_qmd; } + if (checks_pending) { + // TEST: the back keeps what it reads for the signals of this entry from here on. + signal.index = entry_signals; + if (check_wait_position != position) { + check_wait_position = position; + check_first_waiting = entry_signals; + } + } } void ClearWaiting() { @@ -278,12 +357,20 @@ class FrontQmdView { return; } if (!outcome.valued) { + Nx2AcquireStats::NoteQmdLateLost(outcome.lost, waiting.size()); // TEST synced = false; ClearWaiting(); return; } + if (Nx2DrainCheck::g_enabled) { + // TEST: what it read, for the back to compare (CheckLateQmdReads). + late_checks.push_back({first.position, first.index, std::move(late_reads_read)}); + late_reads_read.clear(); + table_check_due = table_check_due || waiting.size() == 1; + } waiting.pop_front(); retry_after = 0; + Nx2AcquireStats::NoteQmdLate(waiting.empty()); // TEST } // Every signal sent before the reads made late is valued: none of their releases waits. late_reads.clear(); @@ -306,6 +393,8 @@ class FrontQmdView { } releases.clear(); attempt_reads.clear(); + late_reads_read.clear(); // TEST + const size_t checked_reads = check_reads.size(); // TEST std::optional failure; const auto read = [&](u64 qmd_address, Descriptor& descriptor) { if (!memory.IsMapped(qmd_address, sizeof(descriptor))) { @@ -327,8 +416,9 @@ class FrontQmdView { } } attempt_reads.push_back(qmd_address); - if (checks_pending && !late) { - check_reads.push_back({qmd_address, descriptor}); // TEST + if (Nx2DrainCheck::g_enabled) { + // TEST: what the back reads too. + (late ? late_reads_read : check_reads).push_back({qmd_address, descriptor}); } return true; }; @@ -347,6 +437,7 @@ class FrontQmdView { } if (failure) { scheduler.RollbackAttempt(); + check_reads.resize(checked_reads); // TEST: tried again later return *failure; } scheduler.CommitAttempt(); @@ -400,7 +491,7 @@ class FrontQmdView { const Lost lost = target == split.LastUnplacedWrite() ? Lost::Unplaced : Lost::Pending; if (target > position) { // Sent after the signal (it waited): memory may hold it, the back read what was before. - return Outcome{false, std::nullopt, Lost::WrittenAfter}; + return Outcome{false, std::nullopt, Lost::WrittenAfter, qmd_address}; } if (split.Ran(target)) { return std::nullopt; @@ -411,7 +502,7 @@ class FrontQmdView { Nx2AcquireStats::NoteQmdInlineRead(); // TEST return std::nullopt; } - return Outcome{false, target, lost}; + return Outcome{false, target, lost, qmd_address}; } /// The last inline QMD sent to the QMD at `device` that the back has not written yet. @@ -530,6 +621,16 @@ class FrontQmdView { std::deque waiting; u64 retry_after{}; std::unordered_set late_reads; + /// TEST (EDEN_NX2_DRAIN_CHECK): the entry the last signal came with and its signals before + /// it; the entry whose signals wait and the first that waits there, for TakeChecks; what the + /// last late attempt read; the signals valued late since TakeChecks; a table copy is due. + u64 entry_position{}; + u32 entry_signals{}; + u64 check_wait_position{}; + u32 check_first_waiting{}; + std::vector late_reads_read; + std::vector late_checks; + bool table_check_due{}; /// The last inline QMDs sent, oldest first from inline_writes_count on: a back a millisecond /// behind has some ten to write, a few times that in long frames. std::array inline_writes{}; diff --git a/src/video_core/gpu_split.cpp b/src/video_core/gpu_split.cpp index 35e8abe626..e6ba2a6d80 100644 --- a/src/video_core/gpu_split.cpp +++ b/src/video_core/gpu_split.cpp @@ -16,6 +16,7 @@ #endif #include #include +#include #include "common/dkb_cpu_trace.h" #include "common/logging.h" @@ -48,6 +49,29 @@ void Pause() { [[maybe_unused]] std::atomic g_back_thread_id{}; +/// TEST: an engine's name, by the class it was bound as (MethodEndingAt). +[[nodiscard]] std::string_view EngineName(std::optional type) { + using Tegra::Engines::EngineTypes; + if (!type) { + return "an unbound engine"; + } + switch (*type) { + case EngineTypes::Nv01Timer: + return "Nv01Timer"; + case EngineTypes::KeplerCompute: + return "KeplerCompute"; + case EngineTypes::Maxwell3D: + return "Maxwell3D"; + case EngineTypes::Fermi2D: + return "Fermi2D"; + case EngineTypes::MaxwellDMA: + return "MaxwellDMA"; + case EngineTypes::KeplerMemory: + return "KeplerMemory"; + } + return "an engine"; +} + } // Anonymous namespace void GpuSplit::CheckOwner(const char* where) { @@ -116,7 +140,8 @@ void GpuSplit::MaybeStart(Core::System& system_) { } system = &system_; check_owner = std::getenv("EDEN_NX2_SPLIT_CHECK") != nullptr; - profile_publishes = Nx2Profile::Enabled(); // TEST + profile_publishes = Nx2Profile::Enabled(); // TEST + log_methods = std::getenv("EDEN_NX2_ACQUIRE_STATS") != nullptr; // TEST ring = std::make_unique(RingWords); back_thread = std::jthread([this](std::stop_token stop_token) { BackLoop(stop_token); }); active = true; @@ -155,6 +180,7 @@ void GpuSplit::Stop() { coarse_sets.fill({}); published_local = 0; publish_held = false; + logged_method_count = 0; consumed_cache = 0; publish_every = PublishEvery; front_engine = nullptr; @@ -503,9 +529,46 @@ void GpuSplit::MultiMethod(Tegra::Engines::EngineInterface* engine, u32 method, amount -= piece; methods_pending -= piece; } + if (log_methods) [[unlikely]] { + LogMethod(engine, method); // TEST + } NoteWrites(writes); } +void GpuSplit::LogMethod(const Tegra::Engines::EngineInterface* engine, u32 method) { + if (!logged_methods) { + logged_methods = std::make_unique(LoggedMethods); + } + logged_methods[logged_method_count++ & (LoggedMethods - 1)] = {write_position, + engine->bound_type, method}; +} + +std::string GpuSplit::MethodEndingAt(u64 end) const { + if (!logged_methods || logged_method_count == 0) { + return {}; + } + // Ends grow with the count: a binary search over the ones kept. + u64 low = logged_method_count > LoggedMethods ? logged_method_count - LoggedMethods : 0; + u64 high = logged_method_count; + while (low < high) { + const u64 middle = low + (high - low) / 2; + if (logged_methods[middle & (LoggedMethods - 1)].end < end) { + low = middle + 1; + } else { + high = middle; + } + } + if (low == logged_method_count) { + return {}; + } + const SentMethod& sent = logged_methods[low & (LoggedMethods - 1)]; + if (sent.end != end) { + return {}; + } + // Not typeid: without RTTI (MSVC's /GR-) it reads type data the engines don't have. + return fmt::format("{} {:#x}", EngineName(sent.engine), sent.method); +} + void GpuSplit::DeferFunction(std::function&& func, Writes writes) { auto* const heap = new std::function(std::move(func)); Defer( diff --git a/src/video_core/gpu_split.h b/src/video_core/gpu_split.h index 97f14c5150..8bd78ca03f 100644 --- a/src/video_core/gpu_split.h +++ b/src/video_core/gpu_split.h @@ -12,6 +12,7 @@ #include #include #include +#include #include #include #include @@ -28,6 +29,7 @@ class MemoryManager; namespace Tegra::Engines { class EngineInterface; +enum class EngineTypes : u32; } namespace VideoCommon { @@ -175,6 +177,9 @@ class GpuSplit { entry[1] = *segment; } Commit(words); + if (log_methods) [[unlikely]] { + LogMethod(engine, method); // TEST + } NoteWrites(writes); } void MultiMethod(Tegra::Engines::EngineInterface* engine, u32 method, const u32* words, @@ -188,6 +193,9 @@ class GpuSplit { entry[0] = static_cast(Kind::Method) | IsLast | Direct | (u64{method & 0xffff} << 8) | (u64{argument} << 32); Commit(1); + if (log_methods) [[unlikely]] { + LogMethod(engine, method); // TEST + } NoteWrites(writes); } @@ -330,6 +338,9 @@ class GpuSplit { u64 device_address) const; /// Whether compute or copy work sent so far is still queued (EDEN_NX2_DRAIN_CHECK). [[nodiscard]] bool WritersPending() const; + /// TEST (EDEN_NX2_ACQUIRE_STATS): the engine and method of the method entry that ends at + /// `end`, among the last LoggedMethods sent, as "KeplerCompute 0x6d"; empty if none. + [[nodiscard]] std::string MethodEndingAt(u64 end) const; /// Front thread: the ring words sent that the back had not run when it last stored its /// position (every 4096 words of a batch and at its end) (EDEN_NX2_LIST_STATS). [[nodiscard]] u64 Backlog() const { @@ -544,6 +555,16 @@ class GpuSplit { /// TEST: Commit publishes before publish_every, because the back waits for the words. void NoteEarlyPublish(); void WaitForSpace(size_t words); + /// TEST (EDEN_NX2_ACQUIRE_STATS): the engine method entries sent, by the position after them. + struct SentMethod { + u64 end; + std::optional engine; ///< The class it was bound as + u32 method; + }; + static constexpr size_t LoggedMethods = size_t{1} << 16; + /// EDEN_NX2_ACQUIRE_STATS as the split started (MaybeStart). + bool log_methods{}; + void LogMethod(const Tegra::Engines::EngineInterface* engine, u32 method); void SelectEngine(Tegra::Engines::EngineInterface* engine) { if (engine != front_engine) [[unlikely]] { SwitchEngine(engine); @@ -610,6 +631,8 @@ class GpuSplit { bool publish_held{}; u64 consumed_cache{}; u64 first_unpublished_ticks{}; ///< TEST: when the first word not published yet was sent + std::unique_ptr logged_methods; ///< TEST: see LogMethod + u64 logged_method_count{}; /// Words between publishes: small right after the front waited for the back (Drain, /// PublishSoon), while the back may have nothing to do. size_t publish_every{PublishEvery}; diff --git a/src/video_core/nx2_drain_check.h b/src/video_core/nx2_drain_check.h index c4df8edd76..efb3326eb7 100644 --- a/src/video_core/nx2_drain_check.h +++ b/src/video_core/nx2_drain_check.h @@ -3,10 +3,12 @@ #pragma once +#include #include #include #include #include +#include #include #include #include @@ -37,8 +39,10 @@ // - the releases the front noted for the QMD launches it valued (FrontQmdView), against the ones // the back then ran, compared on the back right after the QMD methods, and the descriptors the // front read for them (from memory, the signal's inline QMD or a queued one) against the ones -// the back read. Every 512th time, the front's whole QMD table against the back's. Wrong ones -// must stay 0. +// the back read; the descriptors the front read for signals it valued late (they waited) against +// the ones the back read running them, by their position; whether the back could have run an +// entry the front valued when it read memory for it. Every 512th time, and after every signal +// valued late, the front's whole QMD table against the back's. Wrong ones must stay 0. namespace VideoCommon::Nx2DrainCheck { // A plain flag: the front checks it at every command list and acquire. @@ -77,6 +81,15 @@ struct State { std::atomic qmd_reads_logged{}; std::atomic qmd_tables{}; std::atomic qmd_tables_wrong{}; + /// Of the tables, those compared right after the front valued signals late; signals valued + /// late whose descriptors were compared, those the back read otherwise, and those whose back + /// reads were gone (more than LateReadsKept waited); QMD methods the front valued after the + /// back could have run them (their entry published). + std::atomic qmd_tables_late{}; + std::atomic qmd_late_signals{}; + std::atomic qmd_late_wrong{}; + std::atomic qmd_late_missing{}; + std::atomic qmd_published_early{}; std::mutex released_mutex; std::unordered_set released; ///< Device addresses of the semaphores the GPU released std::chrono::steady_clock::time_point last_report{}; ///< The front's @@ -165,6 +178,41 @@ inline std::vector& BackQmdReads() { return reads; } +/// Where one signal's reads end in BackQmdReads (no QMD lives at that address). +inline constexpr u64 SignalEnd = ~u64{0}; + +/// The descriptors the back read for the signals that waited on the front, by the position after +/// their entry and their place in it, until the front valued them late (the back's thread only). +inline std::map, std::vector>& LateBackReads() { + static std::map, std::vector> reads; + return reads; +} +inline constexpr size_t LateReadsKept = 16384; + +/// A signal the front valued late (it waited), at the entry ending at `position`, `index`-th of +/// the entry's signals, and the descriptors it read for it. +struct LateQmdRead { + u64 position; + u32 index; + std::vector reads; +}; + +/// The split's back's scheduling ran one signal (KeplerCompute::ProcessNativeMethod). +inline void NoteQmdSignalEnd(bool on_back) { + if (on_back) { + BackQmdReads().push_back({SignalEnd, {}}); + } +} + +/// The front valued a QMD method after its entry was published: the back could have run it. +inline void NotePublishedEarly() { + State& state = Get(); + state.qmd_published_early.fetch_add(1, std::memory_order_relaxed); + if (state.qmd_reads_logged.fetch_add(1, std::memory_order_relaxed) < 16) { + LOG_ERROR(HW_GPU, "QMD view check: the front valued a QMD method the back could run"); + } +} + /// The split's back runs a release of a QMD launch (KeplerCompute::ProcessNativeMethod). inline void NoteQmdRelease(u64 device_address, u32 value, bool on_back) { if (on_back) { @@ -179,31 +227,109 @@ inline void NoteQmdRead(u64 address, const std::array& words, bool on_b } } -/// The back compared the front's QMD table with its own at the same place (`same`). -inline void NoteQmdTables(bool same, size_t states) { +/// The back compared the front's QMD table with its own at the same place (`same`); `late`: right +/// after the front valued signals late. +inline void NoteQmdTables(bool same, size_t states, bool late = false) { State& state = Get(); state.qmd_tables.fetch_add(1, std::memory_order_relaxed); + if (late) { + state.qmd_tables_late.fetch_add(1, std::memory_order_relaxed); + } if (!same) { state.qmd_tables_wrong.fetch_add(1, std::memory_order_relaxed); if (state.qmd_reads_logged.fetch_add(1, std::memory_order_relaxed) < 16) { - LOG_ERROR(HW_GPU, "QMD view check: the front's table ({} QMDs) differs from the back's", - states); + LOG_ERROR(HW_GPU, + "QMD view check: the front's table ({} QMDs) differs from the back's{}", + states, late ? " after it valued signals late" : ""); + } + } +} + +/// The back, after the signals the front valued late: the descriptors it read for them against +/// the ones it read itself running them. +inline void CheckLateQmdReads(std::span late) { + State& state = Get(); + auto& kept = LateBackReads(); + for (const LateQmdRead& signal : late) { + const auto it = kept.find({signal.position, signal.index}); + if (it == kept.end()) { + state.qmd_late_missing.fetch_add(1, std::memory_order_relaxed); + continue; } + state.qmd_late_signals.fetch_add(1, std::memory_order_relaxed); + const auto& back_reads = it->second; + bool same = back_reads.size() == signal.reads.size(); + for (size_t i = 0; same && i < back_reads.size(); ++i) { + same = back_reads[i].address == signal.reads[i].address && + back_reads[i].words == signal.reads[i].words; + } + if (!same) { + state.qmd_late_wrong.fetch_add(1, std::memory_order_relaxed); + if (state.qmd_reads_logged.fetch_add(1, std::memory_order_relaxed) < 16) { + std::string text; + for (size_t i = 0; i < (std::max)(back_reads.size(), signal.reads.size()); ++i) { + const QmdRead* back = i < back_reads.size() ? &back_reads[i] : nullptr; + const QmdRead* front = i < signal.reads.size() ? &signal.reads[i] : nullptr; + text += fmt::format(" | {:#x} by the back, {:#x} by the front", + back ? back->address : 0, front ? front->address : 0); + for (size_t word = 0; back && front && word < 64; ++word) { + if (back->words[word] != front->words[word]) { + text += fmt::format(", word {} {:#x} for {:#x}", word, + back->words[word], front->words[word]); + break; + } + } + } + LOG_ERROR(HW_GPU, + "QMD view check: the back read other descriptors for a signal the " + "front valued late (position {:#x}, signal {}){}", + signal.position, signal.index, text); + } + } + kept.erase(it); } } /// The back, right after QMD methods the front sent: what it read and released for them against /// what the front read and noted (`front_reads`, `predicted`, in order), or nothing to compare /// where the front could not value them (`valued` false). +/// `wait_position`: the signals of the entry ending there waited on the front from the +/// `first_waiting`-th on (0: none): their reads are kept for CheckLateQmdReads. inline void CheckQmdReleases(std::span predicted, - std::span front_reads, bool valued) { + std::span front_reads, bool valued, + u64 wait_position = 0, u32 first_waiting = 0) { State& state = Get(); auto& released = BackQmdReleases(); - auto& back_reads = BackQmdReads(); + auto& all_reads = BackQmdReads(); + // By signal: each ends with a SignalEnd. + std::vector back_reads; + std::vector> signals(1); + size_t ends = 0; + for (QmdRead& read : all_reads) { + if (read.address == SignalEnd) { + signals.emplace_back(); + ++ends; + continue; + } + back_reads.push_back(read); + signals.back().push_back(std::move(read)); + } + signals.resize(ends); + all_reads.clear(); if (!valued) { state.qmd_unvalued.fetch_add(1, std::memory_order_relaxed); released.clear(); - back_reads.clear(); + auto& kept = LateBackReads(); + if (wait_position != 0) { + // Kept from before the split last started over at position 0. + kept.erase(kept.lower_bound({wait_position, 0}), kept.end()); + } + for (u32 index = first_waiting; wait_position != 0 && index < signals.size(); ++index) { + if (kept.size() >= LateReadsKept) { + kept.erase(kept.begin()); // The oldest: the front lost or dropped it + } + kept.insert_or_assign({wait_position, index}, std::move(signals[index])); + } return; } state.qmd_reads.fetch_add(back_reads.size(), std::memory_order_relaxed); @@ -232,7 +358,6 @@ inline void CheckQmdReleases(std::span predicted, LOG_ERROR(HW_GPU, "QMD view check: the back read other descriptors{}", text); } } - back_reads.clear(); state.qmd_methods.fetch_add(1, std::memory_order_relaxed); state.qmd_releases.fetch_add(released.size(), std::memory_order_relaxed); bool same = released.size() == predicted.size(); @@ -339,6 +464,11 @@ inline void MaybeReport() { const u64 qmd_reads_wrong = take(state.qmd_reads_wrong); const u64 qmd_tables = take(state.qmd_tables); const u64 qmd_tables_wrong = take(state.qmd_tables_wrong); + const u64 qmd_tables_late = take(state.qmd_tables_late); + const u64 qmd_late_signals = take(state.qmd_late_signals); + const u64 qmd_late_wrong = take(state.qmd_late_wrong); + const u64 qmd_late_missing = take(state.qmd_late_missing); + const u64 qmd_published_early = take(state.qmd_published_early); LOG_INFO(HW_GPU, "Drain checks in {:.1f} s: command lists that did not wait for writers sent after the " "last sync {}, GPU-written by those writers {} | drained acquires at addresses the " @@ -346,11 +476,14 @@ inline void MaybeReport() { "at addresses never released {}, at released ones {} | semaphore view: acquires " "passed {}, suspended {}; polls resumed {}, kept waiting {} | QMD view: methods " "valued {} with {} releases, released otherwise than noted {}, with {} descriptors " - "read, read otherwise {}; tables compared {}, differing {}; not valued {}", + "read, read otherwise {}; valued after the back could run them {}; signals valued " + "late {}, read otherwise {}, not compared {}; tables compared {} (after valuing late " + "{}), differing {}; not valued {}", seconds, unwaited, written, acquires_never, acquires_released, polls_never, polls_released, acquire_passes, acquire_waits, poll_passes, poll_waits, qmd_methods, - qmd_releases, qmd_wrong, qmd_reads, qmd_reads_wrong, qmd_tables, qmd_tables_wrong, - qmd_unvalued); + qmd_releases, qmd_wrong, qmd_reads, qmd_reads_wrong, qmd_published_early, + qmd_late_signals, qmd_late_wrong, qmd_late_missing, qmd_tables, qmd_tables_late, + qmd_tables_wrong, qmd_unvalued); } } // namespace VideoCommon::Nx2DrainCheck @@ -454,6 +587,18 @@ struct State { std::array(UnplacedKind::Count)> qmd_lost_mappings{}; std::array(QmdNote::Count)> qmd_notes{}; u64 qmd_inline_reads{}; ///< Descriptors read from an inline QMD still queued + /// Signals that waited first, by why, and behind one; valued late, and how often the last + /// waiting one was; waits that lost the table, by why, and the waiting signals they dropped; + /// waiting signals takeovers dropped (the back had run them). + std::array(QmdLost::Count)> qmd_waits{}; + u64 qmd_behind{}; + u64 qmd_late{}; + u64 qmd_caught_up{}; + std::array(QmdLost::Count)> qmd_late_lost{}; + u64 qmd_late_dropped{}; + u64 qmd_takeover_dropped{}; + /// Of qmd_waits[Pending]: by what wrote the QMD (FrontQmdView::NoteWaitWriter) + std::map qmd_wait_writers; std::array(QmdTakeOver::Count)> qmd_syncs{}; std::chrono::steady_clock::duration qmd_sync_time{}; std::unordered_map noted; ///< Releaser bits by word @@ -506,6 +651,58 @@ inline void NoteQmdSignal(std::optional lost, UnplacedKind kind = Unpla } } +/// A signal the QMD view could not value at once waits: the first by `why` (`kind`: what sent the +/// last write that may write anywhere or change the mappings), or behind one (`why` empty). +inline void NoteQmdWait(std::optional why, UnplacedKind kind = UnplacedKind::Other) { + if (!g_enabled) { + return; + } + State& state = Get(); + if (!why) { + ++state.qmd_behind; + return; + } + ++state.qmd_waits[static_cast(*why)]; + if (*why == QmdLost::Unplaced) { + ++state.qmd_lost_unplaced[static_cast(kind)]; + } else if (*why == QmdLost::Mappings) { + ++state.qmd_lost_mappings[static_cast(kind)]; + } +} + +/// A signal waits for a write into a QMD by `writer`, still queued. +inline void NoteQmdWaitWriter(const std::string& writer) { + if (g_enabled) { + ++Get().qmd_wait_writers[writer]; + } +} + +/// The QMD view valued a waiting signal late; `caught_up`: none waits now. +inline void NoteQmdLate(bool caught_up) { + if (g_enabled) { + State& state = Get(); + ++state.qmd_late; + state.qmd_caught_up += caught_up ? 1 : 0; + } +} + +/// The QMD view lost its table valuing a waiting signal late (`why`), dropping `dropped` waiting +/// signals. +inline void NoteQmdLateLost(QmdLost why, size_t dropped) { + if (g_enabled) { + State& state = Get(); + ++state.qmd_late_lost[static_cast(why)]; + state.qmd_late_dropped += dropped; + } +} + +/// A takeover dropped `dropped` waiting signals: the back had run them. +inline void NoteQmdTakeOverDropped(size_t dropped) { + if (g_enabled) { + Get().qmd_takeover_dropped += dropped; + } +} + /// The QMD view read a descriptor from an inline QMD still queued. inline void NoteQmdInlineRead() { if (g_enabled) { @@ -644,18 +841,52 @@ inline void MaybeReport() { } return text.empty() ? text : text + ")"; }; - LOG_INFO(HW_GPU, - "QMD view in {:.1f} s (EDEN_NX2_ACQUIRE_STATS): signals valued {}; not valued: table " - "not synced yet {}, work that may write anywhere queued {}{}, work that may change " - "the mappings queued {}{}, QMD on a GPU-written page {}, a write into the QMD queued " - "{}, a release of the signal into it {} | descriptors read from an inline QMD still " - "queued {} | releases noted exact {}, at least {}, unknown {} | tables taken over " - "{} in {:.3f} ms (at a front's start {}, between commands {}, at QMD methods {})", - seconds, state.qmd_valued, lost[0], lost[1], kinds(state.qmd_lost_unplaced), lost[5], - kinds(state.qmd_lost_mappings), lost[2], lost[3], lost[4], state.qmd_inline_reads, - notes[0], notes[1], notes[2], syncs[0] + syncs[1] + syncs[2], - std::chrono::duration(state.qmd_sync_time).count(), syncs[0], - syncs[1], syncs[2]); + // The writers a signal waited for most. + std::vector> writers; + for (const auto& [writer, count] : state.qmd_wait_writers) { + writers.emplace_back(count, writer); + } + std::sort(writers.rbegin(), writers.rend()); + std::string by_writer; + for (size_t i = 0; i < writers.size() && i < 6; ++i) { + by_writer += + fmt::format("{}{} {}", i == 0 ? " (by " : ", ", writers[i].second, writers[i].first); + } + if (!by_writer.empty()) { + by_writer += ")"; + } + const auto& waits = state.qmd_waits; + const auto& late_lost = state.qmd_late_lost; + const auto index = [](QmdLost why) { return static_cast(why); }; + u64 waited_first = 0; + u64 lost_late = 0; + for (size_t why = 0; why < waits.size(); ++why) { + waited_first += waits[why]; + lost_late += late_lost[why]; + } + LOG_INFO( + HW_GPU, + "QMD view in {:.1f} s (EDEN_NX2_ACQUIRE_STATS): signals valued {}, valued late {} " + "(caught up {} times) | waited first {} (a write into the QMD queued {}{}, work " + "that may write anywhere queued {}{}, a change of the mappings queued {}{}), behind an " + "earlier one {}; lost while waiting {} (a QMD written after it {}, the mappings " + "changed after it {}, a release into a QMD read late {}, too many waiting {}; {} " + "waiting dropped), dropped at takeovers {} | not valued: table not synced yet {}, " + "QMD on a GPU-written page {}, a release of the signal into it {} | descriptors read " + "from an inline QMD still queued {} | releases noted exact {}, at least {}, unknown " + "{} | tables taken over {} in {:.3f} ms (at a front's start {}, between commands {}, " + "at QMD methods {})", + seconds, state.qmd_valued, state.qmd_late, state.qmd_caught_up, waited_first, + waits[index(QmdLost::Pending)], by_writer, waits[index(QmdLost::Unplaced)], + kinds(state.qmd_lost_unplaced), waits[index(QmdLost::Mappings)], + kinds(state.qmd_lost_mappings), state.qmd_behind, lost_late, + late_lost[index(QmdLost::WrittenAfter)], late_lost[index(QmdLost::MappedAfter)], + late_lost[index(QmdLost::LateReleasedInto)], late_lost[index(QmdLost::TooMany)], + state.qmd_late_dropped, state.qmd_takeover_dropped, lost[index(QmdLost::NotSynced)], + lost[index(QmdLost::GpuWritten)], lost[index(QmdLost::ReleasedInto)], + state.qmd_inline_reads, notes[0], notes[1], notes[2], syncs[0] + syncs[1] + syncs[2], + std::chrono::duration(state.qmd_sync_time).count(), syncs[0], syncs[1], + syncs[2]); auto noted_words = std::move(state.noted); state = State{}; state.noted = std::move(noted_words); From a59f7dc6e41aff348e7d430d30b72cba00dc346b Mon Sep 17 00:00:00 2001 From: jpkto <255363096+jpkto@users.noreply.github.com> Date: Sun, 8 Nov 2026 09:39:22 +0100 Subject: [PATCH 060/105] video_core: let the host GPU keep GPU-written compute list words Ocarina of Time's compute culling writes the group counts of a later dispatch as the three parameters of compute macro 0xe74, in a command list on a GPU-written page. To parse them, the split GPU thread's front waited for that work to run on the back and for the host GPU to copy the page back: about 145 ms of every second ("list copy writers" in the profile), compute lists nearly all of it. The macros only move those words: 0xe74 stores them in the data RAM, 0xe7a uploads them inline as words 1-3 of the dispatch's constant buffer, 0xe76 writes them into the inline QMD as its grid. The CPU needs none of their values. Such a list now stays on the host GPU (Nx2ListIndirect). At the list's place the back copies its words from the buffer cache into a slot of its own (Nx2ListSlots), and the front parses placeholders that name the slot and the word. Before taking a list, the front dry runs the call with the placeholders as tagged values (RunTagged, a copy of the interpreter that follows them): it takes the list only if every tagged value can only be moved, stored in the data RAM or emitted as a word of a linear one-line inline upload it can place or as an inline QMD grid word. A branch, comparison, address, loop count or any other use of one, an emit elsewhere, or a 3D macro that may read a data RAM word holding one, has the front read the words back from their slots first and go on with the words themselves (counted as list copy waits). Other lists, and calls the dry run refuses, take the old path. On the back, a QMD whose grid is one slot's three placeholders is dispatched indirectly from the slot (width, height and depth as 16 bits, as DispatchNativeCompute takes them), and an uploaded placeholder still in guest memory is an owed word: compute dispatches that bind it as constant buffer data get the slot word copied over it. Slots are held while a holder, an owed word or an inline QMD grid names them. All scheduling of the QMDs on the CPU is unchanged: the placeholders are never zero where the scheduler looks, and the releases do not depend on the grid. What differs from the old path, all on the CPU only: guest memory holds the placeholders where the upload and the QMD put the words, a dispatch whose real grid is empty still records an indirect dispatch of no groups, and a width with bit 31 set is not masked. The path is off for devices that dispatch fewer than 2^31-1 by 65535 by 65535 groups, and with EDEN_NX2_LIST_INDIRECT=0. AnalyzeEffects now also gives the data RAM words and registers a macro can read. The stream buffer is a transfer destination too, for the copies over constant buffer placeholders. Tested: the tagged dry run against the interpreter on Ocarina of Time's logged macro code and calls and on 100k random programs (212633 checks), the front's list handling with a mock engine against the old path for six lists of 0xe74, 0xe7a, 0xe84, 0xe76 and 0xe8c plus the escapes and refusals (158 checks, mutations caught), and the slots on lavapipe under synchronization validation: 40 frames with slot reuse in and across submissions, a culling stand-in written again before the launch, owed words patched over a streamed constant buffer, no hazard reported, the launched shader seeing the frame's words and group counts; five controls without the barriers report hazards. The list dump, entry check and macro flow harnesses, the split harness (10 runs, and 10 on one CPU), the front QMD view harness in modes 1-4 with EDEN_NX2_DRAIN_CHECK and EDEN_NX2_ACQUIRE_STATS, syntax checks of video_core and strict warnings on the changed lines pass. Rebased onto upstream: EDEN_NX2_LIST_INDIRECT is a plain flag read again by VideoCore::CreateGPU, as upstream reads its diagnostic switches since #714, so EDEN_NX2_LIST_INDIRECT=0 works from nx2_switches.txt too, not only from the environment. The EDEN_NX2_LIST_DUMP diagnostic it was written next to is not part of this series. Co-Authored-By: Claude Opus 5.5 --- src/video_core/CMakeLists.txt | 3 + src/video_core/buffer_cache/buffer_cache.h | 11 + src/video_core/dma_pusher.cpp | 57 + src/video_core/dma_pusher.h | 5 + src/video_core/engines/kepler_compute.cpp | 185 ++- src/video_core/engines/kepler_compute.h | 17 + src/video_core/engines/maxwell_3d.cpp | 41 + src/video_core/macro_turing.h | 31 +- src/video_core/nx2_list_indirect.h | 1118 +++++++++++++++++ src/video_core/rasterizer_interface.h | 29 + .../renderer_vulkan/nx2_list_slots.cpp | 196 +++ .../renderer_vulkan/nx2_list_slots.h | 157 +++ .../renderer_vulkan/vk_buffer_cache.h | 19 + .../renderer_vulkan/vk_rasterizer.cpp | 85 +- .../renderer_vulkan/vk_rasterizer.h | 7 + .../vk_staging_buffer_pool.cpp | 7 +- src/video_core/video_core.cpp | 2 + 17 files changed, 1960 insertions(+), 10 deletions(-) create mode 100644 src/video_core/nx2_list_indirect.h create mode 100644 src/video_core/renderer_vulkan/nx2_list_slots.cpp create mode 100644 src/video_core/renderer_vulkan/nx2_list_slots.h diff --git a/src/video_core/CMakeLists.txt b/src/video_core/CMakeLists.txt index 6e3306933f..71c1a9c2fa 100644 --- a/src/video_core/CMakeLists.txt +++ b/src/video_core/CMakeLists.txt @@ -122,6 +122,7 @@ add_library(video_core STATIC nx2_debug_control.h nx2_drain_check.h nx2_entry_check.h + nx2_list_indirect.h nx2_list_stats.h nx2_profile.h nx2_perf.h @@ -177,6 +178,8 @@ add_library(video_core STATIC renderer_vulkan/framebuffer_limits.h renderer_vulkan/maxwell_to_vk.cpp renderer_vulkan/maxwell_to_vk.h + renderer_vulkan/nx2_list_slots.cpp + renderer_vulkan/nx2_list_slots.h renderer_vulkan/pipeline_build.h renderer_vulkan/pipeline_build_trace.h renderer_vulkan/pipeline_helper.h diff --git a/src/video_core/buffer_cache/buffer_cache.h b/src/video_core/buffer_cache/buffer_cache.h index 8b471c3339..b2af92ff81 100644 --- a/src/video_core/buffer_cache/buffer_cache.h +++ b/src/video_core/buffer_cache/buffer_cache.h @@ -2830,11 +2830,16 @@ void BufferCache

::BindHostComputeUniformBuffers() { needs_alignment_stream || (has_host_buffer && size <= channel_state->uniform_buffer_skip_cache_size && !memory_tracker.IsRegionGpuModified(binding.device_addr, size)); + // GPU-written list words owed there go over their placeholders (Nx2ListIndirect). + const auto guest_word = [this](DAddr word) { return device_memory.Read(word); }; if constexpr (!IS_OPENGL) { if (use_fast_buffer) { const std::span span = runtime.BindMappedUniformBuffer(0, binding_index, size); device_memory.ReadBlockUnsafe(binding.device_addr, span.data(), size); + const auto stream = runtime.LastStreamUniformRef(); + runtime.Nx2PatchComputeUniform(binding.device_addr, size, stream.buffer, + stream.offset, guest_word); return; } } @@ -2842,6 +2847,12 @@ void BufferCache

::BindHostComputeUniformBuffers() { SynchronizeBuffer(buffer, binding.device_addr, size); buffer.MarkUsage(offset, size); + if constexpr (!IS_OPENGL) { + runtime.Nx2PatchComputeUniform(binding.device_addr, size, buffer.Handle(), offset, + guest_word); + } else { + (void)guest_word; + } if constexpr (NEEDS_BIND_UNIFORM_INDEX) { runtime.BindComputeUniformBuffer(binding_index, buffer, offset, size); ++binding_index; diff --git a/src/video_core/dma_pusher.cpp b/src/video_core/dma_pusher.cpp index 71aabd98be..ec0022ef9b 100644 --- a/src/video_core/dma_pusher.cpp +++ b/src/video_core/dma_pusher.cpp @@ -35,6 +35,7 @@ #include "video_core/nx2_cmdpage_diag.h" #include "video_core/nx2_drain_check.h" #include "video_core/nx2_entry_check.h" +#include "video_core/nx2_list_indirect.h" #include "video_core/nx2_list_stats.h" #include "video_core/rasterizer_interface.h" @@ -338,6 +339,12 @@ bool DmaPusher::Step() { Nx2CmdPageDiag::CountEntry(Nx2CmdPageDiag::Path::Safe); command_resume_offset += ProcessCommands(commands.subspan(command_resume_offset), command_resume_offset); + } else if (gpu_written && TakeGpuWrittenListWords(header)) { + // The words stay on the host GPU: the call takes their placeholders (not recorded + // for EDEN_NX2_PB_CHECK, which would find the entry's memory changed). + const std::span commands{command_headers.data(), header.size}; + command_resume_offset += + ProcessCommands(commands.subspan(command_resume_offset), command_resume_offset); } else if (ReadGpuWrittenCommands(dma_state.dma_get, static_cast(header.size))) { const std::span commands{command_headers.data(), header.size}; RecordParsedEntry(header, commands); @@ -914,6 +921,56 @@ bool DmaPusher::ReadGpuWrittenCommands(GPUVAddr address, u32 count) { return true; } +bool DmaPusher::TakeGpuWrittenListWords(const CommandListHeader& header) { + // Ocarina of Time's compute culling writes the group counts of a later dispatch as the + // parameters of compute macro 0xe74, which only stores them in the data RAM for 0xe7a and + // 0xe76 to upload and launch with. The front need not wait for the culling to run and its + // copy to come back: the macros run on placeholders the host GPU's copy of the words takes + // the place of (Nx2ListIndirect, KeplerCompute::FrontTakeListWords). + auto& split = VideoCommon::GpuSplit::Get(); + if (!Nx2ListIndirect::g_enabled || rasterizer == nullptr || command_resume_offset != 0 || + !split.Active() || !split.FrontMacros()) { + return false; + } + const u32 count = static_cast(header.size); + // All of a macro call's parameters, from its first: its header ended the entry before. + if (count == 0 || count > Nx2ListIndirect::SlotWords || dma_state.method_count != count || + dma_state.method < MacroRegistersStart || (dma_state.method & 1) != 0 || + subchannels[dma_state.subchannel] == nullptr || + subchannel_type[dma_state.subchannel] != Engines::EngineTypes::KeplerCompute) { + return false; + } + auto* const compute = static_cast(subchannels[dma_state.subchannel]); + if (!compute->FrontRunsMacros() || !rasterizer->Nx2ListSlotsSupported()) { + return false; + } + // On one device page, as the back finds it: its mappings are the front's once it ran the + // last change sent. + const u64 bytes = u64{count} * sizeof(u32); + const auto device = memory_manager.GpuToCpuAddress(dma_state.dma_get); + const auto last = memory_manager.GpuToCpuAddress(dma_state.dma_get + bytes - 1); + if (!device || !last || *last != *device + bytes - 1 || !split.Ran(split.LastMappingChange())) { + return false; + } + const std::optional slot = rasterizer->Nx2AcquireListSlot(); + if (!slot) { + return false; + } + if (!compute->FrontTakeListWords(split, dma_state.method, count, *slot)) { + rasterizer->Nx2ReleaseListSlot(*slot); // Nothing sent with it + return false; + } + // At the list's place: after the work sent before it, the GPU fetched the list there. + split.Defer([rasterizer_ = rasterizer, device_ = *device, count, + slot_ = *slot] { rasterizer_->Nx2SnapshotListWords(device_, count, slot_); }, + VideoCommon::GpuSplit::Writes::Noted); + command_headers.resize_destructive(count); + for (u32 k = 0; k < count; ++k) { + command_headers[k].argument = Nx2ListIndirect::Placeholder(*slot, k); + } + return true; +} + void DmaPusher::VerifyGpuWrittenCommands(GPUVAddr address, u32 count) { // The list again as the front read lists before it tracked the words GPU work wrote: after // every dispatch and copy sent so far, from the pages' whole copies, or drained and flushed. diff --git a/src/video_core/dma_pusher.h b/src/video_core/dma_pusher.h index a4416486f9..e551f5e0df 100644 --- a/src/video_core/dma_pusher.h +++ b/src/video_core/dma_pusher.h @@ -247,6 +247,11 @@ class DmaPusher final { /// since. False when not all are current (or the list isn't GPU-written): the caller drains, /// flushes and reads guest memory. bool ReadGpuWrittenCommands(GPUVAddr address, u32 count); + /// A GPU-written list that is all the parameters of the compute macro call about to start, + /// on the split with macros on its front: if the call only moves the words where the back can + /// take them over (Nx2ListIndirect), puts their placeholders in command_headers instead of + /// reading them. The back copies the words at the list's place on the host GPU. + bool TakeGpuWrittenListWords(const CommandListHeader& header); /// EDEN_NX2_LIST_VERIFY: reads the list ReadGpuWrittenCommands just read again as before /// (Nx2ListStats) and leaves the words read last in command_headers. void VerifyGpuWrittenCommands(GPUVAddr address, u32 count); diff --git a/src/video_core/engines/kepler_compute.cpp b/src/video_core/engines/kepler_compute.cpp index 99291b344d..18ec0dc77d 100644 --- a/src/video_core/engines/kepler_compute.cpp +++ b/src/video_core/engines/kepler_compute.cpp @@ -18,6 +18,7 @@ #include "common/assert.h" #include "common/dkb_frame_trace.h" #include "common/logging.h" +#include "common/scope_exit.h" #include "common/settings.h" #include "shader_recompiler/exception.h" #include "core/core.h" @@ -27,6 +28,7 @@ #include "video_core/nx2_profile.h" #include "video_core/nx2.h" #include "video_core/nx2_drain_check.h" +#include "video_core/nx2_list_indirect.h" #include "video_core/engines/fermi_2d.h" #include "video_core/engines/kepler_compute.h" #include "video_core/engines/native_upscale.h" @@ -80,7 +82,11 @@ KeplerCompute::KeplerCompute(MemoryManager& memory_manager_) } } -KeplerCompute::~KeplerCompute() = default; +KeplerCompute::~KeplerCompute() { + if (native_data_ram != nullptr) { + Nx2ListIndirect::ForgetChannel(native_data_ram); + } +} void KeplerCompute::BindRasterizer(VideoCore::RasterizerInterface* rasterizer_) { rasterizer = rasterizer_; @@ -309,6 +315,12 @@ bool KeplerCompute::ProcessNativeMethod(Core::System& system, u32 method, u32 ar } rasterizer->AccelerateInlineToMemory(address, 256, std::span{reinterpret_cast(native_registers.data() + 0xc8), 256}); + if (const auto device = memory_manager.GpuToCpuAddress(address)) { + // The slots the grid's list word placeholders name stay for its launches. + rasterizer->Nx2NoteInlineQmdGrid(*device, native_registers[0xc8 + 12], + native_registers[0xc8 + 13], + native_registers[0xc8 + 14]); + } if ((control & (1U << 24)) || !(native_registers[0xcc] & (1U << 12))) { const u32 previous = native_registers[0xad]; native_registers[0xad] = static_cast(address >> 8); @@ -560,7 +572,37 @@ void KeplerCompute::CallMultiMethod(Core::System& system, u32 method, const u32* } } -bool KeplerCompute::DispatchNativeCompute(const NativeQmdScheduler::Descriptor& qmd) { +bool KeplerCompute::DispatchNativeCompute(const NativeQmdScheduler::Descriptor& descriptor) { + // A grid of GPU-written list words (Nx2ListIndirect): one slot's three words, which the host + // GPU dispatches from, or placeholders of several, whose words the back reads. + NativeQmdScheduler::Descriptor resolved; + const bool listed = Nx2ListIndirect::IsPlaceholder(descriptor[12]) || + Nx2ListIndirect::IsPlaceholder(descriptor[13]) || + Nx2ListIndirect::IsPlaceholder(descriptor[14]); + if (listed) [[unlikely]] { + indirect_list_slot = + Nx2ListIndirect::GridSlot(descriptor[12], descriptor[13], descriptor[14]); + if (!indirect_list_slot) { + resolved = descriptor; + for (const u32 word : {12U, 13U, 14U}) { + if (Nx2ListIndirect::IsPlaceholder(resolved[word])) { + const u32 slot = Nx2ListIndirect::PlaceholderSlot(resolved[word]); + std::array words{}; + rasterizer->Nx2ReadListSlots(std::span{&slot, 1}, words); + resolved[word] = words[Nx2ListIndirect::PlaceholderWord(resolved[word])]; + } + } + // An empty grid dispatches nothing (NativeQmdScheduler::Signal does not call this). + if (!(resolved[12] && (resolved[13] & 0xffff) && (resolved[14] & 0xffff))) { + return true; + } + } + } + SCOPE_EXIT { + indirect_list_slot.reset(); + }; + const NativeQmdScheduler::Descriptor& qmd = + listed && !indirect_list_slot ? resolved : descriptor; launch_description = {}; launch_description.grid_dim_x.Assign(qmd[12]); launch_description.grid_dim_y.Assign(qmd[13] & 0xffff); @@ -1525,6 +1567,62 @@ struct FrontQmdMemory { using QmdTakeOver = VideoCommon::Nx2AcquireStats::QmdTakeOver; // TEST +// A compute macro call as Nx2ListIndirect::ComputeFront sees it (its Context). +template +struct ListCallContext { + std::span code; + u64 generation; + u32 start; + std::vector& params; + std::span data_ram; + std::span registers; + Nx2ListIndirect::UploadState upload; + VideoCommon::GpuSplit& split; + const MemoryManager& memory; + VideoCore::RasterizerInterface* rasterizer; + EngineInterface* engine; + std::array& front_registers; + std::array& engine_registers; ///< Set directly once the split stopped + const EmitFunc& emit; + + [[nodiscard]] std::optional DeviceWord(u64 gpu_address) const { + if (!split.Ran(split.LastMappingChange())) { + return std::nullopt; // The back may find other mappings + } + const auto first = memory.GpuToCpuAddress(gpu_address); + const auto last = memory.GpuToCpuAddress(gpu_address + 3); + if (!first || !last || *last != *first + 3) { + return std::nullopt; + } + return u64{*first}; + } + void Emit(u32 method, u32 value) const { + emit(method, value); + } + void SendOwed(u64 device, u32 value) const { + split.Defer([rasterizer_ = rasterizer, device, + value] { rasterizer_->Nx2NoteOwedListWord(device, value); }, + VideoCommon::GpuSplit::Writes::Noted); + } + void Release(u32 slot) const { + Nx2ListIndirect::ReleaseSlot(split, rasterizer, slot); + } + void ReadSlots(std::span slots, std::span words) const { + Nx2ListIndirect::ReadSlots(split, rasterizer, slots, words); + } + void SetRegister(u32 method, u32 value) const { + front_registers[method] = value; + if (method == Nx2ListIndirect::UploadData) { + return; // The upload's data port: only macros read it back, from the front's copy + } + if (split.Active()) { + split.MacroWrite(engine, method, value, VideoCommon::GpuSplit::Writes::Noted); + } else { + engine_registers[method] = value; + } + } +}; + } // Anonymous namespace // Split GPU thread with macros on its front half (gpu_split.h, EDEN_NX2_SPLIT=2): the front @@ -1538,6 +1636,8 @@ struct KeplerCompute::FrontState { bool compute_uploaded{}; u32 bind_pointer{}; MacroEngine* macro{}; ///< Without a native context: the one macro engine + /// The inline upload the back runs, where a word of it lands (Nx2ListIndirect). + Nx2ListIndirect::UploadState upload; }; KeplerCompute::FrontState& KeplerCompute::Front(VideoCommon::GpuSplit& split) { @@ -1563,6 +1663,25 @@ KeplerCompute::FrontState& KeplerCompute::Front(VideoCommon::GpuSplit& split) { // As Maxwell3D::ReleaseFront: the split stopped, the engine runs its macros again. void KeplerCompute::ReleaseFront() { + if (list_front && native_data_ram != nullptr) { + // They run on the GPU-written list words themselves from now on (Nx2ListIndirect). + const auto no_emit = [](u32, u32) { return false; }; + ListCallContext context{{}, + 0, + 0, + front->parameters, + std::span{*native_data_ram}, + front->registers, + front->upload, + VideoCommon::GpuSplit::Get(), + memory_manager, + rasterizer, + this, + front->registers, + native_registers, + no_emit}; + list_front->ConcretizeAll(context, Nx2ListIndirect::GetChannel(native_data_ram)); + } native_executing_macro = front->executing_macro; native_parameters = std::move(front->parameters); native_compute_uploaded = front->compute_uploaded; @@ -1580,6 +1699,8 @@ void KeplerCompute::FrontApply(FrontState& fs, u32 method, u32 argument) { if (method == 0x45 || method == 0x47) { fs.compute_uploaded = true; } + Nx2ListIndirect::NoteUploadMethod(fs.upload, method, argument, + [&fs](u32 reg) { return fs.registers[reg]; }); fs.registers[method] = argument; switch (method) { case 0x45: @@ -1833,6 +1954,41 @@ void KeplerCompute::FrontCallMultiMethod(Core::System& system, VideoCommon::GpuS FrontQmdCheck(split); } +bool KeplerCompute::FrontTakeListWords(VideoCommon::GpuSplit& split, u32 method, u32 count, + u32 slot) { + if (!native_mode || native_data_ram == nullptr) { + return false; + } + auto& fs = Front(split); + if (fs.executing_macro || !fs.parameters.empty()) { + return false; // Not a call's first parameter + } + if (!list_front) { + list_front = std::make_unique(); + } + const auto positions = FrontMacroPositions(fs); + const u32 start = positions[((method - 0xe00) / 2) % positions.size()]; + const auto& macro = FrontMacroEngine(fs); + const auto no_emit = [](u32, u32) { return false; }; + std::vector no_params; + ListCallContext context{macro.native_code, + macro.native_code_generation, + start, + no_params, + std::span{*native_data_ram}, + fs.registers, + fs.upload, + split, + memory_manager, + rasterizer, + this, + fs.registers, + native_registers, + no_emit}; + return list_front->PrepareCall(context, Nx2ListIndirect::GetChannel(native_data_ram), slot, + count); +} + void KeplerCompute::FrontRunMacro(Core::System& system, VideoCommon::GpuSplit& split, FrontState& fs) { const auto positions = FrontMacroPositions(fs); @@ -1862,6 +2018,31 @@ void KeplerCompute::FrontRunMacro(Core::System& system, VideoCommon::GpuSplit& s FrontQmdCheck(split); return true; }; + if (list_front && native_data_ram != nullptr) { + // GPU-written list words among the call's parameters or within its reach: a tagged run + // that leaves them to the back, or the words themselves first (Nx2ListIndirect). + ListCallContext context{macro.native_code, + macro.native_code_generation, + start, + fs.parameters, + std::span{*native_data_ram}, + fs.registers, + fs.upload, + split, + memory_manager, + rasterizer, + this, + fs.registers, + native_registers, + emit}; + bool concretized{}; + Nx2ListIndirect::Reason reason{}; + if (list_front->Run(context, Nx2ListIndirect::GetChannel(native_data_ram), concretized, + reason)) { + fs.parameters.clear(); + return; + } + } #ifdef ARCHITECTURE_x86_64 if (!macro.native_jit) { macro.native_jit = std::make_shared(); diff --git a/src/video_core/engines/kepler_compute.h b/src/video_core/engines/kepler_compute.h index 3287b81541..68d3292008 100644 --- a/src/video_core/engines/kepler_compute.h +++ b/src/video_core/engines/kepler_compute.h @@ -37,6 +37,10 @@ namespace Tegra { class MemoryManager; } +namespace Tegra::Nx2ListIndirect { +class ComputeFront; +} + namespace VideoCore { class RasterizerInterface; } @@ -251,6 +255,12 @@ class KeplerCompute final : public EngineInterface { return indirect_compute; } + /// The slot of GPU-written list words (Nx2ListIndirect) the dispatch being made takes its + /// group counts from, indirectly: its QMD's grid is the slot's placeholders. + std::optional GetIndirectListSlot() const { + return indirect_list_slot; + } + bool IsNativeMode() const { return native_mode; } bool ProcessGameDlss(u64 program_id, u64 native_key); /// Ocarina of Time with anti-aliasing at No sharpening (oot_network.h): true when this @@ -268,6 +278,10 @@ class KeplerCompute final : public EngineInterface { void FrontCallMultiMethod(Core::System& system, VideoCommon::GpuSplit& split, u32 method, const u32* base_start, u32 amount, u32 methods_pending, const GPUVAddr* segment); + /// The front, at a GPU-written list of `count` words that are all the parameters of the call + /// of macro method `method` about to start: whether the call takes the words as the + /// placeholders of `slot` (Nx2ListIndirect), the back's copy of them, instead of the words. + bool FrontTakeListWords(VideoCommon::GpuSplit& split, u32 method, u32 count, u32 slot); private: struct FrontState; @@ -391,6 +405,9 @@ class KeplerCompute final : public EngineInterface { }; std::vector uploads; std::optional indirect_compute{}; + std::optional indirect_list_slot{}; + /// The front's GPU-written list words in this engine's macros (Nx2ListIndirect). + std::unique_ptr list_front; }; #define ASSERT_REG_POSITION(field_name, position) \ diff --git a/src/video_core/engines/maxwell_3d.cpp b/src/video_core/engines/maxwell_3d.cpp index f6d1a54ae9..ae816214e9 100644 --- a/src/video_core/engines/maxwell_3d.cpp +++ b/src/video_core/engines/maxwell_3d.cpp @@ -30,6 +30,7 @@ #include "video_core/nx2.h" #include "video_core/nx2_cmdpage_diag.h" #include "video_core/nx2_drain_check.h" +#include "video_core/nx2_list_indirect.h" #include "video_core/nx2_perf.h" #include "video_core/nx2_profile.h" #include "video_core/query_cache/frame_trace.h" @@ -1300,6 +1301,30 @@ std::string DescribeWrites(std::span> writes) { return writes.size() > 12 ? text + ",..." : text; } +// GPU-written list words the channel's compute macros left in its data RAM (Nx2ListIndirect): +// the front runs 3D macros on the words themselves, so a macro at `start` that may read one +// gets them first, read back from their slots. +void ConcretizeListWords(VideoCommon::GpuSplit& split, VideoCore::RasterizerInterface* rasterizer, + Nx2ListIndirect::DataRamWords& channel, std::span data_ram, + const MacroEngine& engine, u32 start) { + const auto& reads = + channel.reads_3d.Get(engine.native_code, engine.native_code_generation, start); + if (!Nx2ListIndirect::MayRead(reads.data_ram_known, reads.data_ram, channel.words)) { + return; + } + const auto release = [&](u32 slot) { Nx2ListIndirect::ReleaseSlot(split, rasterizer, slot); }; + Nx2ListIndirect::Prune(channel.words, [data_ram](u32 word) { return data_ram[word]; }, release); + if (!Nx2ListIndirect::MayRead(reads.data_ram_known, reads.data_ram, channel.words)) { + return; + } + Nx2ListIndirect::ConcretizeDataRam( + channel.words, data_ram, + [&](std::span slots, std::span words) { + Nx2ListIndirect::ReadSlots(split, rasterizer, slots, words); + }, + release); +} + } // Anonymous namespace Maxwell3D::FrontState& Maxwell3D::Front(Core::System& system, VideoCommon::GpuSplit& split) { @@ -1326,6 +1351,18 @@ Maxwell3D::FrontState& Maxwell3D::Front(Core::System& system, VideoCommon::GpuSp // split stopped (accuracy settings changed mid-game). Hand the macros back: the front's copy // goes stale from now on, and a restart copies the registers again (Front). void Maxwell3D::ReleaseFront() { + // The macros run on the engine from now on: the data RAM gets the GPU-written list words + // its words hold placeholders of (Nx2ListIndirect). + if (auto* channel = Nx2ListIndirect::FindChannel(&native_data_ram); + channel != nullptr && !channel->words.empty()) { + auto& split = VideoCommon::GpuSplit::Get(); + Nx2ListIndirect::ConcretizeDataRam( + channel->words, std::span{native_data_ram}, + [&](std::span slots, std::span words) { + Nx2ListIndirect::ReadSlots(split, rasterizer, slots, words); + }, + [&](u32 slot) { Nx2ListIndirect::ReleaseSlot(split, rasterizer, slot); }); + } executing_macro = front->executing_macro; macro_params = std::move(front->parameters); // Entries macros the back ran may have written are the engine's already. @@ -1579,6 +1616,10 @@ void Maxwell3D::FrontRunMacro(Core::System& system, VideoCommon::GpuSplit& split VideoCommon::Nx2Profile::CountMethod(method, false); if (!FrontRunOnBack(system, split, fs, engine, start)) { VideoCommon::Nx2Profile::Scope profile_scope{VideoCommon::Nx2Profile::Zone::Macro}; + if (auto* channel = Nx2ListIndirect::FindChannel(&native_data_ram); + channel != nullptr && !channel->words.empty()) { + ConcretizeListWords(split, rasterizer, *channel, native_data_ram, engine, start); + } engine.native_interpreter.active_dram = &native_data_ram; const auto parameter_count = fs.parameters.size(); const auto read = [&](u32 address) { diff --git a/src/video_core/macro_turing.h b/src/video_core/macro_turing.h index 56af4e9dbe..7223512a17 100644 --- a/src/video_core/macro_turing.h +++ b/src/video_core/macro_turing.h @@ -261,6 +261,12 @@ class TuringMacroInterpreter { std::vector methods; ///< Ascending, each once bool methods_known{true}; bool data_ram{}; + /// The data RAM words DREAD can read and the registers MME_READ can read, ascending, each + /// once; not all of them where `..._known` is false (an address the code computes). + std::vector data_ram_reads; + bool data_ram_reads_known{true}; + std::vector register_reads; + bool register_reads_known{true}; }; static Effects AnalyzeEffects(std::span code, Word start) { @@ -420,7 +426,13 @@ class TuringMacroInterpreter { case 17: case 18: case 19: case 20: case 21: value = both ? known(taken(d.op[lane]) ? ~0U : 0) : unknown; break; - case 22: value = unknown; break; // A register + case 22: // A register + value = unknown; + if (both) + effects.register_reads.push_back((a + b) & 0xffff); + else + effects.register_reads_known = false; + break; case 23: if (imm < 2) break; // Entered, or skipped with a count of 0. Its last instruction may start @@ -453,7 +465,14 @@ class TuringMacroInterpreter { if (!both || !jumps) flow[flow_count++] = {Flow::Next, 0, false}; break; } - case 30: effects.data_ram = true; value = unknown; break; + case 30: + effects.data_ram = true; + value = unknown; + if (x.known) + effects.data_ram_reads.push_back(a); + else + effects.data_ram_reads_known = false; + break; case 31: effects.data_ram = true; break; default: break; // 7: the load barrier } @@ -513,9 +532,11 @@ class TuringMacroInterpreter { } } } - std::ranges::sort(effects.methods); - effects.methods.erase(std::unique(effects.methods.begin(), effects.methods.end()), - effects.methods.end()); + for (std::vector* list : + {&effects.methods, &effects.data_ram_reads, &effects.register_reads}) { + std::ranges::sort(*list); + list->erase(std::unique(list->begin(), list->end()), list->end()); + } return effects; } diff --git a/src/video_core/nx2_list_indirect.h b/src/video_core/nx2_list_indirect.h new file mode 100644 index 0000000000..784524e7f9 --- /dev/null +++ b/src/video_core/nx2_list_indirect.h @@ -0,0 +1,1118 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "common/common_types.h" +#include "video_core/gpu_split.h" +#include "video_core/macro_turing.h" + +// GPU-written command lists whose words the host GPU takes over (EDEN_NX2_LIST_INDIRECT, on +// unless =0). Ocarina of Time's compute culling writes the group counts of a later dispatch as +// the parameters of a macro call (compute method 0xe74): the split GPU thread's front had to +// wait for that work to run on the host GPU and its copy to come back before it could parse +// them (~145 ms of every second). The macros only move those words: 0xe74 stores them in the +// data RAM, 0xe7a uploads them with the dispatch's constant buffer (an inline upload), 0xe76 +// writes them into the inline QMD as its grid. So the front need not know them: +// - At the list's place, the back copies the words on the host GPU into a slot of its own (the +// snapshot: the list may be written again before the QMD launches). +// - The front parses placeholders for them, values that name the slot and the word, and runs the +// macros that see them in a tagged dry run (RunTagged): a tagged value may be moved, stored in +// the data RAM or emitted as an inline upload's data or as an inline QMD's grid, nothing else +// (no branch, computation or address depends on it). Any other use, and any 3D macro that +// may read a tagged data RAM word, concretizes the words first: the front reads the slot back +// and goes on with the real words, as before. +// - The back dispatches a QMD whose grid words are one slot's placeholders indirectly from the +// slot, and patches the uploaded words into the constant buffers compute dispatches bind +// (the "owed" words), from the slot. +namespace Tegra::Nx2ListIndirect { + +[[nodiscard]] inline bool SwitchSet() { + const char* value = std::getenv("EDEN_NX2_LIST_INDIRECT"); + return value == nullptr || value[0] != '0'; +} + +// A plain flag: checked at every GPU-written list. +inline bool g_enabled = SwitchSet(); + +/// Called by VideoCore::CreateGPU: static initialization runs before the frontend loads +/// nx2_switches.txt, so a switch set only there takes effect from that point on. +inline void ReadSwitch() { + g_enabled = SwitchSet(); +} + +/// Slots the back keeps snapshots of list words in, and the words a slot holds. +inline constexpr u32 SlotCount = 4096; +inline constexpr u32 SlotWords = 3; + +/// The value the front carries for word `word` of the list in `slot`. A real grid never has a +/// width this large (it is over 4 billion groups). Word 0 is never 0 and the low 16 bits of words +/// 1 and 2 never are, so the QMD scheduler's empty grid check never drops a dispatch whose grid +/// is placeholders (the host GPU dispatches no groups for an empty one). +inline constexpr u32 PlaceholderMagic = 0xF1A90000U; +static_assert((SlotCount << 2) <= 0x10000); + +[[nodiscard]] constexpr u32 Placeholder(u32 slot, u32 word) { + return PlaceholderMagic | (slot << 2) | word; +} +[[nodiscard]] constexpr bool IsPlaceholder(u32 value) { + return (value & 0xFFFF0000U) == PlaceholderMagic && (value & 3) < SlotWords; +} +[[nodiscard]] constexpr u32 PlaceholderSlot(u32 value) { + return (value & 0xFFFFU) >> 2; +} +[[nodiscard]] constexpr u32 PlaceholderWord(u32 value) { + return value & 3; +} +static_assert(Placeholder(0, 0) != 0 && (Placeholder(0, 1) & 0xFFFF) != 0 && + (Placeholder(0, 2) & 0xFFFF) != 0); + +/// The slot whose three words a QMD's grid (words 12-14) is, if it is one. +[[nodiscard]] constexpr std::optional GridSlot(u32 x, u32 y, u32 z) { + if (!IsPlaceholder(x) || x != Placeholder(PlaceholderSlot(x), 0)) { + return std::nullopt; + } + const u32 slot = PlaceholderSlot(x); + if (y != Placeholder(slot, 1) || z != Placeholder(slot, 2)) { + return std::nullopt; + } + return slot; +} + +/// What a tagged dry run wrote, in order: emits (method, value) and data RAM stores (word, +/// value), `tagged` where the value is a list word moved there unchanged. +struct TaggedWrite { + u32 address; + u32 value; + bool tagged; +}; + +struct TaggedRun { + enum class Outcome { + Done, ///< Ran to its end: emits and data_ram are what Execute does + Escaped, ///< A tagged value reached something other than a move, a store or an emit + Failed, ///< The call fails (as Execute fails it): run it the old way + }; + Outcome status{}; + std::string_view why; + std::vector emits; + std::vector data_ram; + size_t consumed{}; +}; + +/// A dry run of a macro call, as TuringMacroInterpreter::Execute runs it, that follows the list +/// words through it: `params` (the load FIFO) with `params_tagged`, the data RAM words as +/// `read_data_ram(word)` returns them (value, tagged; word < 0xc00), the registers as +/// `read_register(address)` (value, tagged; as the engine's read callback). The call's own +/// stores and emits are kept in `run` and read back from there, as Execute's go to the data RAM +/// and, through the engine, to its registers. Nothing outside `run` changes. +/// +/// A tagged value stays tagged through a move: an ALU operation whose other operand is the +/// identity of the operation and not tagged itself (x + 0, x - 0, x | 0, x ^ 0, x & ~0, +/// x << 0 and the shifts by 0, a merge of no bits), a register, the data RAM, an emit. Any other +/// use (a branch, a predicate, a loop count, an address, a method, any other computation) ends +/// the run as Escaped. Every tagged value is then a list word unchanged: the call takes the same +/// path whatever the words are, and writes them, and only them, where it wrote the tagged values. +template +void RunTagged(std::span code, u32 start, std::span params, + std::span params_tagged, ReadDataRam&& read_data_ram, + ReadRegister&& read_register, std::vector& decoded, + TaggedRun& run) { + using Word = u32; + using Decoded = TuringMacroInterpreter::Decoded; + run = {}; + std::array regs{}; + std::array regs_tagged{}; + Word initialized{}, carry{}, method{}, increment{}; + Word pc = start, loop_count{}, loop_start{}, loop_end{}; + size_t consumed{}; + bool end_next{}, stopped{}; + std::string_view error; + const auto fail = [&](std::string_view message) { + if (error.empty()) + error = message; + }; + bool escaped{}; + const auto escape = [&](std::string_view why) { + if (!escaped) { + escaped = true; + run.why = why; + } + }; + const auto data_ram = [&](Word address) -> std::pair { + for (auto it = run.data_ram.rbegin(); it != run.data_ram.rend(); ++it) { + if (it->address == address) { + return {it->value, it->tagged}; + } + } + return read_data_ram(address); + }; + const auto register_value = [&](Word address) -> std::pair { + // The engine's registers hold what the call emitted to them (FrontApply). + for (auto it = run.emits.rbegin(); it != run.emits.rend(); ++it) { + if (it->address == address) { + return {it->value, it->tagged}; + } + } + return read_register(address); + }; + if (decoded.size() < code.size() / 3) + decoded.resize(code.size() / 3); + constexpr size_t data_ram_size = std::tuple_size_v; + for (Word steps = 0; steps < 1000000 && !escaped; ++steps) { + if (static_cast(pc) * 3 + 2 >= code.size()) { + fail("instruction outside uploaded RAM"); + break; + } + const std::array bits{code[pc * 3 + 2], code[pc * 3 + 1], code[pc * 3]}; + Decoded& d = decoded[pc]; + if (d.raw != bits) + d = TuringMacroInterpreter::Decode(bits); + std::array load{}, alu{}; + std::array load_tagged{}, alu_tagged{}; + for (Word i = 0; i < 2; ++i) { + if (d.load[i]) { + if (consumed >= params.size()) { + fail("parameter FIFO underflow"); + break; + } + load_tagged[i] = consumed < params_tagged.size() && params_tagged[consumed] != 0; + load[i] = params[consumed++]; + } + } + bool operand_tagged{}; + const auto get = [&](Word reg, Word lane) -> Word { + operand_tagged = false; + if (reg < 24) { + if (!(initialized & (1U << reg))) + fail("uninitialized register read"); + operand_tagged = regs_tagged[reg]; + return regs[reg]; + } + switch (reg) { + case 24: + return 0; + case 25: + return Word(std::int32_t(std::int16_t(d.imm[lane]))); + case 26: + return Word(std::int32_t(std::int16_t(d.imm[1 - lane]))); + case 27: + return (d.imm[0] << 16) | d.imm[1]; + case 28: + operand_tagged = load_tagged[0]; + return load[0]; + case 29: + operand_tagged = load_tagged[1]; + return load[1]; + default: + fail("invalid source register"); + return 0; + } + }; + Word mask = 15; + if (d.pred_mode) { + const Word predicate = get(d.pred_reg, 0); + if (operand_tagged) { + escape("a predicate"); + break; + } + mask = predicate ? d.mask_true : d.mask_false; + } + std::array xs{}, ys{}; + std::array xs_tagged{}, ys_tagged{}; + for (Word lane = 0; lane < 2; ++lane) { + if (mask & (1U << lane)) { + xs[lane] = get(d.src_a[lane], lane); + xs_tagged[lane] = operand_tagged; + ys[lane] = get(d.src_b[lane], lane); + ys_tagged[lane] = operand_tagged; + } + } + Word next_pc = pc + 1; + bool branch_cancels_end{}; + for (Word lane = 0; lane < 2 && error.empty() && !escaped; ++lane) { + if (!(mask & (1U << lane))) + continue; + const Word x = xs[lane], y = ys[lane]; + const bool tx = xs_tagged[lane], ty = ys_tagged[lane]; + const Word imm = d.imm[lane]; + Word value{}; + bool tagged{}; + // A tagged operand moved unchanged through the operation: x with y its identity, or + // y with x its identity (`commutes`). + const auto move = [&](Word identity, bool commutes, std::string_view what) { + if (tx && !ty && y == identity) { + tagged = true; + return x; + } + if (commutes && ty && !tx && x == identity) { + tagged = true; + return y; + } + escape(what); + return Word{}; + }; + const auto condition = [&](Word comparison) { + switch (comparison) { + case 17: + case 25: + return std::int32_t(x) < std::int32_t(y); + case 18: + case 26: + return x < y; + case 19: + case 27: + return std::int32_t(x) <= std::int32_t(y); + case 20: + case 28: + return x <= y; + default: + return x == y; + } + }; + const bool any_tagged = tx || ty; + switch (d.op[lane]) { + case 0: + if (any_tagged) { + value = move(0, true, "an addition"); + carry = 0; // x + 0 never carries + } else { + value = x + y; + carry = value < x; + } + break; + case 1: + if (!any_tagged) { + value = x + y + carry; + } else if (carry == 0) { + value = move(0, true, "an addition"); + } else { + escape("an addition"); + } + break; + case 2: + if (any_tagged) { + value = move(0, false, "a subtraction"); + carry = 0; // x - 0 never borrows + } else { + value = x - y; + carry = value > x; + } + break; + case 3: + if (!any_tagged) { + value = x - y - carry; + } else if (carry == 0) { + value = move(0, false, "a subtraction"); + } else { + escape("a subtraction"); + } + break; + case 4: + case 6: { + if (any_tagged) { + escape("a multiplication"); + break; + } + const std::uint64_t a = + d.op[lane] == 4 ? std::uint64_t(std::int64_t(std::int32_t(x))) : x; + const std::uint64_t b = + d.op[lane] == 4 ? std::uint64_t(std::int64_t(std::int32_t(y))) : y; + const auto product = a * b; + value = Word(product); + carry = Word(product >> 32); + break; + } + case 5: + value = carry; + break; + case 7: + if (tx) { + escape("an extended operation"); + break; + } + if (x != 0x1000) + fail("unsupported extended operation"); + break; + case 8: + if (tx) { + escape("a leading zero count"); + break; + } + value = Word(std::countl_zero(x)); + break; + case 9: + case 10: + case 11: + if (any_tagged) { + // Shifts take y's low 5 bits: a tagged x moves by a shift of 0. + if (!ty && (y & 31) == 0) { + value = x; + tagged = true; + } else { + escape("a shift"); + } + } else { + value = d.op[lane] == 9 ? x << (y & 31) + : d.op[lane] == 10 ? x >> (y & 31) + : Word(std::int32_t(x) >> (y & 31)); + } + break; + case 12: + value = any_tagged ? move(~0U, true, "an and") : x & y; + break; + case 13: + if (any_tagged) { + escape("a nand"); + break; + } + value = ~(x & y); + break; + case 14: + value = any_tagged ? move(0, true, "an or") : x | y; + break; + case 15: + value = any_tagged ? move(0, true, "an exclusive or") : x ^ y; + break; + case 16: { + const Word dest = (imm >> 10) & 63, count = (imm >> 5) & 31; + const Word source = imm & 31; + if (dest >= 32) { + fail("invalid merge destination"); + break; + } + const Word bits_mask = (1U << count) - 1; + if (any_tagged) { + // Merging no bits leaves x unchanged; a tagged y never moves whole. + if (tx && !ty && count == 0) { + value = x; + tagged = true; + } else { + escape("a merge"); + } + break; + } + value = (x & ~(bits_mask << dest)) | (((y >> source) & bits_mask) << dest); + break; + } + case 17: + case 18: + case 19: + case 20: + case 21: + if (any_tagged) { + escape("a comparison"); + break; + } + value = condition(d.op[lane]) ? ~0U : 0; + break; + case 22: { + if (any_tagged) { + escape("a register address"); + break; + } + const auto [read, read_tagged] = register_value((x + y) & 0xffff); + value = read; + tagged = read_tagged; + break; + } + case 23: + if (tx) { + escape("a loop count"); + break; + } + if (loop_count || imm < 2) { + fail("invalid loop"); + break; + } + if (x == 0) { + next_pc = pc + imm; + break; + } + loop_count = x - 1; + loop_start = pc; + loop_end = pc + imm - 1; + break; + case 24: + next_pc = pc + (imm & 0x7fff); + break; + case 25: + case 26: + case 27: + case 28: + case 29: + if (any_tagged) { + escape("a branch"); + break; + } + if (condition(d.op[lane]) == bool(imm & 0x8000)) { + const auto offset = std::int32_t(imm << 19) >> 19; + if (offset == -4096) + stopped = true; + else { + next_pc = Word(std::int32_t(pc) + offset); + branch_cancels_end = (imm & 0x4000) != 0; + } + } + break; + case 30: { + if (tx) { + escape("a data RAM address"); + break; + } + if (x >= data_ram_size) { + fail("data RAM read out of bounds"); + break; + } + const auto [read, read_tagged] = data_ram(x); + value = read; + tagged = read_tagged; + break; + } + case 31: + if (tx) { + escape("a data RAM address"); + break; + } + if (x >= data_ram_size) { + fail("data RAM write out of bounds"); + break; + } + run.data_ram.push_back({x, y, ty}); + break; + } + if (escaped) + break; + alu[lane] = value; + alu_tagged[lane] = tagged; + if (d.dst[lane] < 24) { + regs[d.dst[lane]] = value; + regs_tagged[d.dst[lane]] = tagged; + initialized |= 1U << d.dst[lane]; + } else if (d.dst[lane] != 24) + fail("invalid destination register"); + } + if (escaped) + break; + bool output_tagged{}; + const auto output = [&](Word selector) -> Word { + output_tagged = false; + switch (selector) { + case 1: + output_tagged = alu_tagged[0]; + return alu[0]; + case 2: + output_tagged = alu_tagged[1]; + return alu[1]; + case 3: + output_tagged = load_tagged[0]; + return load[0]; + case 4: + output_tagged = load_tagged[1]; + return load[1]; + case 5: + return d.imm[0]; + case 6: + return d.imm[1]; + case 8: + return d.imm[0] >> 12; + case 9: + return d.imm[1] >> 12; + case 10: + return (d.imm[0] << 16) | d.imm[1]; + default: + fail("invalid output selector"); + return 0; + } + }; + for (Word lane = 0; lane < 2 && error.empty() && !escaped; ++lane) { + if (!(mask & (4U << lane))) + continue; + if (d.out_method[lane]) { + const Word address = output(d.out_method[lane]); + if (output_tagged) { + escape("a method"); + break; + } + method = address & 0xfff; + increment = (address >> 12) & 15; + } + if (d.out_emit[lane]) { + const Word value = output(d.out_emit[lane]); + if (error.empty()) { + // The engines' emit refuses macro methods. + if (method >= 0xe00) + fail("unsupported emitted method"); + else + run.emits.push_back({method, value, output_tagged}); + } + method += increment; + } + } + if (escaped || !error.empty()) + break; + if (end_next || stopped) { + run.status = TaggedRun::Outcome::Done; + run.consumed = consumed; + return; + } + end_next = d.end && !branch_cancels_end; + if (loop_count && pc == loop_end) { + --loop_count; + next_pc = loop_start + 1; + } + pc = next_pc; + } + run.consumed = consumed; + if (escaped) { + run.status = TaggedRun::Outcome::Escaped; + return; + } + run.status = TaggedRun::Outcome::Failed; + run.why = error.empty() ? std::string_view{"instruction limit exceeded"} : error; +} + +/// A data RAM word or a register that holds a list word: as long as it holds `value`, the +/// placeholder of word PlaceholderWord(value) of slot PlaceholderSlot(value). Anything that +/// writes it another value ends that (checked where it is used). +struct Holder { + u32 address; + u32 value; +}; + +/// What the macros of one engine's code can read (TuringMacroInterpreter::AnalyzeEffects), by +/// start, for the code generation it was asked for last. +class ReadsCache { +public: + struct Reads { + bool data_ram_known{}; + bool registers_known{}; + std::vector data_ram; + std::vector registers; + }; + + [[nodiscard]] const Reads& Get(std::span code, u64 generation, u32 start) { + if (generation != code_generation || code.data() != code_data) { + index.clear(); + reads.clear(); + code_generation = generation; + code_data = code.data(); + } + if (start >= index.size()) { + index.resize(static_cast(start) + 1, NoEntry); + } + if (index[start] == NoEntry) { + auto effects = TuringMacroInterpreter::AnalyzeEffects(code, start); + reads.push_back({effects.data_ram_reads_known, effects.register_reads_known, + std::move(effects.data_ram_reads), std::move(effects.register_reads)}); + index[start] = static_cast(reads.size() - 1); + } + return reads[index[start]]; + } + +private: + static constexpr u32 NoEntry = ~0U; + std::vector index; + std::vector reads; + u64 code_generation{~u64{0}}; + const u32* code_data{}; +}; + +/// Whether a macro that reads `reads` (any word unless `known`) may read one of `holders`. +[[nodiscard]] inline bool MayRead(bool known, std::span reads, + std::span holders) { + if (holders.empty()) { + return false; + } + if (!known) { + return true; + } + return std::ranges::any_of(holders, [reads](const Holder& holder) { + return std::ranges::binary_search(reads, holder.address); + }); +} + +/// The back drops a hold of `slot` (the rasterizer's Nx2ReleaseListSlot), after what the front +/// sent before; at once while the split is off. +template +void ReleaseSlot(VideoCommon::GpuSplit& split, Rasterizer* rasterizer, u32 slot) { + if (!split.Active()) { + rasterizer->Nx2ReleaseListSlot(slot); + return; + } + split.Defer([rasterizer, slot] { rasterizer->Nx2ReleaseListSlot(slot); }, + VideoCommon::GpuSplit::Writes::Noted); +} + +/// The words of `slots` (three a slot), once the back ran what the front sent before and the +/// host GPU copied them (the rasterizer's Nx2ReadListSlots): the front waits for both. +template +void ReadSlots(VideoCommon::GpuSplit& split, Rasterizer* rasterizer, std::span slots, + std::span words) { + split.Run( + [rasterizer, slot_data = slots.data(), count = slots.size(), out = words.data()] { + rasterizer->Nx2ReadListSlots(std::span{slot_data, count}, + std::span{out, count * SlotWords}); + }, + VideoCommon::DrainSite::ListCopies); +} + +/// The front's holds of the back's slots: one per holder and per call that takes list words. +/// Once the front holds a slot no more, it drops the hold the back gave it the slot with, after +/// everything it sent with the slot's placeholders (in order on the split's ring). +inline std::array& FrontHolds() { + static std::array holds{}; + return holds; +} + +/// A channel's data RAM words that hold list words, and what its 3D macros can read. Its compute +/// engine's front puts list words there; its 3D engine's front checks its macros against them. +struct DataRamWords { + const void* data_ram{}; + std::vector words; + ReadsCache reads_3d; +}; + +/// Every channel's, by data RAM (front thread only; a channel's goes with its compute engine). +inline std::vector& Channels() { + static std::vector channels; + return channels; +} + +[[nodiscard]] inline DataRamWords* FindChannel(const void* data_ram) { + for (DataRamWords& channel : Channels()) { + if (channel.data_ram == data_ram) { + return &channel; + } + } + return nullptr; +} + +[[nodiscard]] inline DataRamWords& GetChannel(const void* data_ram) { + if (DataRamWords* channel = FindChannel(data_ram)) { + return *channel; + } + return Channels().emplace_back(DataRamWords{data_ram, {}, {}}); +} + +/// The channel's compute engine is destroyed. With no channel left (the session ended, and the +/// back's slots went with its rasterizer) the front holds none either. +inline void ForgetChannel(const void* data_ram) { + std::erase_if(Channels(), + [data_ram](const DataRamWords& channel) { return channel.data_ram == data_ram; }); + if (Channels().empty()) { + FrontHolds().fill(0); + } +} + +/// Drops a front hold of `slot`; `release(slot)` once the front holds it no more. +template +void DropHold(u32 slot, Release&& release) { + if (--FrontHolds()[slot] == 0) { + release(slot); + } +} + +/// Drops the holders that no longer hold their placeholder (`value(address)` differs). +template +void Prune(std::vector& holders, Value&& value, Release&& release) { + std::erase_if(holders, [&](const Holder& holder) { + if (value(holder.address) == holder.value) { + return false; + } + DropHold(PlaceholderSlot(holder.value), release); + return true; + }); +} + +/// From now on `address` holds `value`: a list word if `tagged`, else nothing of the front's. +template +void Hold(std::vector& holders, u32 address, u32 value, bool tagged, Release&& release) { + if (tagged) { + ++FrontHolds()[PlaceholderSlot(value)]; // First: an old hold of the slot stays above 0 + } + const auto it = std::ranges::find(holders, address, &Holder::address); + if (it != holders.end()) { + const u32 slot = PlaceholderSlot(it->value); + holders.erase(it); + DropHold(slot, release); + } + if (tagged) { + holders.push_back({address, value}); + } +} + +/// The words of `slots` as `read_slots(slots, words)` returns them (three a slot), by +/// placeholder. +struct SlotWordsRead { + std::vector slots; + std::vector words; + + void Add(u32 placeholder) { + const u32 slot = PlaceholderSlot(placeholder); + if (std::ranges::find(slots, slot) == slots.end()) { + slots.push_back(slot); + } + } + template + void Read(ReadSlots&& read_slots) { + words.assign(slots.size() * SlotWords, 0); + if (!slots.empty()) { + read_slots(std::span{slots}, std::span{words}); + } + } + [[nodiscard]] u32 Word(u32 placeholder) const { + const size_t index = static_cast( + std::ranges::find(slots, PlaceholderSlot(placeholder)) - slots.begin()); + return words[index * SlotWords + PlaceholderWord(placeholder)]; + } +}; + +/// Replaces the placeholders the data RAM words `holders` still hold by their words, read back +/// from the slots (`read_slots` waits for the back and the host GPU), and drops the holders. +/// Returns how many slots it read. +template +u32 ConcretizeDataRam(std::vector& holders, std::span data_ram, ReadSlots&& read_slots, + Release&& release) { + SlotWordsRead read; + for (const Holder& holder : holders) { + read.Add(holder.value); + } + read.Read(read_slots); + for (const Holder& holder : holders) { + if (data_ram[holder.address] == holder.value) { + data_ram[holder.address] = read.Word(holder.value); + } + DropHold(PlaceholderSlot(holder.value), release); + } + holders.clear(); + return static_cast(read.slots.size()); +} + +/// What the front knows of a compute engine's inline upload as the back runs it +/// (Upload::State): the words the data methods sent since the last exec, and the size and layout +/// the exec started it with. +struct UploadState { + u32 words{}; + u32 size{}; + bool linear{}; +}; + +/// The compute class's inline upload methods (KeplerCompute::Regs::upload and the two after). +inline constexpr u32 UploadLineLength = 0x60; +inline constexpr u32 UploadLineCount = 0x61; +inline constexpr u32 UploadAddressHigh = 0x62; +inline constexpr u32 UploadAddressLow = 0x63; +inline constexpr u32 UploadExec = 0x6c; +inline constexpr u32 UploadData = 0x6d; +/// The inline QMD's grid words (LOAD_INLINE_QMD_DATA, QMD words 12-14). +inline constexpr u32 InlineQmdGridFirst = 0xc8 + 12; +inline constexpr u32 InlineQmdGridLast = 0xc8 + 14; + +/// As Upload::State::ProcessExec and ProcessData count an inline upload: `method` with +/// `argument` sent to the engine, `reg(method)` the registers as they were before it. +template +void NoteUploadMethod(UploadState& upload, u32 method, u32 argument, Register&& reg) { + if (method == UploadExec) { + upload.words = 0; + upload.size = reg(UploadLineLength) * reg(UploadLineCount); + upload.linear = (argument & 1) != 0; + } else if (method == UploadData) { + ++upload.words; + } +} + +/// Why the front read list words after all (the counters). +enum class Reason : u32 { + Escaped, ///< A macro used a tagged value otherwise than moving it (RunTagged) + Failed, ///< The dry run failed: the call runs the old way, as it fails there + Sink, ///< A tagged value reached another register, or an upload the back can't place + Upload, ///< A macro that takes list words loads macro code or binds macros + Macro3d, ///< A 3D macro may read a data RAM word that holds one + Stop, ///< The split stopped + Count, +}; + +/// One compute engine's front while list words are about (KeplerCompute::FrontRunMacro): the +/// registers that hold list words, the call whose parameters are list words, and the caches of +/// the dry runs. `Context` is the engine's, for one call: +/// - std::span code; u64 generation; u32 start: the call's macro; +/// - std::vector& params; std::span data_ram; std::span registers; +/// - UploadState upload: before the call; +/// - std::optional DeviceWord(u64 gpu_address): the device address of the 4 bytes there, +/// if they are mapped, on one page, as the back will find them; +/// - void Emit(u32 method, u32 value): the engine's emit (FrontApply and the split); +/// - void SendOwed(u64 device, u32 value): the back patches the slot word that `value` is the +/// placeholder of into the constant buffers compute dispatches bind there; +/// - void Release(u32 slot): the back drops the front's hold of the slot; +/// - void ReadSlots(std::span slots, std::span words): the words (waits); +/// - void SetRegister(u32 method, u32 value): a concretized register, on the front and the back. +class ComputeFront { +public: + /// DmaPusher, at a list of `count` words on GPU-written pages that are all the parameters of + /// the call about to run: whether a dry run of the call with them as tagged parameters leaves + /// them only where the back takes them over. If so the call takes `slot`'s placeholders as + /// its parameters (the front holds the slot for it) and runs tagged (Run). + template + [[nodiscard]] bool PrepareCall(Context& ctx, DataRamWords& channel, u32 slot, u32 count) { + // Holders the macros the front ran untagged wrote over give their slots back. + const auto release = [&ctx](u32 held) { ctx.Release(held); }; + Prune(channel.words, [&ctx](u32 address) { return ctx.data_ram[address]; }, release); + Prune(registers, [&ctx](u32 address) { return ctx.registers[address]; }, release); + std::vector params(count); + for (u32 k = 0; k < count; ++k) { + params[k] = Placeholder(slot, k); + } + TaggedRun run; + DryRun(ctx, channel, params, true, run); + if (run.status != TaggedRun::Outcome::Done || !Places(ctx, run)) { + return false; + } + if (pending.active) [[unlikely]] { + DropPending(ctx); + } + pending = Pending{slot, count, true}; + ++FrontHolds()[slot]; + return true; + } + + /// The call `ctx` describes, at its last parameter. If it takes list words, or may read a + /// data RAM word or register that holds one, runs it tagged and returns true. Otherwise, and + /// where its list words go elsewhere than the back follows them (it concretizes every list + /// word about first: `concretized`, `reason`), the caller runs it as before (false). + template + [[nodiscard]] bool Run(Context& ctx, DataRamWords& channel, bool& concretized, Reason& reason) { + concretized = false; + const bool takes = + pending.active && pending.count == ctx.params.size() && TakesPlaceholders(ctx.params); + if (pending.active && !takes) [[unlikely]] { + DropPending(ctx); // Not the call it was prepared for + } + if (!takes) { + if (channel.words.empty() && registers.empty()) { + return false; + } + if (!MayReadHeld(ctx, channel)) { + return false; + } + const auto release = [&ctx](u32 slot) { ctx.Release(slot); }; + Prune(channel.words, [&ctx](u32 address) { return ctx.data_ram[address]; }, release); + Prune(registers, [&ctx](u32 address) { return ctx.registers[address]; }, release); + if (!MayReadHeld(ctx, channel)) { + return false; + } + } + TaggedRun run; + DryRun(ctx, channel, ctx.params, takes, run); + if (run.status == TaggedRun::Outcome::Done && Places(ctx, run)) { + Commit(ctx, channel, run); + if (takes) { + DropPending(ctx); + } + return true; + } + reason = run.status == TaggedRun::Outcome::Escaped ? Reason::Escaped + : run.status == TaggedRun::Outcome::Failed ? Reason::Failed + : place_reason; + ConcretizeAll(ctx, channel); + concretized = true; + return false; + } + + /// Replaces every list word the channel's data RAM, this engine's registers and the pending + /// call's parameters hold by the word itself, read back from its slot. + template + void ConcretizeAll(Context& ctx, DataRamWords& channel) { + const auto release = [&ctx](u32 slot) { ctx.Release(slot); }; + const bool params = + pending.active && pending.count == ctx.params.size() && TakesPlaceholders(ctx.params); + SlotWordsRead read; + for (const Holder& holder : channel.words) { + read.Add(holder.value); + } + for (const Holder& holder : registers) { + read.Add(holder.value); + } + if (params) { + read.Add(Placeholder(pending.slot, 0)); + } + read.Read([&ctx](std::span slots, std::span words) { + ctx.ReadSlots(slots, words); + }); + for (const Holder& holder : channel.words) { + if (ctx.data_ram[holder.address] == holder.value) { + ctx.data_ram[holder.address] = read.Word(holder.value); + } + DropHold(PlaceholderSlot(holder.value), release); + } + channel.words.clear(); + for (const Holder& holder : registers) { + if (holder.address < ctx.registers.size() && + ctx.registers[holder.address] == holder.value) { + ctx.SetRegister(holder.address, read.Word(holder.value)); + } + DropHold(PlaceholderSlot(holder.value), release); + } + registers.clear(); + if (params) { + for (u32 k = 0; k < pending.count; ++k) { + ctx.params[k] = read.Word(Placeholder(pending.slot, k)); + } + } + DropPending(ctx); + } + + /// The registers that hold list words. + [[nodiscard]] std::span Registers() const { + return registers; + } + +private: + struct Pending { + u32 slot{}; + u32 count{}; + bool active{}; + }; + + [[nodiscard]] bool TakesPlaceholders(std::span params) const { + for (u32 k = 0; k < pending.count; ++k) { + if (params[k] != Placeholder(pending.slot, k)) { + return false; + } + } + return true; + } + + template + void DropPending(Context& ctx) { + if (!pending.active) { + return; + } + pending.active = false; + DropHold(pending.slot, [&ctx](u32 slot) { ctx.Release(slot); }); + } + + /// Whether the call's macro may read a data RAM word or register that holds a list word. + template + [[nodiscard]] bool MayReadHeld(Context& ctx, const DataRamWords& channel) { + const auto& reads = cache.Get(ctx.code, ctx.generation, ctx.start); + return Nx2ListIndirect::MayRead(reads.data_ram_known, reads.data_ram, channel.words) || + Nx2ListIndirect::MayRead(reads.registers_known, reads.registers, registers); + } + + /// The dry run of the call with `params` (tagged if `tagged`), the data RAM and the + /// registers with their holders. + template + void DryRun(Context& ctx, const DataRamWords& channel, std::span params, bool tagged, + TaggedRun& run) { + params_tagged.assign(params.size(), static_cast(tagged ? 1 : 0)); + RunTagged( + ctx.code, ctx.start, params, std::span{params_tagged}, + [&](u32 word) -> std::pair { + const u32 value = ctx.data_ram[word]; + const auto it = std::ranges::find(channel.words, word, &Holder::address); + return {value, it != channel.words.end() && it->value == value}; + }, + [&](u32 address) -> std::pair { + if (address >= ctx.registers.size()) { + return {0U, false}; + } + const u32 value = ctx.registers[address]; + const auto it = std::ranges::find(registers, address, &Holder::address); + return {value, it != registers.end() && it->value == value}; + }, + decoded, run); + } + + /// Whether every tagged value the run emits lands where the back takes it over: an inline + /// QMD grid word (the back dispatches from the slot) or a word of a linear one-line inline + /// upload the front can place (the back patches it into constant buffers). The data RAM + /// keeps the others. Notes where each such upload word lands in `owed`. + template + [[nodiscard]] bool Places(Context& ctx, const TaggedRun& run) { + owed.assign(run.emits.size(), 0); + UploadState upload = ctx.upload; + const auto reg = [&](size_t before, u32 method) { + for (size_t i = before; i-- > 0;) { + if (run.emits[i].address == method) { + return run.emits[i].value; + } + } + return method < ctx.registers.size() ? ctx.registers[method] : 0U; + }; + for (size_t i = 0; i < run.emits.size(); ++i) { + const TaggedWrite& emit = run.emits[i]; + if (emit.address >= 0x45 && emit.address <= 0x48) { + place_reason = Reason::Upload; // FrontApply loads code and bindings + return false; + } + if (emit.address == UploadData && emit.tagged) { + // As Upload::State::ProcessData writes a word of a linear upload of one line. + if (!upload.linear || reg(i, UploadLineCount) != 1 || + u64{upload.words} * 4 + 4 > upload.size) { + place_reason = Reason::Sink; + return false; + } + const u64 destination = + ((u64{reg(i, UploadAddressHigh)} << 32) | reg(i, UploadAddressLow)) + + u64{upload.words} * 4; + const std::optional device = ctx.DeviceWord(destination); + if (!device) { + place_reason = Reason::Sink; + return false; + } + owed[i] = *device; + } else if (emit.tagged && + (emit.address < InlineQmdGridFirst || emit.address > InlineQmdGridLast)) { + place_reason = Reason::Sink; + return false; + } + NoteUploadMethod(upload, emit.address, emit.value, + [&](u32 method) { return reg(i, method); }); + } + return true; + } + + /// Applies the run: its data RAM stores, its emits through the engine, and where list words + /// are now. + template + void Commit(Context& ctx, DataRamWords& channel, const TaggedRun& run) { + const auto release = [&ctx](u32 slot) { ctx.Release(slot); }; + for (const TaggedWrite& store : run.data_ram) { + ctx.data_ram[store.address] = store.value; + Hold(channel.words, store.address, store.value, store.tagged, release); + } + for (size_t i = 0; i < run.emits.size(); ++i) { + const TaggedWrite& emit = run.emits[i]; + ctx.Emit(emit.address, emit.value); + if (emit.address == UploadData && emit.tagged) { + ctx.SendOwed(owed[i], emit.value); + } + if (emit.tagged || !registers.empty()) { + Hold(registers, emit.address, emit.value, emit.tagged, release); + } + } + } + + std::vector registers; + Pending pending; + ReadsCache cache; + std::vector decoded; + std::vector params_tagged; + std::vector owed; + Reason place_reason{}; +}; + +} // namespace Tegra::Nx2ListIndirect diff --git a/src/video_core/rasterizer_interface.h b/src/video_core/rasterizer_interface.h index 335bccf634..d269a78aad 100644 --- a/src/video_core/rasterizer_interface.h +++ b/src/video_core/rasterizer_interface.h @@ -146,6 +146,35 @@ class RasterizerInterface { /// Switch 2: the front flushed this command list page with the split drained. virtual void NoteGpuWrittenCommandListSynced([[maybe_unused]] DAddr page) {} + /// Switch 2, GPU-written command list words the host GPU takes over (Nx2ListIndirect): the + /// back's slots for them. Whether this rasterizer has them. + [[nodiscard]] virtual bool Nx2ListSlotsSupported() const { + return false; + } + /// Any thread: a slot nobody holds, held for the caller; none while every slot is held. + [[nodiscard]] virtual std::optional Nx2AcquireListSlot() { + return std::nullopt; + } + /// Any thread: drops a hold of `slot` (Nx2AcquireListSlot's or one the back took). + virtual void Nx2ReleaseListSlot([[maybe_unused]] u32 slot) {} + /// The back, at a list's place: the host GPU copies the list's `count` words at + /// `device_addr` into `slot`. + virtual void Nx2SnapshotListWords([[maybe_unused]] DAddr device_addr, + [[maybe_unused]] u32 count, [[maybe_unused]] u32 slot) {} + /// The back: the word at `device_addr` was just written `value`, the placeholder of a slot + /// word. + /// While it holds it, compute dispatches that bind the word as constant buffer data get the + /// slot word there instead. + virtual void Nx2NoteOwedListWord([[maybe_unused]] DAddr device_addr, + [[maybe_unused]] u32 value) {} + /// The back: an inline QMD with grid words `x`, `y`, `z` was just written at `device_addr`. + /// Holds the slots placeholders among them name, until the next inline QMD written there. + virtual void Nx2NoteInlineQmdGrid([[maybe_unused]] DAddr device_addr, [[maybe_unused]] u32 x, + [[maybe_unused]] u32 y, [[maybe_unused]] u32 z) {} + /// The back: the words of `slots`, three each, read back from the host GPU (waits for it). + virtual void Nx2ReadListSlots([[maybe_unused]] std::span slots, + [[maybe_unused]] std::span words) {} + virtual void FlushGpuWrittenCommandListPages(std::span pages) { for (const DAddr page : pages) { FlushRegion(page, 0x1000, VideoCommon::CacheType::BufferCache); diff --git a/src/video_core/renderer_vulkan/nx2_list_slots.cpp b/src/video_core/renderer_vulkan/nx2_list_slots.cpp new file mode 100644 index 0000000000..d424b62483 --- /dev/null +++ b/src/video_core/renderer_vulkan/nx2_list_slots.cpp @@ -0,0 +1,196 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later + +#include + +#include "common/logging.h" +#include "video_core/renderer_vulkan/nx2_list_slots.h" +#include "video_core/renderer_vulkan/vk_scheduler.h" +#include "video_core/vulkan_common/vulkan_memory_allocator.h" + +namespace Vulkan { + +void Nx2ListSlots::Release(u32 slot) { + if (slot >= SlotCount) { + return; + } + if (holds[slot].fetch_sub(1, std::memory_order_acq_rel) == 0) [[unlikely]] { + holds[slot].fetch_add(1, std::memory_order_relaxed); + LOG_ERROR(Render_Vulkan, "Nx2 list slot {} released without a hold", slot); + } +} + +void Nx2ListSlots::Snapshot(VkBuffer src, u64 src_offset, u32 count, u32 slot) { + if (src == VK_NULL_HANDLE || slot >= SlotCount || count == 0 || + count > Tegra::Nx2ListIndirect::SlotWords) { + return; + } + Prepare(); + const VkDeviceSize base = VkDeviceSize{slot} * SlotBytes; + copies.clear(); + copies.push_back(VkBufferCopy{src_offset, base, VkDeviceSize{count} * 4}); + if (count == Tegra::Nx2ListIndirect::SlotWords) { + // The grid as DispatchNativeCompute takes it: height and depth are 16 bits. The upper + // halves of their words in the slot stay 0 from the fill (Prepare). + copies.push_back(VkBufferCopy{src_offset, base + DispatchOffset, 4}); + copies.push_back(VkBufferCopy{src_offset + 4, base + DispatchOffset + 4, 2}); + copies.push_back(VkBufferCopy{src_offset + 8, base + DispatchOffset + 8, 2}); + } + RecordCopies(src, *buffer, copies); +} + +void Nx2ListSlots::NoteOwed(DAddr device_addr, u32 value, u32 guest_word) { + // The upload wrote over older owed words at these bytes. + for (auto it = owed.lower_bound(device_addr >= 3 ? device_addr - 3 : 0); + it != owed.end() && it->first < device_addr + 4;) { + Release(Tegra::Nx2ListIndirect::PlaceholderSlot(it->second)); + it = owed.erase(it); + } + if (guest_word != value || !Tegra::Nx2ListIndirect::IsPlaceholder(value)) { + return; // It wrote nothing there + } + holds[Tegra::Nx2ListIndirect::PlaceholderSlot(value)].fetch_add(1, std::memory_order_acq_rel); + owed.emplace(device_addr, value); +} + +void Nx2ListSlots::NoteInlineQmdGrid(DAddr device_addr, u32 x, u32 y, u32 z) { + std::array slots{NoEntry, NoEntry, NoEntry}; + size_t count = 0; + for (const u32 word : {x, y, z}) { + if (!Tegra::Nx2ListIndirect::IsPlaceholder(word)) { + continue; + } + const u32 slot = Tegra::Nx2ListIndirect::PlaceholderSlot(word); + if (std::ranges::find(slots, slot) == slots.end()) { + slots[count++] = slot; + } + } + if (count == 0 && qmd_slots.empty()) { + return; + } + // The new holds first: an old hold of the same slot is dropped after them. + for (size_t i = 0; i < count; ++i) { + holds[slots[i]].fetch_add(1, std::memory_order_acq_rel); + } + const auto it = qmd_slots.find(device_addr); + if (it == qmd_slots.end()) { + if (count != 0) { + qmd_slots.emplace(device_addr, slots); + } + return; + } + for (const u32 slot : it->second) { + if (slot != NoEntry) { + Release(slot); + } + } + if (count == 0) { + qmd_slots.erase(it); + } else { + it->second = slots; + } +} + +void Nx2ListSlots::Read(std::span slots, std::span words) { + std::ranges::fill(words, 0U); + if (!buffer) { + return; + } + if (!readback) { + readback = memory_allocator.CreateBuffer( + VkBufferCreateInfo{ + .sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, + .pNext = nullptr, + .flags = 0, + .size = VkDeviceSize{ReadbackSlots} * SlotWordBytes, + .usage = VK_BUFFER_USAGE_TRANSFER_DST_BIT, + .sharingMode = VK_SHARING_MODE_EXCLUSIVE, + .queueFamilyIndexCount = 0, + .pQueueFamilyIndices = nullptr, + }, + MemoryUsage::Download); + } + for (size_t first = 0; first < slots.size(); first += ReadbackSlots) { + const size_t count = (std::min)(ReadbackSlots, slots.size() - first); + copies.clear(); + for (size_t i = 0; i < count; ++i) { + copies.push_back(VkBufferCopy{VkDeviceSize{slots[first + i]} * SlotBytes, + VkDeviceSize{i} * SlotWordBytes, SlotWordBytes}); + } + RecordCopies(*buffer, *readback, copies); + scheduler.Record([](vk::CommandBuffer cmdbuf) { + // A fence makes device writes available, not visible to the host. + static constexpr VkMemoryBarrier HOST_BARRIER{ + .sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER, + .pNext = nullptr, + .srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT, + .dstAccessMask = VK_ACCESS_HOST_READ_BIT, + }; + cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_HOST_BIT, 0, + HOST_BARRIER); + }); + scheduler.Finish(); + readback.Invalidate(); + const size_t bytes = + (std::min)(count * SlotWordBytes, words.size_bytes() - first * SlotWordBytes); + std::memcpy(words.data() + first * Tegra::Nx2ListIndirect::SlotWords, + readback.Mapped().data(), bytes); + } +} + +void Nx2ListSlots::Prepare() { + if (buffer) { + return; + } + buffer = memory_allocator.CreateBuffer( + VkBufferCreateInfo{ + .sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, + .pNext = nullptr, + .flags = 0, + .size = VkDeviceSize{SlotCount} * SlotBytes, + .usage = VK_BUFFER_USAGE_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT | + VK_BUFFER_USAGE_INDIRECT_BUFFER_BIT, + .sharingMode = VK_SHARING_MODE_EXCLUSIVE, + .queueFamilyIndexCount = 0, + .pQueueFamilyIndices = nullptr, + }, + MemoryUsage::DeviceLocal); + scheduler.RequestOutsideRenderPassOperationContext(); + scheduler.Record([dst = *buffer](vk::CommandBuffer cmdbuf) { + static constexpr VkMemoryBarrier WRITE_BARRIER{ + .sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER, + .pNext = nullptr, + .srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT, + .dstAccessMask = VK_ACCESS_MEMORY_READ_BIT | VK_ACCESS_MEMORY_WRITE_BIT, + }; + cmdbuf.FillBuffer(dst, 0, VK_WHOLE_SIZE, 0); + cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_TRANSFER_BIT, + vk::PIPELINE_STAGE_GRAPHICS_COMPUTE_TRANSFER, 0, WRITE_BARRIER); + }); +} + +void Nx2ListSlots::RecordCopies(VkBuffer src, VkBuffer dst, std::span regions) { + static constexpr VkMemoryBarrier READ_BARRIER{ + .sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER, + .pNext = nullptr, + .srcAccessMask = VK_ACCESS_MEMORY_WRITE_BIT, + .dstAccessMask = VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT, + }; + static constexpr VkMemoryBarrier WRITE_BARRIER{ + .sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER, + .pNext = nullptr, + .srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT, + .dstAccessMask = VK_ACCESS_MEMORY_READ_BIT | VK_ACCESS_MEMORY_WRITE_BIT, + }; + scheduler.RequestOutsideRenderPassOperationContext(); + scheduler.Record([src, dst, list = std::vector(regions.begin(), regions.end())]( + vk::CommandBuffer cmdbuf) { + cmdbuf.PipelineBarrier(vk::PIPELINE_STAGE_GRAPHICS_COMPUTE_TRANSFER, + VK_PIPELINE_STAGE_TRANSFER_BIT, 0, READ_BARRIER); + cmdbuf.CopyBuffer(src, dst, list); + cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_TRANSFER_BIT, + vk::PIPELINE_STAGE_GRAPHICS_COMPUTE_TRANSFER, 0, WRITE_BARRIER); + }); +} + +} // namespace Vulkan diff --git a/src/video_core/renderer_vulkan/nx2_list_slots.h b/src/video_core/renderer_vulkan/nx2_list_slots.h new file mode 100644 index 0000000000..c25c73f5bd --- /dev/null +++ b/src/video_core/renderer_vulkan/nx2_list_slots.h @@ -0,0 +1,157 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "common/common_types.h" +#include "video_core/nx2_list_indirect.h" +#include "video_core/vulkan_common/vulkan_wrapper.h" + +namespace Vulkan { + +class MemoryAllocator; +class Scheduler; + +/// The split GPU thread's back's slots for GPU-written command list words the host GPU takes +/// over (Nx2ListIndirect). A slot is SlotBytes of device-local memory: the list's words as the +/// back found them at the list's place (the snapshot, raw), and at DispatchOffset the same three +/// as DispatchNativeCompute takes a grid: width, height & 0xffff, depth & 0xffff (a +/// VkDispatchIndirectCommand). Holds keep a slot from being used again: the front's while its +/// macros' words hold the slot's placeholders, the back's while a QMD's grid or a word compute +/// dispatches bind as constant buffer data holds them. +/// +/// The words the front's inline uploads put placeholders in are owed words: while one holds its +/// placeholder in guest memory, a compute dispatch that binds it as constant buffer data gets the +/// slot word there (PatchUniform, recorded after the binding filled its buffer). +class Nx2ListSlots { +public: + static constexpr u32 SlotCount = Tegra::Nx2ListIndirect::SlotCount; + static constexpr u32 SlotBytes = 32; + static constexpr u32 DispatchOffset = 16; + + explicit Nx2ListSlots(MemoryAllocator& memory_allocator_, Scheduler& scheduler_) + : memory_allocator{memory_allocator_}, scheduler{scheduler_} {} + + /// Any thread (the front's): a slot nobody holds, held for the caller. + [[nodiscard]] std::optional Acquire() { + for (u32 i = 0; i < SlotCount; ++i) { + const u32 slot = (next + i) % SlotCount; + u32 expected = 0; + if (holds[slot].compare_exchange_strong(expected, 1, std::memory_order_acq_rel)) { + next = slot + 1; + return slot; + } + } + return std::nullopt; + } + + /// Any thread: drops a hold of `slot`. + void Release(u32 slot); + + [[nodiscard]] bool Held(u32 slot) const { + return slot < SlotCount && holds[slot].load(std::memory_order_acquire) != 0; + } + + /// The back: records the copy of the `count` words at `src_offset` of `src` (the list's host + /// buffer, the copy's use of the words marked) into `slot`. + void Snapshot(VkBuffer src, u64 src_offset, u32 count, u32 slot); + + /// The back: the word at `device_addr` was just written `value` (a placeholder) by an inline + /// upload; `guest_word` is what guest memory holds there now. + void NoteOwed(DAddr device_addr, u32 value, u32 guest_word); + + /// The back: an inline QMD with grid words x, y, z was just written at `device_addr`. + void NoteInlineQmdGrid(DAddr device_addr, u32 x, u32 y, u32 z); + + [[nodiscard]] bool HasOwed() const { + return !owed.empty(); + } + + /// The back, binding a compute dispatch's constant buffer [device_addr, +size) at `offset` + /// of `dst`, filled already: records copies of the slot words owed there over it. `read` + /// gives the word guest memory holds at a device address: a word that no longer holds its + /// placeholder is no longer owed. + template + void PatchUniform(DAddr device_addr, u32 size, VkBuffer dst, u64 offset, Read&& read) { + if (dst == VK_NULL_HANDLE || size == 0) { + return; + } + copies.clear(); + const DAddr end = device_addr + size; + for (auto it = owed.lower_bound(device_addr >= 3 ? device_addr - 3 : 0); + it != owed.end() && it->first < end;) { + const DAddr word = it->first; + const u32 value = it->second; + if (read(word) != value) { + Release(Tegra::Nx2ListIndirect::PlaceholderSlot(value)); + it = owed.erase(it); + continue; + } + const DAddr first = (std::max)(word, device_addr); + const DAddr last = (std::min)(word + 4, end); + if (first < last) { + const VkDeviceSize source = + VkDeviceSize{Tegra::Nx2ListIndirect::PlaceholderSlot(value)} * SlotBytes + + VkDeviceSize{Tegra::Nx2ListIndirect::PlaceholderWord(value)} * 4 + + (first - word); + copies.push_back( + VkBufferCopy{source, offset + (first - device_addr), last - first}); + } + ++it; + } + if (!copies.empty()) { + ++patched_binds; + RecordCopies(*buffer, dst, copies); + } + } + + /// The buffer and offset of `slot`'s VkDispatchIndirectCommand. + [[nodiscard]] std::pair DispatchArguments(u32 slot) const { + return {*buffer, VkDeviceSize{slot} * SlotBytes + DispatchOffset}; + } + + /// The back: the raw words of `slots`, three each, once the host GPU wrote them (submits and + /// waits for it). + void Read(std::span slots, std::span words); + + /// Dispatches whose constant buffers got owed words. + [[nodiscard]] u64 PatchedBinds() const { + return patched_binds; + } + +private: + static constexpr u32 NoEntry = ~0U; + static constexpr u32 SlotWordBytes = Tegra::Nx2ListIndirect::SlotWords * sizeof(u32); + static constexpr u32 ReadbackSlots = 256; + + /// The slots' buffer, created on first use, zero-filled. + void Prepare(); + + /// Records `regions` from `src` to `dst` outside a render pass, between barriers as + /// BufferCacheRuntime::CopyBuffer puts them: after every write and read before (a slot a copy + /// writes may be read by an earlier dispatch or copy), before every read and write after. + void RecordCopies(VkBuffer src, VkBuffer dst, std::span regions); + + MemoryAllocator& memory_allocator; + Scheduler& scheduler; + vk::Buffer buffer; + vk::Buffer readback; ///< Read's, host-visible + std::array, SlotCount> holds{}; + u32 next{}; ///< Where Acquire looks first + std::map owed; ///< By device address (the back's) + std::unordered_map> qmd_slots; ///< By QMD device address + std::vector copies; + u64 patched_binds{}; +}; + +} // namespace Vulkan diff --git a/src/video_core/renderer_vulkan/vk_buffer_cache.h b/src/video_core/renderer_vulkan/vk_buffer_cache.h index d0a5d1a26f..5fa6567670 100644 --- a/src/video_core/renderer_vulkan/vk_buffer_cache.h +++ b/src/video_core/renderer_vulkan/vk_buffer_cache.h @@ -18,6 +18,7 @@ #include "video_core/buffer_cache/upload_order.h" #include "video_core/engines/maxwell_3d.h" #include "video_core/renderer_vulkan/vk_compute_pass.h" +#include "video_core/renderer_vulkan/nx2_list_slots.h" #include "video_core/renderer_vulkan/vk_multi_range_buffer.h" #include "video_core/renderer_vulkan/vk_staging_buffer_pool.h" #include "video_core/renderer_vulkan/vk_update_descriptor.h" @@ -247,6 +248,22 @@ class BufferCacheRuntime { return last_stream_uniform; } + /// GPU-written list words the host GPU took over (Nx2ListIndirect), owed in a compute + /// dispatch's constant buffer [device_addr, +size) that `buffer` holds at `offset`, filled + /// already: the slot words go over the placeholders. `read(device)`: guest memory's word. + template + void Nx2PatchComputeUniform(DAddr device_addr, u32 size, VkBuffer buffer, u64 offset, + Read&& read) { + if (nx2_list_slots.HasOwed()) [[unlikely]] { + nx2_list_slots.PatchUniform(device_addr, size, buffer, offset, read); + } + } + + /// The slots of GPU-written list words (Nx2ListIndirect). + [[nodiscard]] Nx2ListSlots& Nx2Slots() noexcept { + return nx2_list_slots; + } + /// Adds `ref` to the descriptor payload exactly as BindMappedUniformBuffer did when it made /// the allocation (it is the very same call). The caller guarantees the allocation is still /// live: made on the scheduler tick that is current now, so the staging pool has not recycled @@ -387,6 +404,8 @@ class BufferCacheRuntime { VkIndexType bound_index_type = VK_INDEX_TYPE_UINT32; u64 bound_index_epoch = 0; bool bound_index_valid = false; + + Nx2ListSlots nx2_list_slots{memory_allocator, scheduler}; }; struct BufferCacheParams { diff --git a/src/video_core/renderer_vulkan/vk_rasterizer.cpp b/src/video_core/renderer_vulkan/vk_rasterizer.cpp index c65c5d1aa1..746241eb0e 100644 --- a/src/video_core/renderer_vulkan/vk_rasterizer.cpp +++ b/src/video_core/renderer_vulkan/vk_rasterizer.cpp @@ -6,6 +6,7 @@ #include #include +#include #include #include #include @@ -1794,9 +1795,10 @@ void RasterizerVulkan::DispatchCompute() { } }; const auto indirect_address = kepler_compute->GetIndirectComputeAddress(); + const auto list_slot = kepler_compute->GetIndirectListSlot(); #ifdef EGLANTINE_STREAMLINE const bool can_replace = OcarinaDlssEnabled() && pipeline->CanReplaceOcarinaUpscale() && - !indirect_address && !frame_dump.Capturing(); + !indirect_address && !list_slot && !frame_dump.Capturing(); #else constexpr bool can_replace = false; #endif @@ -1843,6 +1845,41 @@ void RasterizerVulkan::DispatchCompute() { {qmd.block_dim_x, qmd.block_dim_y, qmd.block_dim_z}, indirect_address, indirect_grid); } + if (list_slot) { + // The grid is GPU-written list words (Nx2ListIndirect): the host GPU copied them into the + // slot at the list's place, and dispatches from there. + auto& slots = buffer_cache_runtime.Nx2Slots(); + if (!slots.Held(*list_slot)) [[unlikely]] { + LOG_ERROR(Render_Vulkan, "Nx2 list slot {} dispatched without a hold", *list_slot); + return; + } + const auto [args_buffer, args_offset] = slots.DispatchArguments(*list_slot); + scheduler.RequestOutsideRenderPassOperationContext(); + scheduler.NoteComputeWrites(); + scheduler.TraceComputePipeline(pipeline->ShaderHash()); + scheduler.Record([this, pipeline, byte_addressed_shadow, indirect_buffer = args_buffer, + indirect_offset = args_offset](vk::CommandBuffer cmdbuf) { + RecordDispatchBarrier(cmdbuf); + if (!pipeline->IsBound()) { + return; + } + if (byte_addressed_shadow) { + TextureCacheRuntime::CopyByteAddressedShadow(cmdbuf, *byte_addressed_shadow, false); + } + scheduler.TraceGpuWork(cmdbuf, pipeline->ShaderHash(), 0, {}, [&] { + cmdbuf.DispatchIndirect(indirect_buffer, indirect_offset); + }); + if (byte_addressed_shadow) { + TextureCacheRuntime::CopyByteAddressedShadow(cmdbuf, *byte_addressed_shadow, true); + } + }); + buffer_cache.NoteWorkRecorded(true); + SubmitNx2ListCopies(); +#ifdef EGLANTINE_STREAMLINE + TryOcarinaDlss(*pipeline, false); // As after a direct dispatch +#endif + return; + } if (indirect_address) { // DispatchIndirect static constexpr auto sync_info = VideoCommon::ObtainBufferSynchronize::FullSynchronize; @@ -2006,6 +2043,52 @@ void RasterizerVulkan::NoteGpuWrittenCommandListSynced(DAddr page) { buffer_cache.Nx2NoteListSynced(page); } +bool RasterizerVulkan::Nx2ListSlotsSupported() const { + // A direct dispatch skips a grid over the device's limits; an indirect one must not get one. + // The slot's height and depth are 16 bits, and a width over 31 bits is no culling result. + const std::array max_dim{device.GetMaxComputeWorkGroupCount()}; + const bool fits = max_dim[0] >= 0x7fffffffU && max_dim[1] >= 0xffffU && max_dim[2] >= 0xffffU; + if (static std::atomic logged{}; !fits && !logged.exchange(true)) { + LOG_WARNING(Render_Vulkan, + "Nx2 list indirect off: the device dispatches at most {}x{}x{} groups", + max_dim[0], max_dim[1], max_dim[2]); + } + return Tegra::Nx2ListIndirect::g_enabled && fits; +} + +std::optional RasterizerVulkan::Nx2AcquireListSlot() { + return buffer_cache_runtime.Nx2Slots().Acquire(); +} + +void RasterizerVulkan::Nx2ReleaseListSlot(u32 slot) { + buffer_cache_runtime.Nx2Slots().Release(slot); +} + +void RasterizerVulkan::Nx2SnapshotListWords(DAddr device_addr, u32 count, u32 slot) { + // The words as GPU work sent before the list left them: the buffer cache's, its CPU writes + // applied (as a dispatch reading them finds them). + gpu_memory->FlushCaching(); + std::scoped_lock lock{buffer_cache.mutex}; + const auto [buffer, offset] = + buffer_cache.ObtainCPUBuffer(device_addr, count * static_cast(sizeof(u32)), + VideoCommon::ObtainBufferSynchronize::FullSynchronize, + VideoCommon::ObtainBufferOperation::DoNothing); + buffer_cache_runtime.Nx2Slots().Snapshot(buffer->Handle(), offset, count, slot); +} + +void RasterizerVulkan::Nx2NoteOwedListWord(DAddr device_addr, u32 value) { + buffer_cache_runtime.Nx2Slots().NoteOwed(device_addr, value, + device_memory.Read(device_addr)); +} + +void RasterizerVulkan::Nx2NoteInlineQmdGrid(DAddr device_addr, u32 x, u32 y, u32 z) { + buffer_cache_runtime.Nx2Slots().NoteInlineQmdGrid(device_addr, x, y, z); +} + +void RasterizerVulkan::Nx2ReadListSlots(std::span slots, std::span words) { + buffer_cache_runtime.Nx2Slots().Read(slots, words); +} + void RasterizerVulkan::FlushGpuWrittenCommandListPages(std::span pages) { gpu.InvalidateGPUCache(); std::scoped_lock lock{buffer_cache.mutex}; diff --git a/src/video_core/renderer_vulkan/vk_rasterizer.h b/src/video_core/renderer_vulkan/vk_rasterizer.h index 94ee9a2f6a..9d89c4a764 100644 --- a/src/video_core/renderer_vulkan/vk_rasterizer.h +++ b/src/video_core/renderer_vulkan/vk_rasterizer.h @@ -112,6 +112,13 @@ class RasterizerVulkan final : public VideoCore::RasterizerInterface, bool past_sync) override; void NoteGpuWrittenCommandListSynced(DAddr page) override; void FlushGpuWrittenCommandListPages(std::span pages) override; + bool Nx2ListSlotsSupported() const override; + std::optional Nx2AcquireListSlot() override; + void Nx2ReleaseListSlot(u32 slot) override; + void Nx2SnapshotListWords(DAddr device_addr, u32 count, u32 slot) override; + void Nx2NoteOwedListWord(DAddr device_addr, u32 value) override; + void Nx2NoteInlineQmdGrid(DAddr device_addr, u32 x, u32 y, u32 z) override; + void Nx2ReadListSlots(std::span slots, std::span words) override; bool MustFlushRegion(DAddr addr, u64 size, VideoCommon::CacheType which = VideoCommon::CacheType::All) override; VideoCore::RasterizerDownloadArea GetFlushArea(DAddr addr, u64 size) override; diff --git a/src/video_core/renderer_vulkan/vk_staging_buffer_pool.cpp b/src/video_core/renderer_vulkan/vk_staging_buffer_pool.cpp index 9a3a122c37..ac89195b6b 100644 --- a/src/video_core/renderer_vulkan/vk_staging_buffer_pool.cpp +++ b/src/video_core/renderer_vulkan/vk_staging_buffer_pool.cpp @@ -95,8 +95,11 @@ StagingBufferPool::StagingBufferPool(const Device& device_, MemoryAllocator& mem .pNext = nullptr, .flags = 0, .size = stream_buffer_size + STREAM_BUFFER_TAIL, - .usage = VK_BUFFER_USAGE_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT | - VK_BUFFER_USAGE_INDEX_BUFFER_BIT | VK_BUFFER_USAGE_STORAGE_BUFFER_BIT, + // Transfer destination too: GPU-written list words are copied over their placeholders + // in the constant buffers streamed here (Nx2ListSlots::PatchUniform). + .usage = VK_BUFFER_USAGE_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT | + VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT | VK_BUFFER_USAGE_INDEX_BUFFER_BIT | + VK_BUFFER_USAGE_STORAGE_BUFFER_BIT, .sharingMode = VK_SHARING_MODE_EXCLUSIVE, .queueFamilyIndexCount = 0, .pQueueFamilyIndices = nullptr, diff --git a/src/video_core/video_core.cpp b/src/video_core/video_core.cpp index 4753d685a7..7667e9c5d8 100644 --- a/src/video_core/video_core.cpp +++ b/src/video_core/video_core.cpp @@ -18,6 +18,7 @@ #include "video_core/nx2_download_trace.h" #include "video_core/nx2_drain_check.h" #include "video_core/nx2_entry_check.h" +#include "video_core/nx2_list_indirect.h" #include "video_core/nx2_list_stats.h" #include "video_core/nx2_perf.h" #include "video_core/nx2_profile.h" @@ -66,6 +67,7 @@ void CreateGPU(std::optional& gpu, Core::Frontend::EmuWindow& emu_wi VideoCommon::Nx2AcquireStats::ReadSwitch(); VideoCommon::Nx2Perf::ReadSwitch(); Tegra::Nx2EntryCheck::ReadSwitch(); + Tegra::Nx2ListIndirect::ReadSwitch(); VideoCommon::ReadUniformStreamReuseSwitch(); const auto nvdec_value = Settings::values.nvdec_emulation.GetValue(); From 9e0cd4f2d7ce3453e5418cb1d393cc9b850c51f5 Mon Sep 17 00:00:00 2001 From: jpkto <255363096+jpkto@users.noreply.github.com> Date: Sun, 8 Nov 2026 09:55:37 +0100 Subject: [PATCH 061/105] TEST: count and check the GPU-written list words the host GPU keeps What Nx2ListIndirect did, logged every 10 s by the front with EDEN_NX2_GPU_PROFILE or EDEN_NX2_LIST_INDIRECT_CHECK ("list indirect"): the GPU-written lists the front took without waiting for their writers, by the compute macro method that takes them, the lists it read as before, by why (3D lists and commands show as not all of a compute macro call's parameters), how often and how long it read list words back after all and why, and on the back the snapshots, indirect dispatches, grids read back, owed words and the dispatches that got them. EDEN_NX2_LIST_INDIRECT_CHECK also has the front read every list it takes as the old path does (waiting for its writers) and send the words with the snapshot. The back then copies what the host GPU used to a host-visible buffer and compares it, once the GPU ran it, with what the old path would have used: the slot's words with the list's, the group counts each indirect dispatch took with the grid the old path dispatches, the words copied over constant buffer placeholders with the list's, the words read back with the list's, and guest memory with each placeholder an inline upload just wrote. "list indirect check" lines give the comparisons and the differences, which must stay 0; the first 32 differences are logged as errors. It also counts the dispatches the old path would have skipped for an empty grid and widths over 31 bits, which differ only on the CPU. Tested on lavapipe under synchronization validation with the slots harness: with the checks on, 40 snapshots, 40 dispatches, 120 patched words, 120 owed words and a read compared, none differing, and no hazard outside the controls; with one frame's expected words deliberately wrong, exactly its snapshot, dispatch and patched word differ and are logged. Syntax checks of video_core and strict warnings on the changed lines pass. Rebased onto upstream: EDEN_NX2_LIST_INDIRECT_CHECK is a plain flag read again by VideoCore::CreateGPU, as upstream reads its diagnostic switches since #714, so it works from nx2_switches.txt too, not only from the environment. Co-Authored-By: Claude Opus 5.5 --- src/video_core/CMakeLists.txt | 1 + src/video_core/dma_pusher.cpp | 37 ++- src/video_core/engines/kepler_compute.cpp | 17 +- src/video_core/engines/maxwell_3d.cpp | 6 + src/video_core/nx2_list_indirect_stats.h | 237 ++++++++++++++++++ src/video_core/rasterizer_interface.h | 5 + .../renderer_vulkan/nx2_list_slots.cpp | 147 ++++++++++- .../renderer_vulkan/nx2_list_slots.h | 48 ++++ .../renderer_vulkan/vk_rasterizer.cpp | 8 + .../renderer_vulkan/vk_rasterizer.h | 1 + src/video_core/video_core.cpp | 2 + 11 files changed, 494 insertions(+), 15 deletions(-) create mode 100644 src/video_core/nx2_list_indirect_stats.h diff --git a/src/video_core/CMakeLists.txt b/src/video_core/CMakeLists.txt index 71c1a9c2fa..4cefa6c760 100644 --- a/src/video_core/CMakeLists.txt +++ b/src/video_core/CMakeLists.txt @@ -123,6 +123,7 @@ add_library(video_core STATIC nx2_drain_check.h nx2_entry_check.h nx2_list_indirect.h + nx2_list_indirect_stats.h nx2_list_stats.h nx2_profile.h nx2_perf.h diff --git a/src/video_core/dma_pusher.cpp b/src/video_core/dma_pusher.cpp index ec0022ef9b..f62b8bdc55 100644 --- a/src/video_core/dma_pusher.cpp +++ b/src/video_core/dma_pusher.cpp @@ -36,6 +36,7 @@ #include "video_core/nx2_drain_check.h" #include "video_core/nx2_entry_check.h" #include "video_core/nx2_list_indirect.h" +#include "video_core/nx2_list_indirect_stats.h" #include "video_core/nx2_list_stats.h" #include "video_core/rasterizer_interface.h" @@ -928,8 +929,13 @@ bool DmaPusher::TakeGpuWrittenListWords(const CommandListHeader& header) { // copy to come back: the macros run on placeholders the host GPU's copy of the words takes // the place of (Nx2ListIndirect, KeplerCompute::FrontTakeListWords). auto& split = VideoCommon::GpuSplit::Get(); + using Kept = Nx2ListIndirect::Stats::Kept; // TEST + if (Nx2ListIndirect::Stats::Enabled()) [[unlikely]] { + Nx2ListIndirect::Stats::MaybeLog(); + } if (!Nx2ListIndirect::g_enabled || rasterizer == nullptr || command_resume_offset != 0 || !split.Active() || !split.FrontMacros()) { + Nx2ListIndirect::Stats::NoteKept(Kept::Off); return false; } const u32 count = static_cast(header.size); @@ -938,10 +944,12 @@ bool DmaPusher::TakeGpuWrittenListWords(const CommandListHeader& header) { dma_state.method < MacroRegistersStart || (dma_state.method & 1) != 0 || subchannels[dma_state.subchannel] == nullptr || subchannel_type[dma_state.subchannel] != Engines::EngineTypes::KeplerCompute) { + Nx2ListIndirect::Stats::NoteKept(Kept::NotCall); return false; } auto* const compute = static_cast(subchannels[dma_state.subchannel]); if (!compute->FrontRunsMacros() || !rasterizer->Nx2ListSlotsSupported()) { + Nx2ListIndirect::Stats::NoteKept(Kept::Off); return false; } // On one device page, as the back finds it: its mappings are the front's once it ran the @@ -950,20 +958,43 @@ bool DmaPusher::TakeGpuWrittenListWords(const CommandListHeader& header) { const auto device = memory_manager.GpuToCpuAddress(dma_state.dma_get); const auto last = memory_manager.GpuToCpuAddress(dma_state.dma_get + bytes - 1); if (!device || !last || *last != *device + bytes - 1 || !split.Ran(split.LastMappingChange())) { + Nx2ListIndirect::Stats::NoteKept(Kept::Placement); return false; } const std::optional slot = rasterizer->Nx2AcquireListSlot(); if (!slot) { + Nx2ListIndirect::Stats::NoteKept(Kept::NoSlot); return false; } if (!compute->FrontTakeListWords(split, dma_state.method, count, *slot)) { rasterizer->Nx2ReleaseListSlot(*slot); // Nothing sent with it + Nx2ListIndirect::Stats::NoteKept(Kept::Refused); return false; } + Nx2ListIndirect::Stats::NoteTaken(dma_state.method, count); + // TEST (EDEN_NX2_LIST_INDIRECT_CHECK): the words as the old path reads them (waiting for + // their writers), for the back to compare what the host GPU uses of the slot with. + const bool check = Nx2ListIndirect::Stats::g_check; + std::array expected{}; + if (check) [[unlikely]] { + if (ReadGpuWrittenCommands(dma_state.dma_get, count)) { + for (u32 k = 0; k < count; ++k) { + expected[k] = command_headers[k].argument; + } + } else { + FlushGpuWrittenCommands(dma_state.dma_get, bytes); + memory_manager.ReadBlockUnsafe(dma_state.dma_get, expected.data(), bytes); + } + } // At the list's place: after the work sent before it, the GPU fetched the list there. - split.Defer([rasterizer_ = rasterizer, device_ = *device, count, - slot_ = *slot] { rasterizer_->Nx2SnapshotListWords(device_, count, slot_); }, - VideoCommon::GpuSplit::Writes::Noted); + split.Defer( + [rasterizer_ = rasterizer, device_ = *device, count, slot_ = *slot, check, expected] { + rasterizer_->Nx2SnapshotListWords(device_, count, slot_); + if (check) { + rasterizer_->Nx2CheckListSlot(slot_, std::span{expected}.first(count)); + } + }, + VideoCommon::GpuSplit::Writes::Noted); command_headers.resize_destructive(count); for (u32 k = 0; k < count; ++k) { command_headers[k].argument = Nx2ListIndirect::Placeholder(*slot, k); diff --git a/src/video_core/engines/kepler_compute.cpp b/src/video_core/engines/kepler_compute.cpp index 18ec0dc77d..d2fbed3a9f 100644 --- a/src/video_core/engines/kepler_compute.cpp +++ b/src/video_core/engines/kepler_compute.cpp @@ -29,6 +29,7 @@ #include "video_core/nx2.h" #include "video_core/nx2_drain_check.h" #include "video_core/nx2_list_indirect.h" +#include "video_core/nx2_list_indirect_stats.h" #include "video_core/engines/fermi_2d.h" #include "video_core/engines/kepler_compute.h" #include "video_core/engines/native_upscale.h" @@ -589,6 +590,7 @@ bool KeplerCompute::DispatchNativeCompute(const NativeQmdScheduler::Descriptor& const u32 slot = Nx2ListIndirect::PlaceholderSlot(resolved[word]); std::array words{}; rasterizer->Nx2ReadListSlots(std::span{&slot, 1}, words); + Nx2ListIndirect::Stats::Count(Nx2ListIndirect::Stats::Get().grids_read); resolved[word] = words[Nx2ListIndirect::PlaceholderWord(resolved[word])]; } } @@ -1608,7 +1610,13 @@ struct ListCallContext { Nx2ListIndirect::ReleaseSlot(split, rasterizer, slot); } void ReadSlots(std::span slots, std::span words) const { + const u64 read_start = + Nx2ListIndirect::Stats::Enabled() ? Nx2ListIndirect::Stats::NowNs() : 0; // TEST Nx2ListIndirect::ReadSlots(split, rasterizer, slots, words); + if (Nx2ListIndirect::Stats::Enabled()) [[unlikely]] { + Nx2ListIndirect::Stats::NoteRead(slots.size(), + Nx2ListIndirect::Stats::NowNs() - read_start); + } } void SetRegister(u32 method, u32 value) const { front_registers[method] = value; @@ -1680,7 +1688,11 @@ void KeplerCompute::ReleaseFront() { front->registers, native_registers, no_emit}; - list_front->ConcretizeAll(context, Nx2ListIndirect::GetChannel(native_data_ram)); + auto& channel = Nx2ListIndirect::GetChannel(native_data_ram); + if (!channel.words.empty() || !list_front->Registers().empty()) { + Nx2ListIndirect::Stats::NoteConcretized(Nx2ListIndirect::Reason::Stop); // TEST + } + list_front->ConcretizeAll(context, channel); } native_executing_macro = front->executing_macro; native_parameters = std::move(front->parameters); @@ -2042,6 +2054,9 @@ void KeplerCompute::FrontRunMacro(Core::System& system, VideoCommon::GpuSplit& s fs.parameters.clear(); return; } + if (concretized) { + Nx2ListIndirect::Stats::NoteConcretized(reason); // TEST + } } #ifdef ARCHITECTURE_x86_64 if (!macro.native_jit) { diff --git a/src/video_core/engines/maxwell_3d.cpp b/src/video_core/engines/maxwell_3d.cpp index ae816214e9..71e03846e0 100644 --- a/src/video_core/engines/maxwell_3d.cpp +++ b/src/video_core/engines/maxwell_3d.cpp @@ -31,6 +31,7 @@ #include "video_core/nx2_cmdpage_diag.h" #include "video_core/nx2_drain_check.h" #include "video_core/nx2_list_indirect.h" +#include "video_core/nx2_list_indirect_stats.h" #include "video_core/nx2_perf.h" #include "video_core/nx2_profile.h" #include "video_core/query_cache/frame_trace.h" @@ -1317,10 +1318,14 @@ void ConcretizeListWords(VideoCommon::GpuSplit& split, VideoCore::RasterizerInte if (!Nx2ListIndirect::MayRead(reads.data_ram_known, reads.data_ram, channel.words)) { return; } + Nx2ListIndirect::Stats::NoteConcretized(Nx2ListIndirect::Reason::Macro3d); // TEST Nx2ListIndirect::ConcretizeDataRam( channel.words, data_ram, [&](std::span slots, std::span words) { + const u64 read_start = Nx2ListIndirect::Stats::NowNs(); // TEST Nx2ListIndirect::ReadSlots(split, rasterizer, slots, words); + Nx2ListIndirect::Stats::NoteRead(slots.size(), + Nx2ListIndirect::Stats::NowNs() - read_start); }, release); } @@ -1356,6 +1361,7 @@ void Maxwell3D::ReleaseFront() { if (auto* channel = Nx2ListIndirect::FindChannel(&native_data_ram); channel != nullptr && !channel->words.empty()) { auto& split = VideoCommon::GpuSplit::Get(); + Nx2ListIndirect::Stats::NoteConcretized(Nx2ListIndirect::Reason::Stop); // TEST Nx2ListIndirect::ConcretizeDataRam( channel->words, std::span{native_data_ram}, [&](std::span slots, std::span words) { diff --git a/src/video_core/nx2_list_indirect_stats.h b/src/video_core/nx2_list_indirect_stats.h new file mode 100644 index 0000000000..7940b02151 --- /dev/null +++ b/src/video_core/nx2_list_indirect_stats.h @@ -0,0 +1,237 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include "common/common_types.h" +#include "common/logging.h" +#include "video_core/nx2_list_indirect.h" +#include "video_core/nx2_profile.h" + +// TEST: what Nx2ListIndirect did, logged every 10 s by the front with EDEN_NX2_GPU_PROFILE or +// EDEN_NX2_LIST_INDIRECT_CHECK: +// - the GPU-written lists the front took without waiting for their writers, by what takes them +// (the compute macro method), and the lists it read as before, by why; +// - the times it read list words back after all (waits the profile counts as "list copy +// writers"), how long it waited, and why; +// - on the back: snapshots, indirect dispatches, grids read back, owed words and the dispatches +// whose constant buffers got them. +// EDEN_NX2_LIST_INDIRECT_CHECK also has the front read every list it takes as before (it waits for +// the list's writers, as the old path does), and the back compare what the host GPU used, read +// back, with what the old path would have used: the slot's words with the list's (snapshot), the +// group counts the dispatch took with the grid the old path dispatches (dispatch), the words +// copied over a constant buffer's placeholders with the list's (patch), the words read back with +// the list's (read), and guest memory with the placeholder an inline upload just wrote there +// (owed: the patch takes over only words guest memory holds). Differences must stay 0; the first +// ones are logged. It counts the dispatches the old path would have skipped (an empty grid: the +// indirect one dispatches no groups) and widths over 31 bits (the old path drops the top bit), +// which differ only on the CPU. +namespace Tegra::Nx2ListIndirect::Stats { + +// A plain flag: checked at every list the host GPU takes over. +inline bool g_check = std::getenv("EDEN_NX2_LIST_INDIRECT_CHECK") != nullptr; + +/// Called by VideoCore::CreateGPU: static initialization runs before the frontend loads +/// nx2_switches.txt, so a switch set only there takes effect from that point on. +inline void ReadSwitch() { + g_check = std::getenv("EDEN_NX2_LIST_INDIRECT_CHECK") != nullptr; +} + +[[nodiscard]] inline bool Enabled() { + return g_check || VideoCommon::Nx2Profile::Enabled(); +} + +/// Why a GPU-written list took the old path (the front read its words). +enum class Kept : u32 { + Off, ///< EDEN_NX2_LIST_INDIRECT=0, the split without macros on its front, no slots + NotCall, ///< Not all the parameters of a compute macro call (3D lists, commands, ...) + Placement, ///< Not on one mapped page, or map changes the back has yet to run + NoSlot, ///< Every slot held + Refused, ///< The engine's front or the call's dry run refused it + Count, +}; +inline constexpr std::array(Kept::Count)> kept_names{ + "off", + "not all of a compute macro call's parameters", + "not on one mapped page or map changes queued", + "no free slot", + "the call's dry run refused", +}; + +inline constexpr std::array(Reason::Count)> reason_names{ + "a macro used one", "a dry run failed", "one went elsewhere", + "macro code or binding", "a 3D macro may read one", "the split stopped", +}; + +/// The back's comparisons (EDEN_NX2_LIST_INDIRECT_CHECK). +enum class Check : u32 { Snapshot, Dispatch, Patch, Read, Owed, Count }; +inline constexpr std::array(Check::Count)> check_names{ + "snapshot", "dispatch", "patch", "read", "owed"}; + +struct Counters { + // The front's. + std::map> taken; ///< By macro method: lists, words + std::array(Kept::Count)> kept{}; + std::array(Reason::Count)> concretized{}; + u64 reads{}; ///< Waits for list words read back + u64 slots_read{}; + u64 read_ns{}; + std::chrono::steady_clock::time_point last_log{std::chrono::steady_clock::now()}; + // The back's. + std::atomic snapshots{}; + std::atomic slot_dispatches{}; + std::atomic grids_read{}; ///< Grids of several slots' words, read back to dispatch + std::atomic owed{}; + std::atomic owed_dropped{}; ///< Owed words guest memory no longer held + std::atomic patched_binds{}; + std::array, static_cast(Check::Count)> compared{}; + std::array, static_cast(Check::Count)> differ{}; + std::atomic unchecked{}; ///< Comparisons dropped: the check buffer was full + std::atomic empty_grids{}; + std::atomic wide_grids{}; + std::atomic differences_logged{}; +}; + +[[nodiscard]] inline Counters& Get() { + static Counters counters; + return counters; +} + +/// The front: a GPU-written list took the old path. +inline void NoteKept(Kept why) { + if (Enabled()) [[unlikely]] { + ++Get().kept[static_cast(why)]; + } +} + +/// The front: a list of `words` words the call of compute macro method `method` takes. +inline void NoteTaken(u32 method, u32 words) { + if (Enabled()) [[unlikely]] { + auto& entry = Get().taken[method]; + ++entry[0]; + entry[1] += words; + } +} + +/// The front read every list word about back after all. +inline void NoteConcretized(Reason reason) { + if (Enabled()) [[unlikely]] { + ++Get().concretized[static_cast(reason)]; + } +} + +/// The front waited `ns` for the words of `slots` slots. +inline void NoteRead(size_t slots, u64 ns) { + if (Enabled()) [[unlikely]] { + ++Get().reads; + Get().slots_read += slots; + Get().read_ns += ns; + } +} + +[[nodiscard]] inline u64 NowNs() { + return static_cast(std::chrono::duration_cast( + std::chrono::steady_clock::now().time_since_epoch()) + .count()); +} + +/// Any thread: `counter` of the back's. +inline void Count(std::atomic& counter, u64 amount = 1) { + if (Enabled()) [[unlikely]] { + counter.fetch_add(amount, std::memory_order_relaxed); + } +} + +/// The back: a comparison of `check` for `slot`, the bytes the host GPU used (`got`) against the +/// ones the old path would have (`expected`). +inline void NoteCompared(Check check, u32 slot, std::span got, + std::span expected) { + auto& counters = Get(); + counters.compared[static_cast(check)].fetch_add(1, std::memory_order_relaxed); + if (std::ranges::equal(got, expected)) { + return; + } + counters.differ[static_cast(check)].fetch_add(1, std::memory_order_relaxed); + if (counters.differences_logged.fetch_add(1, std::memory_order_relaxed) < 32) { + LOG_ERROR(HW_GPU, "Nx2 list indirect check: {} of slot {} differs: {:02x}, expected {:02x}", + check_names[static_cast(check)], slot, fmt::join(got, " "), + fmt::join(expected, " ")); + } +} + +/// The front, every 10 s. +inline void MaybeLog() { + auto& counters = Get(); + const auto now = std::chrono::steady_clock::now(); + const double seconds = std::chrono::duration(now - counters.last_log).count(); + if (seconds < 10.0) { + return; + } + counters.last_log = now; + u64 taken = 0; + u64 taken_words = 0; + std::string by_method; + for (const auto& [method, entry] : counters.taken) { + taken += entry[0]; + taken_words += entry[1]; + by_method += fmt::format("{}compute {:#x} {} ({} words)", by_method.empty() ? "" : ", ", + method, entry[0], entry[1]); + } + u64 kept = 0; + std::string why_kept; + for (size_t i = 0; i < counters.kept.size(); ++i) { + kept += counters.kept[i]; + why_kept += + fmt::format("{}{} {}", why_kept.empty() ? "" : ", ", kept_names[i], counters.kept[i]); + } + std::string why_read; + for (size_t i = 0; i < counters.concretized.size(); ++i) { + why_read += fmt::format("{}{} {}", why_read.empty() ? "" : ", ", reason_names[i], + counters.concretized[i]); + } + const auto take = [](std::atomic& counter) { + return counter.exchange(0, std::memory_order_relaxed); + }; + LOG_INFO(HW_GPU, + "list indirect {:.1f}s: GPU-written lists taken without waiting {} ({} words): {} | " + "read as before {}: {} | list words read back {} times ({} slots, {:.1f} ms), all " + "of them after: {} | back: snapshots {}, indirect dispatches {}, grids read back {}, " + "owed words {} ({} overwritten), dispatches given owed words {}", + seconds, taken, taken_words, by_method.empty() ? "-" : by_method, kept, why_kept, + counters.reads, counters.slots_read, static_cast(counters.read_ns) / 1e6, + why_read, take(counters.snapshots), take(counters.slot_dispatches), + take(counters.grids_read), take(counters.owed), take(counters.owed_dropped), + take(counters.patched_binds)); + if (g_check) { + std::string checks; + for (size_t i = 0; i < counters.compared.size(); ++i) { + checks += fmt::format("{}{} {} ({} differ)", checks.empty() ? "" : ", ", check_names[i], + take(counters.compared[i]), take(counters.differ[i])); + } + LOG_INFO(HW_GPU, + "list indirect check {:.1f}s: compared {}; unchecked {} | dispatches the old " + "path skips (empty grid) {}, widths over 31 bits {}", + seconds, checks, take(counters.unchecked), take(counters.empty_grids), + take(counters.wide_grids)); + } + counters.taken.clear(); + counters.kept = {}; + counters.concretized = {}; + counters.reads = 0; + counters.slots_read = 0; + counters.read_ns = 0; +} + +} // namespace Tegra::Nx2ListIndirect::Stats diff --git a/src/video_core/rasterizer_interface.h b/src/video_core/rasterizer_interface.h index d269a78aad..baf287caf0 100644 --- a/src/video_core/rasterizer_interface.h +++ b/src/video_core/rasterizer_interface.h @@ -174,6 +174,11 @@ class RasterizerInterface { /// The back: the words of `slots`, three each, read back from the host GPU (waits for it). virtual void Nx2ReadListSlots([[maybe_unused]] std::span slots, [[maybe_unused]] std::span words) {} + /// TEST (EDEN_NX2_LIST_INDIRECT_CHECK), the back, right after Nx2SnapshotListWords: the + /// words the front read as before for that list, for what the host GPU uses of the slot to + /// be compared with. + virtual void Nx2CheckListSlot([[maybe_unused]] u32 slot, + [[maybe_unused]] std::span words) {} virtual void FlushGpuWrittenCommandListPages(std::span pages) { for (const DAddr page : pages) { diff --git a/src/video_core/renderer_vulkan/nx2_list_slots.cpp b/src/video_core/renderer_vulkan/nx2_list_slots.cpp index d424b62483..265cc42446 100644 --- a/src/video_core/renderer_vulkan/nx2_list_slots.cpp +++ b/src/video_core/renderer_vulkan/nx2_list_slots.cpp @@ -2,6 +2,7 @@ // SPDX-License-Identifier: GPL-3.0-or-later #include +#include #include "common/logging.h" #include "video_core/renderer_vulkan/nx2_list_slots.h" @@ -10,6 +11,16 @@ namespace Vulkan { +namespace Stats = Tegra::Nx2ListIndirect::Stats; // TEST + +namespace { +/// TEST: the bytes of `words`. +template +[[nodiscard]] std::span Bytes(std::span words) { + return {reinterpret_cast(words.data()), words.size_bytes()}; +} +} // Anonymous namespace + void Nx2ListSlots::Release(u32 slot) { if (slot >= SlotCount) { return; @@ -26,6 +37,7 @@ void Nx2ListSlots::Snapshot(VkBuffer src, u64 src_offset, u32 count, u32 slot) { return; } Prepare(); + Stats::Count(Stats::Get().snapshots); // TEST const VkDeviceSize base = VkDeviceSize{slot} * SlotBytes; copies.clear(); copies.push_back(VkBufferCopy{src_offset, base, VkDeviceSize{count} * 4}); @@ -46,9 +58,15 @@ void Nx2ListSlots::NoteOwed(DAddr device_addr, u32 value, u32 guest_word) { Release(Tegra::Nx2ListIndirect::PlaceholderSlot(it->second)); it = owed.erase(it); } + if (Stats::g_check && Tegra::Nx2ListIndirect::IsPlaceholder(value)) [[unlikely]] { + // TEST: the upload wrote the placeholder at once (a word it did not write is not owed). + Stats::NoteCompared(Stats::Check::Owed, Tegra::Nx2ListIndirect::PlaceholderSlot(value), + Bytes(std::span{&guest_word, 1}), Bytes(std::span{&value, 1})); + } if (guest_word != value || !Tegra::Nx2ListIndirect::IsPlaceholder(value)) { return; // It wrote nothing there } + Stats::Count(Stats::Get().owed); // TEST holds[Tegra::Nx2ListIndirect::PlaceholderSlot(value)].fetch_add(1, std::memory_order_acq_rel); owed.emplace(device_addr, value); } @@ -118,17 +136,7 @@ void Nx2ListSlots::Read(std::span slots, std::span words) { VkDeviceSize{i} * SlotWordBytes, SlotWordBytes}); } RecordCopies(*buffer, *readback, copies); - scheduler.Record([](vk::CommandBuffer cmdbuf) { - // A fence makes device writes available, not visible to the host. - static constexpr VkMemoryBarrier HOST_BARRIER{ - .sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER, - .pNext = nullptr, - .srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT, - .dstAccessMask = VK_ACCESS_HOST_READ_BIT, - }; - cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_HOST_BIT, 0, - HOST_BARRIER); - }); + RecordHostBarrier(); scheduler.Finish(); readback.Invalidate(); const size_t bytes = @@ -136,6 +144,20 @@ void Nx2ListSlots::Read(std::span slots, std::span words) { std::memcpy(words.data() + first * Tegra::Nx2ListIndirect::SlotWords, readback.Mapped().data(), bytes); } + if (Stats::g_check) [[unlikely]] { + // TEST: the words read back against the list's, and what the checks copied. + for (size_t i = 0; i < slots.size() && (i + 1) * SlotWordBytes <= words.size_bytes(); ++i) { + const u32 slot = slots[i]; + if (slot < SlotCount && check_counts[slot] != 0) { + const size_t known = check_counts[slot]; + Stats::NoteCompared( + Stats::Check::Read, slot, + Bytes(words.subspan(i * Tegra::Nx2ListIndirect::SlotWords, known)), + Bytes(std::span{check_words[slot]}.first(known))); + } + } + CheckFlush(); + } } void Nx2ListSlots::Prepare() { @@ -193,4 +215,107 @@ void Nx2ListSlots::RecordCopies(VkBuffer src, VkBuffer dst, std::span words) { + if (slot >= SlotCount || words.empty() || words.size() > Tegra::Nx2ListIndirect::SlotWords || + !buffer) { + return; + } + check_words[slot] = {}; + std::ranges::copy(words, check_words[slot].begin()); + check_counts[slot] = static_cast(words.size()); + CheckRecord(Stats::Check::Snapshot, slot, *buffer, VkDeviceSize{slot} * SlotBytes, + Bytes(words)); + if (check_entries.size() >= CheckEntries / 2) { + CheckFlush(); // At the list's place on the back: between methods + } +} + +void Nx2ListSlots::CheckDispatch(u32 slot) { + if (slot >= SlotCount || check_counts[slot] != Tegra::Nx2ListIndirect::SlotWords) { + return; + } + const auto& words = check_words[slot]; + // The grid DispatchNativeCompute dispatches from the words (the width as the slot has it). + const std::array grid{words[0], words[1] & 0xffff, words[2] & 0xffff}; + if (!(grid[0] && grid[1] && grid[2])) { + Stats::Count(Stats::Get().empty_grids); + } + if (grid[0] >> 31) { + Stats::Count(Stats::Get().wide_grids); + } + CheckRecord(Stats::Check::Dispatch, slot, *buffer, + VkDeviceSize{slot} * SlotBytes + DispatchOffset, Bytes(std::span{grid})); +} + +void Nx2ListSlots::CheckPatch(VkBuffer dst) { + for (size_t i = 0; i < copies.size() && i < patch_words.size(); ++i) { + const u32 slot = Tegra::Nx2ListIndirect::PlaceholderSlot(patch_words[i].value); + const u32 word = Tegra::Nx2ListIndirect::PlaceholderWord(patch_words[i].value); + if (slot >= SlotCount || word >= check_counts[slot]) { + continue; + } + const auto bytes = Bytes(std::span{&check_words[slot][word], 1}) + .subspan(patch_words[i].skip, copies[i].size); + CheckRecord(Stats::Check::Patch, slot, dst, copies[i].dstOffset, bytes); + } +} + +void Nx2ListSlots::CheckRecord(Stats::Check check, u32 slot, VkBuffer src, VkDeviceSize src_offset, + std::span expected) { + if (check_entries.size() >= CheckEntries || expected.size() > CheckEntryBytes) { + Stats::Count(Stats::Get().unchecked); + return; + } + if (!check_buffer) { + check_buffer = memory_allocator.CreateBuffer( + VkBufferCreateInfo{ + .sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, + .pNext = nullptr, + .flags = 0, + .size = VkDeviceSize{CheckEntries} * CheckEntryBytes, + .usage = VK_BUFFER_USAGE_TRANSFER_DST_BIT, + .sharingMode = VK_SHARING_MODE_EXCLUSIVE, + .queueFamilyIndexCount = 0, + .pQueueFamilyIndices = nullptr, + }, + MemoryUsage::Download); + } + CheckEntry entry{check, slot, static_cast(expected.size()), {}}; + std::memcpy(entry.expected.data(), expected.data(), expected.size()); + const VkBufferCopy copy{src_offset, VkDeviceSize{check_entries.size()} * CheckEntryBytes, + expected.size()}; + check_entries.push_back(entry); + RecordCopies(src, *check_buffer, std::span{©, 1}); +} + +void Nx2ListSlots::CheckFlush() { + if (check_entries.empty()) { + return; + } + RecordHostBarrier(); + scheduler.Finish(); + check_buffer.Invalidate(); + const std::span data = std::as_const(check_buffer).Mapped(); + for (size_t i = 0; i < check_entries.size(); ++i) { + const CheckEntry& entry = check_entries[i]; + Stats::NoteCompared(entry.check, entry.slot, data.subspan(i * CheckEntryBytes, entry.bytes), + std::span{entry.expected}.first(entry.bytes)); + } + check_entries.clear(); +} + } // namespace Vulkan diff --git a/src/video_core/renderer_vulkan/nx2_list_slots.h b/src/video_core/renderer_vulkan/nx2_list_slots.h index c25c73f5bd..c9cc6d4c6d 100644 --- a/src/video_core/renderer_vulkan/nx2_list_slots.h +++ b/src/video_core/renderer_vulkan/nx2_list_slots.h @@ -15,6 +15,7 @@ #include "common/common_types.h" #include "video_core/nx2_list_indirect.h" +#include "video_core/nx2_list_indirect_stats.h" #include "video_core/vulkan_common/vulkan_wrapper.h" namespace Vulkan { @@ -87,12 +88,15 @@ class Nx2ListSlots { return; } copies.clear(); + patch_words.clear(); // TEST const DAddr end = device_addr + size; for (auto it = owed.lower_bound(device_addr >= 3 ? device_addr - 3 : 0); it != owed.end() && it->first < end;) { const DAddr word = it->first; const u32 value = it->second; if (read(word) != value) { + Tegra::Nx2ListIndirect::Stats::Count( + Tegra::Nx2ListIndirect::Stats::Get().owed_dropped); // TEST Release(Tegra::Nx2ListIndirect::PlaceholderSlot(value)); it = owed.erase(it); continue; @@ -106,12 +110,18 @@ class Nx2ListSlots { (first - word); copies.push_back( VkBufferCopy{source, offset + (first - device_addr), last - first}); + patch_words.push_back({value, static_cast(first - word)}); // TEST } ++it; } if (!copies.empty()) { ++patched_binds; + Tegra::Nx2ListIndirect::Stats::Count( + Tegra::Nx2ListIndirect::Stats::Get().patched_binds); // TEST RecordCopies(*buffer, dst, copies); + if (Tegra::Nx2ListIndirect::Stats::g_check) [[unlikely]] { + CheckPatch(dst); + } } } @@ -129,6 +139,13 @@ class Nx2ListSlots { return patched_binds; } + /// TEST (EDEN_NX2_LIST_INDIRECT_CHECK): the words the front read as before for the list + /// whose snapshot `slot` just got. What the host GPU then uses of the slot is compared with + /// them once it ran (CheckFlush): the snapshot, the dispatches from it, the patches. + void CheckSnapshot(u32 slot, std::span words); + /// TEST: the dispatch from `slot` just recorded. + void CheckDispatch(u32 slot); + private: static constexpr u32 NoEntry = ~0U; static constexpr u32 SlotWordBytes = Tegra::Nx2ListIndirect::SlotWords * sizeof(u32); @@ -142,6 +159,31 @@ class Nx2ListSlots { /// writes may be read by an earlier dispatch or copy), before every read and write after. void RecordCopies(VkBuffer src, VkBuffer dst, std::span regions); + /// Records a barrier that makes transfer writes visible to the host, after a fence wait. + void RecordHostBarrier(); + + // TEST (EDEN_NX2_LIST_INDIRECT_CHECK): copies of what the host GPU used, compared with what + // the old path would have used once it ran them. + struct CheckEntry { + Tegra::Nx2ListIndirect::Stats::Check check; + u32 slot; + u32 bytes; + std::array expected; + }; + struct PatchWord { + u32 value; ///< The owed placeholder + u32 skip; ///< Bytes of it before the binding + }; + static constexpr u32 CheckEntries = 1024; + static constexpr u32 CheckEntryBytes = 16; + /// Records a copy of the `expected.size()` bytes at `src_offset` of `src`, for CheckFlush. + void CheckRecord(Tegra::Nx2ListIndirect::Stats::Check check, u32 slot, VkBuffer src, + VkDeviceSize src_offset, std::span expected); + /// The patch just recorded into `dst` (copies, patch_words). + void CheckPatch(VkBuffer dst); + /// Waits for the host GPU and compares what CheckRecord copied. + void CheckFlush(); + MemoryAllocator& memory_allocator; Scheduler& scheduler; vk::Buffer buffer; @@ -152,6 +194,12 @@ class Nx2ListSlots { std::unordered_map> qmd_slots; ///< By QMD device address std::vector copies; u64 patched_binds{}; + // TEST (EDEN_NX2_LIST_INDIRECT_CHECK). + std::vector patch_words; + vk::Buffer check_buffer; + std::vector check_entries; + std::array, SlotCount> check_words{}; + std::array check_counts{}; ///< Words of check_words known (0: none) }; } // namespace Vulkan diff --git a/src/video_core/renderer_vulkan/vk_rasterizer.cpp b/src/video_core/renderer_vulkan/vk_rasterizer.cpp index 746241eb0e..51d5864cf6 100644 --- a/src/video_core/renderer_vulkan/vk_rasterizer.cpp +++ b/src/video_core/renderer_vulkan/vk_rasterizer.cpp @@ -1873,6 +1873,10 @@ void RasterizerVulkan::DispatchCompute() { TextureCacheRuntime::CopyByteAddressedShadow(cmdbuf, *byte_addressed_shadow, true); } }); + Tegra::Nx2ListIndirect::Stats::Count(Tegra::Nx2ListIndirect::Stats::Get().slot_dispatches); + if (Tegra::Nx2ListIndirect::Stats::g_check) [[unlikely]] { + slots.CheckDispatch(*list_slot); // TEST + } buffer_cache.NoteWorkRecorded(true); SubmitNx2ListCopies(); #ifdef EGLANTINE_STREAMLINE @@ -2089,6 +2093,10 @@ void RasterizerVulkan::Nx2ReadListSlots(std::span slots, std::span words) { + buffer_cache_runtime.Nx2Slots().CheckSnapshot(slot, words); // TEST +} + void RasterizerVulkan::FlushGpuWrittenCommandListPages(std::span pages) { gpu.InvalidateGPUCache(); std::scoped_lock lock{buffer_cache.mutex}; diff --git a/src/video_core/renderer_vulkan/vk_rasterizer.h b/src/video_core/renderer_vulkan/vk_rasterizer.h index 9d89c4a764..285e2d4f91 100644 --- a/src/video_core/renderer_vulkan/vk_rasterizer.h +++ b/src/video_core/renderer_vulkan/vk_rasterizer.h @@ -119,6 +119,7 @@ class RasterizerVulkan final : public VideoCore::RasterizerInterface, void Nx2NoteOwedListWord(DAddr device_addr, u32 value) override; void Nx2NoteInlineQmdGrid(DAddr device_addr, u32 x, u32 y, u32 z) override; void Nx2ReadListSlots(std::span slots, std::span words) override; + void Nx2CheckListSlot(u32 slot, std::span words) override; // TEST bool MustFlushRegion(DAddr addr, u64 size, VideoCommon::CacheType which = VideoCommon::CacheType::All) override; VideoCore::RasterizerDownloadArea GetFlushArea(DAddr addr, u64 size) override; diff --git a/src/video_core/video_core.cpp b/src/video_core/video_core.cpp index 7667e9c5d8..eb91e49a58 100644 --- a/src/video_core/video_core.cpp +++ b/src/video_core/video_core.cpp @@ -19,6 +19,7 @@ #include "video_core/nx2_drain_check.h" #include "video_core/nx2_entry_check.h" #include "video_core/nx2_list_indirect.h" +#include "video_core/nx2_list_indirect_stats.h" #include "video_core/nx2_list_stats.h" #include "video_core/nx2_perf.h" #include "video_core/nx2_profile.h" @@ -68,6 +69,7 @@ void CreateGPU(std::optional& gpu, Core::Frontend::EmuWindow& emu_wi VideoCommon::Nx2Perf::ReadSwitch(); Tegra::Nx2EntryCheck::ReadSwitch(); Tegra::Nx2ListIndirect::ReadSwitch(); + Tegra::Nx2ListIndirect::Stats::ReadSwitch(); VideoCommon::ReadUniformStreamReuseSwitch(); const auto nvdec_value = Settings::values.nvdec_emulation.GetValue(); From f3702f3b97b0897f48ffdd8cc082ec3cb29a353c Mon Sep 17 00:00:00 2001 From: jpkto <255363096+jpkto@users.noreply.github.com> Date: Sun, 8 Nov 2026 10:33:30 +0100 Subject: [PATCH 062/105] cmake: reuse a cached CI package when its checksum can't be downloaded AddCIPackage downloads the artifact's sha512sum on every configure, and fails if it can't. The checksum only verifies a new download: when the artifact is already extracted in the package cache, fetch_package returns before it is used. So a GitHub outage (release assets answering 504) or no network stopped every re-configure, even with everything needed already on disk. When the checksum download fails and the cache holds the artifact, as AddPackage would find it (same cache path and patch key), the configure now goes on with a warning naming the cached copy. Without a cached copy it fails as before. Tested with a CMake script running the changed block with the real get_cache_path and needs_refetch: without the cached artifact it stops with the old error; with it, it warns and goes on. Co-Authored-By: Claude Opus 5.5 --- CMakeModules/CPMUtil.cmake | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/CMakeModules/CPMUtil.cmake b/CMakeModules/CPMUtil.cmake index a628e3fc89..45f89614a4 100644 --- a/CMakeModules/CPMUtil.cmake +++ b/CMakeModules/CPMUtil.cmake @@ -1055,12 +1055,26 @@ function(AddCIPackage) list(GET sha512sum_status 0 sha512sum_error) if(sha512sum_error) - message(FATAL_ERROR "[CPMUtil] Failed to download sha512sum " - "for ${ARTIFACT_NAME} from ${sha512sum_url}") + # The checksum only verifies a new download (fetch_package). When the + # artifact is already extracted in the package cache, as AddPackage would + # find it, a failed checksum download (GitHub down, no network) need not + # stop the configure. + get_cache_path(${ARTIFACT_PACKAGE} "${ARTIFACT_VERSION}-${pkgname}" + cached_path) + needs_refetch(${cached_path} "none" cached_invalid) + if (cached_invalid) + file(REMOVE "${sha512sum_file}") + message(FATAL_ERROR "[CPMUtil] Failed to download sha512sum " + "for ${ARTIFACT_NAME} from ${sha512sum_url}") + endif() + message(WARNING "[CPMUtil] Failed to download sha512sum for " + "${ARTIFACT_NAME} from ${sha512sum_url}; using the cached " + "${ARTIFACT} in ${cached_path}") + set(sha512sum_hash "cached") + else() + file(READ "${sha512sum_file}" sha512sum_hash) + string(STRIP "${sha512sum_hash}" sha512sum_hash) endif() - - file(READ "${sha512sum_file}" sha512sum_hash) - string(STRIP "${sha512sum_hash}" sha512sum_hash) file(REMOVE "${sha512sum_file}") AddPackage( From 611a09d4a03f5c28c2ad7ebacd57e58854851edb Mon Sep 17 00:00:00 2001 From: jpkto <255363096+jpkto@users.noreply.github.com> Date: Sun, 8 Nov 2026 13:37:14 +0100 Subject: [PATCH 063/105] video_core: have the QMD view read a descriptor only once the back ran its writes The user's verify run of f3702f3b97 logged one wrong QMD note in 3.3 million, in plain Hyrule Field gameplay: QMD view check: the back read other descriptors | 0x21143a9c00 by the back, 0x21143a9c00 by the front, word 54 0x70ce for 0x0 QMD view check: the back released 0x2b010=0x8305 0xd6d2008=0x70ce | noted 0x2b010=0x8305 0xd6d2008=0x0 Word 54 is the payload of the QMD's queue sync release (slot 2, words 52-54). Ocarina of Time writes its QMDs inline with that payload still 0. A few ring words before the decrement that launches a QMD, it patches the payload with an inline upload (KeplerCompute 0x6d). The front notes that upload's range, and the view waits for it about a thousand times a second ("a write into the QMD queued ... by KeplerCompute 0x6d"). FrontQmdView::ReadDescriptor read the QMD from guest memory first. Only then did it look whether the back had run the last write sent there (Ran of MemoryWritesTarget). The patch is often published just before the decrement: the back waits for work, so Commit hands it the words after eight. While the front values the decrement, the back writes the payload and stores its position. Suppose the front reads before that write and looks after that store, slowed or preempted in between. Then it takes the old payload for what the back reads. The log shows the back that close: signals that waited for the patch were logged with the back already past it ("sent 7 ring words before it; the back is 5 words behind"). ReadDescriptor now reads memory only after Ran saw the back past the write. The back stores its position after the writes before it, so the read holds them. Late valuing goes through the same function. Nothing else changes: the view still waits, values late or loses its table for the same reasons. The test line (oot-amd-test49m2) has the same code. A stale descriptor gives the view an old payload, address or dependency for the back's. Here the note was 0 where the back released 0x70ce, so greater-or-equal acquires on that word would only have waited longer for it. An older payload or address could also pass an acquire early or skew the view's table. Tested: - QMD harness: a new scripted case does the game's patch and launch. The back runs what is published at every look the view takes at memory, right after its read too. Before this change, the case logs "the back read other descriptors | 0x100000 by the back, 0x100000 by the front, word 54 0x70ce for 0x0" and "the back released 0x108000=0x8305 0x108020=0x70ce | noted 0x108000=0x8305 0x108020=0x0", the shape of the user's lines. Two new fuzz modes race the back the same way. Before this change they fail: the QMD ring with patches has 7,836 releases noted otherwise, 9,386 descriptors read otherwise and 9 tables differing in 200 runs; all writes has 66, 99 and 3. With this change every mode passes with EDEN_NX2_DRAIN_CHECK=1 and EDEN_NX2_ACQUIRE_STATS=1 and without them. The new modes value 375,162 and 524,138 methods at once, nothing released or read otherwise, 40,241 tables equal. The older modes give the counts they gave before. The race mode, whose back runs right before each read, values 126 of 594,126 fewer at once: the old code valued those because the back ran between its read and its look. - Split harness: 20 runs on 2 CPUs and 10 pinned to one passed. The copy and upload write range harness passed. - The compile check, with -fno-rtti, passes on kepler_compute.cpp and gpu_thread.cpp, the users of front_qmd.h. clang's strict warnings report nothing on the changed lines. Co-Authored-By: Claude Opus 5.5 --- src/video_core/front_qmd.h | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/video_core/front_qmd.h b/src/video_core/front_qmd.h index e4ddd0fe96..c99d99eb4d 100644 --- a/src/video_core/front_qmd.h +++ b/src/video_core/front_qmd.h @@ -481,7 +481,6 @@ class FrontQmdView { GPUVAddr qmd_address, u64 position, Descriptor& descriptor) const { using Lost = Nx2AcquireStats::QmdLost; - memory.Read(qmd_address, descriptor.data(), sizeof(descriptor)); const auto device = memory.Device(qmd_address); if (!device || memory.GpuWritten(*device, sizeof(descriptor))) { return Outcome{false, std::nullopt, Lost::GpuWritten}; @@ -494,6 +493,11 @@ class FrontQmdView { return Outcome{false, std::nullopt, Lost::WrittenAfter, qmd_address}; } if (split.Ran(target)) { + // Read only once Ran saw the back past the write: the back stores its position after + // the writes before it. A read before that look may be older than the write. The back + // can run a write published shortly before the signal (to an idle back, after a few + // words) while the front reads, and store its position before the front looks. + memory.Read(qmd_address, descriptor.data(), sizeof(descriptor)); return std::nullopt; } if (const InlineWrite* written = QueuedInlineWrite(split, *device); From 0a6bd6e98b34d66d8262adb5163f86c0675c0ad9 Mon Sep 17 00:00:00 2001 From: Burak Date: Sun, 8 Nov 2026 15:56:57 +0300 Subject: [PATCH 064/105] vulkan: recycle images instead of creating and freeing them every frame (Ocarina of Time) Ocarina of Time creates and frees ~8 render targets a frame: its per-frame allocator moves them around and its shadow map changes size. Creating, binding and freeing each took ~0.2 ms of the GPU thread (dedicated allocations are kernel calls). Released images on device-local memory are now kept, keyed by their create info, and an image with the same create info takes one instead of allocating: up to 512 MiB, for 10 seconds. Images dropped from the pool are destroyed on a thread of their own, so the VMA allocator is now synchronized internally. The texture cache counts the kept memory as free, and while it is past the critical memory the pool gives back half of what it keeps each frame. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01CqQ96AP4a2yY5zMzWNrFdE --- .../renderer_vulkan/vk_texture_cache.cpp | 8 + .../renderer_vulkan/vk_texture_cache.h | 5 + src/video_core/texture_cache/texture_cache.h | 10 + .../vulkan_common/vulkan_device.cpp | 5 +- .../vulkan_common/vulkan_memory_allocator.cpp | 15 ++ .../vulkan_common/vulkan_wrapper.cpp | 227 ++++++++++++++++++ src/video_core/vulkan_common/vulkan_wrapper.h | 34 ++- 7 files changed, 302 insertions(+), 2 deletions(-) diff --git a/src/video_core/renderer_vulkan/vk_texture_cache.cpp b/src/video_core/renderer_vulkan/vk_texture_cache.cpp index 7afcbdb515..c17c0d43fb 100644 --- a/src/video_core/renderer_vulkan/vk_texture_cache.cpp +++ b/src/video_core/renderer_vulkan/vk_texture_cache.cpp @@ -2838,6 +2838,14 @@ u64 TextureCacheRuntime::GetDeviceMemoryUsage() const { return device.GetDeviceMemoryUsage(); } +u64 TextureCacheRuntime::GetRecycledImageMemory() const { + return vk::ImageRecycler::KeptBytes(); +} + +void TextureCacheRuntime::ShrinkRecycledImages(u64 bytes) { + vk::ImageRecycler::Shrink(bytes); +} + u64 TextureCacheRuntime::CurrentTick() const noexcept { return scheduler.CurrentTick(); } diff --git a/src/video_core/renderer_vulkan/vk_texture_cache.h b/src/video_core/renderer_vulkan/vk_texture_cache.h index 5888bf5e57..51a2e9552c 100644 --- a/src/video_core/renderer_vulkan/vk_texture_cache.h +++ b/src/video_core/renderer_vulkan/vk_texture_cache.h @@ -101,6 +101,11 @@ class TextureCacheRuntime { u64 GetDeviceMemoryUsage() const; + /// Device memory the image recycler keeps (vk::ImageRecycler): the collector counts it as + /// free, and when memory runs short the recycler gives it back first. + u64 GetRecycledImageMemory() const; + void ShrinkRecycledImages(u64 bytes); + /// GPU progress: the texture cache frees a deleted image once the GPU is past its last use. [[nodiscard]] u64 CurrentTick() const noexcept; diff --git a/src/video_core/texture_cache/texture_cache.h b/src/video_core/texture_cache/texture_cache.h index 680986f5b2..c04be34866 100644 --- a/src/video_core/texture_cache/texture_cache.h +++ b/src/video_core/texture_cache/texture_cache.h @@ -407,6 +407,16 @@ void TextureCache

::TickFrame() { total_used_memory -= (std::min)(total_used_memory, bytes); } total_used_memory -= (std::min)(total_used_memory, gpu_sentenced_bytes); + // Images the Vulkan recycler keeps are free memory to the collector: counted, a larger + // pool pushed it into high priority (more deletes and write-backs). With them past the + // critical memory, the pool gives back half of what it keeps each frame. + if constexpr (requires { runtime.GetRecycledImageMemory(); }) { + const u64 kept = runtime.GetRecycledImageMemory(); + total_used_memory -= (std::min)(total_used_memory, kept); + if (kept != 0 && total_used_memory + kept >= critical_memory) { + runtime.ShrinkRecycledImages(kept / 2); + } + } } // Every 600 frames: device memory in use against the collector's thresholds. // Skip expensive heap allocation, map construction, and sorting unless debug logging is explicitly enabled. diff --git a/src/video_core/vulkan_common/vulkan_device.cpp b/src/video_core/vulkan_common/vulkan_device.cpp index b2c5468a8f..60c739675b 100644 --- a/src/video_core/vulkan_common/vulkan_device.cpp +++ b/src/video_core/vulkan_common/vulkan_device.cpp @@ -798,7 +798,9 @@ Device::Device(VkInstance instance_, vk::PhysicalDevice physical_, VkSurfaceKHR functions.vkGetInstanceProcAddr = dld.vkGetInstanceProcAddr; functions.vkGetDeviceProcAddr = dld.vkGetDeviceProcAddr; - VmaAllocatorCreateFlags flags = VMA_ALLOCATOR_CREATE_EXTERNALLY_SYNCHRONIZED_BIT; + // Synchronized inside VMA: the image recycler destroys dropped images on a thread of its own + // while the GPU thread allocates (vulkan_wrapper.cpp). + VmaAllocatorCreateFlags flags = 0; if (extensions.memory_budget) { flags |= VMA_ALLOCATOR_CREATE_EXT_MEMORY_BUDGET_BIT; } @@ -861,6 +863,7 @@ Device::~Device() { } } } + vk::ImageRecycler::Clear(allocator); vmaDestroyAllocator(allocator); } diff --git a/src/video_core/vulkan_common/vulkan_memory_allocator.cpp b/src/video_core/vulkan_common/vulkan_memory_allocator.cpp index 7fd03f4a37..b282ce3bd9 100644 --- a/src/video_core/vulkan_common/vulkan_memory_allocator.cpp +++ b/src/video_core/vulkan_common/vulkan_memory_allocator.cpp @@ -243,6 +243,15 @@ VkResult MemoryAllocator::AllocateImage(const VkImageCreateInfo &ci, bool dedica VkImage handle{}; VmaAllocation allocation{}; VmaAllocationInfo alloc_info{}; + // An image of the same create info released before is taken from the recycler instead + // (vk::ImageRecycler). + const u64 recycle_key = vk::ImageRecycler::KeyOf(ci, dedicated); + if (recycle_key != 0 && vk::ImageRecycler::Take(recycle_key, allocator, handle, allocation)) { + image = vk::Image(handle, ci.usage, *device.GetLogical(), allocator, allocation, + device.GetDispatchLoader()); + image.SetRecycleKey(recycle_key); + return VK_SUCCESS; + } // Images this large (title 1's sparse texture arrays reach 4.5G texels, ~23 GB) are never // backed: VMA would quietly place them in system memory once VRAM is full. They take the // unbacked sparse fallback below. @@ -323,6 +332,12 @@ VkResult MemoryAllocator::AllocateImage(const VkImageCreateInfo &ci, bool dedica image = vk::Image(handle, ci.usage, *device.GetLogical(), allocator, allocation, device.GetDispatchLoader()); + // Images placed in host memory under memory pressure are freed rather than kept. + if (recycle_key != 0 && alloc_info.memoryType < properties.memoryTypeCount && + (properties.memoryTypes[alloc_info.memoryType].propertyFlags & + VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT) != 0) { + image.SetRecycleKey(recycle_key); + } return VK_SUCCESS; } diff --git a/src/video_core/vulkan_common/vulkan_wrapper.cpp b/src/video_core/vulkan_common/vulkan_wrapper.cpp index 6eb0d9e37e..bdcaabf7d8 100644 --- a/src/video_core/vulkan_common/vulkan_wrapper.cpp +++ b/src/video_core/vulkan_common/vulkan_wrapper.cpp @@ -5,13 +5,18 @@ // SPDX-License-Identifier: GPL-2.0-or-later #include +#include +#include #include +#include #include +#include #include #include #include "common/common_types.h" #include "common/logging.h" +#include "common/thread.h" #include "video_core/vulkan_common/vk_enum_string_helper.h" #include "video_core/vulkan_common/vma.h" #include "video_core/vulkan_common/vulkan_wrapper.h" @@ -525,10 +530,232 @@ void Image::SetObjectNameEXT(const char* name) const { void Image::Release() const noexcept { if (handle) { + if (recycle_key != 0 && allocation != nullptr && + ImageRecycler::Put(recycle_key, allocator, handle, allocation)) { + return; + } vmaDestroyImage(allocator, handle, allocation); } } +namespace { +struct RecycledImage { + u64 key; + VmaAllocator allocator; + VkImage handle; + VmaAllocation allocation; + u64 size; + std::chrono::steady_clock::time_point kept; +}; + +struct RecyclerState { + std::mutex mutex; + std::vector images; // oldest first + u64 bytes = 0; + VmaAllocator closed{}; ///< Torn down: keeps nothing more of it +}; + +RecyclerState& Recycler() { + static RecyclerState state; + return state; +} + +/// What the recycler keeps at most, and how long a kept image waits to be taken again. +constexpr u64 RecyclerCap = u64{512} << 20; +constexpr std::chrono::seconds RecycleKeepTime{10}; + +/// Images dropped from the recycler are destroyed on a thread of their own: freeing dedicated +/// memory is a kernel call (~0.2 ms), which the GPU thread would make otherwise. +struct Graveyard { + std::mutex mutex; + std::condition_variable cv; + std::vector images; + std::mutex destroy_mutex; ///< Held while a batch is destroyed (Clear waits for it) + bool started = false; +}; + +Graveyard& Reaper() { + // Never destroyed: its thread may still wait on it while the process exits. + static Graveyard* const graveyard = new Graveyard; + return *graveyard; +} + +void ReaperLoop(Graveyard& graveyard) { + std::vector batch; + while (true) { + std::unique_lock destroy_lock{graveyard.destroy_mutex, std::defer_lock}; + { + std::unique_lock lock{graveyard.mutex}; + graveyard.cv.wait(lock, [&graveyard] { return !graveyard.images.empty(); }); + // Taken with the batch, so Clear sees either the images or a batch in progress. + destroy_lock.lock(); + batch.swap(graveyard.images); + } + for (const RecycledImage& image : batch) { + vmaDestroyImage(image.allocator, image.handle, image.allocation); + } + batch.clear(); + } +} + +void Bury(std::vector&& images) { + Graveyard& graveyard = Reaper(); + { + std::scoped_lock lock{graveyard.mutex}; + if (!graveyard.started) { + graveyard.started = true; + std::thread{[&graveyard] { + Common::SetCurrentThreadName("VulkanReaper"); + ReaperLoop(graveyard); + }}.detach(); + } + graveyard.images.insert(graveyard.images.end(), images.begin(), images.end()); + } + graveyard.cv.notify_one(); +} + +/// Drops kept images that are too old, or the oldest until `room` more bytes fit the cap. +void TrimRecycler(RecyclerState& state, u64 room, std::chrono::steady_clock::time_point now) { + size_t drop = 0; + u64 bytes = state.bytes; + while (drop < state.images.size() && + (now - state.images[drop].kept > RecycleKeepTime || bytes + room > RecyclerCap)) { + bytes -= state.images[drop].size; + ++drop; + } + if (drop == 0) { + return; + } + const auto first = state.images.begin(); + const auto last = first + static_cast(drop); + Bury(std::vector(first, last)); + state.images.erase(first, last); + state.bytes = bytes; +} +} // Anonymous namespace + +u64 ImageRecycler::KeyOf(const VkImageCreateInfo& ci, bool dedicated) noexcept { + constexpr u64 Mul = 0x100000001b3ULL; + u64 key = 0xcbf29ce484222325ULL; + const auto mix = [&key](u64 value) { key = (key ^ value) * Mul; }; + mix(ci.flags); + mix(ci.imageType); + mix(ci.format); + mix((u64{ci.extent.width} << 32) | ci.extent.height); + mix(ci.extent.depth); + mix((u64{ci.mipLevels} << 32) | ci.arrayLayers); + mix(ci.samples); + mix(ci.tiling); + mix(ci.usage); + mix(ci.sharingMode); + mix(ci.initialLayout); + mix(dedicated ? 1 : 0); + if (ci.sharingMode == VK_SHARING_MODE_CONCURRENT) { + for (u32 index = 0; index < ci.queueFamilyIndexCount; ++index) { + mix(ci.pQueueFamilyIndices[index]); + } + } + for (auto* next = static_cast(ci.pNext); next != nullptr; + next = next->pNext) { + if (next->sType != VK_STRUCTURE_TYPE_IMAGE_FORMAT_LIST_CREATE_INFO) { + return 0; + } + const auto* list = reinterpret_cast(next); + mix(list->viewFormatCount); + for (u32 index = 0; index < list->viewFormatCount; ++index) { + mix(list->pViewFormats[index]); + } + } + return key == 0 ? 1 : key; +} + +bool ImageRecycler::Put(u64 key, VmaAllocator allocator, VkImage handle, + VmaAllocation allocation) noexcept { + VmaAllocationInfo info{}; + vmaGetAllocationInfo(allocator, allocation, &info); + if (info.size > RecyclerCap) { + return false; + } + RecyclerState& state = Recycler(); + std::scoped_lock lock{state.mutex}; + if (state.closed == allocator) { + return false; + } + const auto now = std::chrono::steady_clock::now(); + TrimRecycler(state, info.size, now); + state.images.push_back({key, allocator, handle, allocation, info.size, now}); + state.bytes += info.size; + return true; +} + +bool ImageRecycler::Take(u64 key, VmaAllocator allocator, VkImage& handle, + VmaAllocation& allocation) noexcept { + if (key == 0) { + return false; + } + RecyclerState& state = Recycler(); + std::scoped_lock lock{state.mutex}; + // Looked for before old images are dropped: after a few seconds without new images (the + // camera standing still), the first images of the next scene found theirs dropped just then. + bool found = false; + for (size_t index = state.images.size(); index-- > 0;) { + const RecycledImage& image = state.images[index]; + if (image.key == key && image.allocator == allocator) { + handle = image.handle; + allocation = image.allocation; + state.bytes -= image.size; + state.images.erase(state.images.begin() + static_cast(index)); + found = true; + break; + } + } + TrimRecycler(state, 0, std::chrono::steady_clock::now()); + return found; +} + +u64 ImageRecycler::KeptBytes() noexcept { + RecyclerState& state = Recycler(); + std::scoped_lock lock{state.mutex}; + return state.bytes; +} + +void ImageRecycler::Shrink(u64 bytes) noexcept { + RecyclerState& state = Recycler(); + std::scoped_lock lock{state.mutex}; + if (state.bytes <= bytes) { + return; + } + // Kept bytes never pass the cap, so here bytes < cap: dropping the oldest until + // `cap - bytes` more fit leaves at most `bytes`. + TrimRecycler(state, RecyclerCap - bytes, std::chrono::steady_clock::now()); +} + +void ImageRecycler::Clear(VmaAllocator allocator) noexcept { + RecyclerState& state = Recycler(); + std::scoped_lock lock{state.mutex}; + state.closed = allocator; + std::erase_if(state.images, [&state, allocator](const RecycledImage& image) { + if (image.allocator != allocator) { + return false; + } + vmaDestroyImage(image.allocator, image.handle, image.allocation); + state.bytes -= image.size; + return true; + }); + // And the dropped ones the reaper has not destroyed yet, before the allocator goes. + Graveyard& graveyard = Reaper(); + std::scoped_lock graveyard_lock{graveyard.mutex}; + std::erase_if(graveyard.images, [allocator](const RecycledImage& image) { + if (image.allocator != allocator) { + return false; + } + vmaDestroyImage(image.allocator, image.handle, image.allocation); + return true; + }); + // A batch being destroyed: wait for it. + std::scoped_lock destroy_lock{graveyard.destroy_mutex}; +} + Image::MemoryBlock Image::GetMemoryBlock() const noexcept { if (allocation == nullptr) { return {}; diff --git a/src/video_core/vulkan_common/vulkan_wrapper.h b/src/video_core/vulkan_common/vulkan_wrapper.h index 7b94bb97fe..422b91cec0 100644 --- a/src/video_core/vulkan_common/vulkan_wrapper.h +++ b/src/video_core/vulkan_common/vulkan_wrapper.h @@ -705,7 +705,8 @@ class Image { Image(Image&& rhs) noexcept : handle{std::exchange(rhs.handle, VkImage{})}, usage{rhs.usage}, owner{rhs.owner}, - allocator{rhs.allocator}, allocation{rhs.allocation}, dld{rhs.dld} {} + allocator{rhs.allocator}, allocation{rhs.allocation}, dld{rhs.dld}, + recycle_key{rhs.recycle_key} {} Image& operator=(Image&& rhs) noexcept { Release(); @@ -715,9 +716,16 @@ class Image { allocator = rhs.allocator; allocation = rhs.allocation; dld = rhs.dld; + recycle_key = rhs.recycle_key; return *this; } + /// Released into ImageRecycler (for an image of the same create info) instead of + /// destroyed; 0 for images that are not recycled. + void SetRecycleKey(u64 key) noexcept { + recycle_key = key; + } + ~Image() noexcept { Release(); } @@ -759,6 +767,30 @@ class Image { VmaAllocator allocator = nullptr; VmaAllocation allocation = nullptr; const DeviceDispatch* dld = nullptr; + u64 recycle_key = 0; +}; + +/// Images on device-local memory, kept after release so an image of the same create info takes +/// one instead of allocating: Ocarina of Time makes and drops ~8 render targets a frame (its +/// per-frame allocator moves them, its shadow map changes size), and making, binding and freeing +/// each took ~0.2 ms of the GPU thread. Up to 512 MiB, for 10 seconds. +class ImageRecycler { +public: + /// Key of a create info and its allocation request, or 0 when it cannot be recycled (an + /// unknown pNext structure). + [[nodiscard]] static u64 KeyOf(const VkImageCreateInfo& ci, bool dedicated) noexcept; + /// Keeps the image for a later Take; false when it is not kept (the caller destroys it). + static bool Put(u64 key, VmaAllocator allocator, VkImage handle, + VmaAllocation allocation) noexcept; + /// An image kept under this key, if any. + static bool Take(u64 key, VmaAllocator allocator, VkImage& handle, + VmaAllocation& allocation) noexcept; + /// Destroys every kept image of this allocator and keeps nothing more (allocator teardown). + static void Clear(VmaAllocator allocator) noexcept; + /// Device memory the kept images hold (free memory to the texture cache's collector). + [[nodiscard]] static u64 KeptBytes() noexcept; + /// Drops the oldest kept images until at most `bytes` are kept. + static void Shrink(u64 bytes) noexcept; }; struct MemoryLocation { From eb64bfbf87dedfd8c6224f0cc78ebb9982eb1ace Mon Sep 17 00:00:00 2001 From: jpkto <255363096+jpkto@users.noreply.github.com> Date: Sun, 8 Nov 2026 14:31:44 +0100 Subject: [PATCH 065/105] TEST: read download check: leave out landed bytes newer writes replaced The Hyrule Field verify run of 611a09d4a0 reported one byte as Outdated (guest memory 0x30473f8f held 0x46, the value an earlier download landed, the check copy of the host buffer 0xbf). Its log's version line names f3702f3b97 because it is written at configure time; the build included #778, whose staging buffer change (the upload pool's idle sweep, still gated on IsFree) and fence manager change (profile logging) don't touch read downloads. The check reaches that result in three ways while guest memory holds what a synchronous flush leaves, all in the check and not in the downloads: - An early download landed the byte, so it was no longer GPU-modified. A guest write applied later (WriteMemory) reached no ClearDownload, so early_landed kept the byte with its landed value. Written to another value that a draw uploaded, then back to the landed one before the next upload, guest memory equals the landed value while the host buffer holds the upload's: the read's check took it for a GPU write the early download missed. Without early downloads that write takes the byte out of the GPU-modified ones and the read's flush leaves it. - A query result copy (SyncValues) gives landed bytes a new value in the host buffer that the cache hears of only as outdating downloads. Guest memory gets the result once the query cache writes it there, and a synchronous flush doesn't download those bytes either. - A later read's check kept bytes an earlier download brings. That download landed alone and kept the bytes of a guest write still collected (PendingCpuWriteRanges), but NoteLandedBytes noted guest memory there as landed. Once the write was applied it no longer counted as collected, so the later check compared the guest's value with the GPU's as if the download had brought it. WriteMemory and OutdateReadDownloads now forget early landed bytes they change, as ClearDownload does for GPU-modified ones, and NoteLandedBytes takes the bytes a collected write kept out of the later check (changed), as CheckReadDownload does for the download's own check. For MarkWrittenBuffer's calls of OutdateReadDownloads nothing changes: its bytes are GPU-modified from then on, which TakeEarlyLanded leaves out, and every way out of GPU-modified (ClearDownload, WriteDownloadedMemory) forgets them too. All of this is EDEN_NX2_READ_CHECK state only. Guest writes over early landed bytes no longer count as "rewritten" (no read compares those bytes any more); that total keeps the writers the buffer cache doesn't hear of, and guest writes over bytes a read's check holds already. Ruled out on the way: the downloads' own copies record their barriers (with d612710df3, b62f49104d and 8b5be87f79 on this branch), uploads leave out GPU-modified bytes, and reordered uploads can't reach bytes still on their way. Tests: three buffer_cache_ports cases, one for each way, each reporting exactly that Outdated line without the change. Read download harness: all cases pass. Co-Authored-By: Claude Opus 5.5 --- src/tests/video_core/buffer_cache_ports.cpp | 104 ++++++++++++++++++ src/video_core/buffer_cache/buffer_cache.h | 28 ++++- .../buffer_cache/buffer_cache_base.h | 6 +- 3 files changed, 134 insertions(+), 4 deletions(-) diff --git a/src/tests/video_core/buffer_cache_ports.cpp b/src/tests/video_core/buffer_cache_ports.cpp index fe6311603c..246ff590e2 100644 --- a/src/tests/video_core/buffer_cache_ports.cpp +++ b/src/tests/video_core/buffer_cache_ports.cpp @@ -289,6 +289,11 @@ struct BufferCacheTestAccess { static void Download(Cache& cache, std::span> ranges) { cache.DownloadRangesMemory(ranges); } + /// A draw or dispatch binds [address, address + size) of buffer `id`: its CPU-modified bytes + /// are uploaded. + static void Upload(Cache& cache, BufferId id, DAddr address, u32 size) { + (void)cache.SynchronizeBuffer(cache.slot_buffers[id], address, size); + } static void Uniform(Cache& cache, BufferCacheChannelInfo& channel, BufferId id, DAddr address, u32 size) { cache.channel_state = &channel; @@ -1831,3 +1836,102 @@ TEST_CASE("Guest writes racing read downloads keep their bytes", "[buffer_ports] REQUIRE(Access::ReadChecks(*f.cache).mismatched == 0); REQUIRE(Access::ReadChecks(*f.cache).uncovered == 0); } + +TEST_CASE("Read download check leaves out early downloaded bytes the guest wrote since", + "[buffer_ports]") { + Fixture f; + const Nx2TitleScope nx2_title; + const auto id = Access::Add(*f.cache, 0x20000, 0x10000); + auto& buffer = Access::Get(*f.cache, id); + Access::CheckEveryRead(*f.cache); + for (u8 frame = 1; frame <= 2; ++frame) { + WriteByDispatch(f, id, 0x20000, 0x100, frame); + ReadWord(f, 0x20010); + } + WriteByDispatch(f, id, 0x20000, 0x100, 3); + f.runtime.Complete(f.runtime.submitted_tick); + f.cache->NoteWorkRecorded(true); + REQUIRE(Access::ReadDownloads(*f.cache) == 0); + // The guest writes 16 of the landed bytes, as GPU::InvalidateGPUCache applies it, and a draw + // uploads them. Then it writes the landed value back, which no upload has taken yet. + const std::array, 1> written{{{0x20040, 0x10}}}; + const auto guest_write = [&](u8 value) { + std::memset(f.bytes + 0x10040, value, 0x10); + f.cache->ClaimCpuWrites(written); + f.cache->WriteMemory(0x20040, 0x10); + }; + guest_write(0xee); + Access::Upload(*f.cache, id, 0x20000, 0x1000); + REQUIRE(buffer.bytes[0x40] == 0xee); + guest_write(3); + // A synchronous flush at the read leaves them to the guest, as the other landed bytes are + // what it would bring. + REQUIRE_FALSE(ReadWord(f, 0x20010).preemtive); + REQUIRE(f.bytes[0x10040] == 3); + REQUIRE(f.bytes[0x10010] == 3); + REQUIRE(Access::ReadChecks(*f.cache).compared != 0); + REQUIRE(Access::ReadChecks(*f.cache).mismatched == 0); + REQUIRE(Access::ReadChecks(*f.cache).uncovered == 0); +} + +TEST_CASE("Read download check leaves out a guest write an earlier read's landing kept", + "[buffer_ports]") { + Fixture f; + const auto id = Access::Add(*f.cache, 0x20000, 0x10000); + std::ranges::fill(Access::Get(*f.cache, id).bytes, 0x12); + Access::CheckEveryRead(*f.cache); + GuestCpuWrites writes{*f.cache}; + // A read's download, then a read whose bytes it brings: that read's check copy comes alone, + // in the next submission. + Access::MarkWritten(*f.cache, id, 0x20000, 0x100); + f.cache->DownloadMemoryForRead(0x20000, 0x1000); + f.cache->DownloadMemoryForRead(0x20000, 0x1000); + REQUIRE(Access::ReadDownloads(*f.cache) == 2); + REQUIRE(Access::ReadChecks(*f.cache).alone_recorded == 1); + // The guest writes into those bytes. The first download lands at a fence and keeps the + // write, collected and not yet applied; then the write is applied, before the second lands. + writes.Write(f.bytes + 0x10040, 0x20040, NewGuestValue); + f.runtime.Complete(f.runtime.submitted_tick - 1); + f.cache->PopAsyncFlushes(); + REQUIRE(Access::ReadDownloads(*f.cache) == 1); + writes.Apply(); + f.cache->WriteMemory(0x20040, 0x40); + f.cache->FinishReadDownloads(0x20000, 0x1000); + REQUIRE(Access::ReadDownloads(*f.cache) == 0); + REQUIRE(GuestHolds(f, 0x20040, NewGuestValue)); + REQUIRE(f.bytes[0x1003f] == 0x12); + REQUIRE(f.bytes[0x10080] == 0x12); + REQUIRE(Access::ReadChecks(*f.cache).checked == 2); + REQUIRE(Access::ReadChecks(*f.cache).mismatched == 0); + REQUIRE(Access::ReadChecks(*f.cache).uncovered == 0); +} + +TEST_CASE("Read download check leaves out early downloaded bytes a query result copy replaced", + "[buffer_ports]") { + Fixture f; + const Nx2TitleScope nx2_title; + const auto id = Access::Add(*f.cache, 0x20000, 0x10000); + auto& buffer = Access::Get(*f.cache, id); + Access::CheckEveryRead(*f.cache); + for (u8 frame = 1; frame <= 2; ++frame) { + WriteByDispatch(f, id, 0x20000, 0x100, frame); + ReadWord(f, 0x20010); + } + WriteByDispatch(f, id, 0x20000, 0x100, 3); + f.runtime.Complete(f.runtime.submitted_tick); + f.cache->NoteWorkRecorded(true); + REQUIRE(Access::ReadDownloads(*f.cache) == 0); + // A query result copied into landed bytes on the GPU (SyncValues), which the buffer cache + // hears of only as outdating downloads. The query cache writes guest memory after this read, + // and a synchronous flush at the read leaves guest memory as it is. + f.runtime.pending.push_back( + [&buffer] { std::fill_n(buffer.bytes.begin() + 0x40, 8, u8{0x56}); }); + f.cache->OutdateReadDownloads(0x20040, 8); + REQUIRE_FALSE(ReadWord(f, 0x20010).preemtive); + REQUIRE(buffer.bytes[0x40] == 0x56); + REQUIRE(f.bytes[0x10040] == 3); + REQUIRE(f.bytes[0x10048] == 3); + REQUIRE(Access::ReadChecks(*f.cache).compared != 0); + REQUIRE(Access::ReadChecks(*f.cache).mismatched == 0); + REQUIRE(Access::ReadChecks(*f.cache).uncovered == 0); +} diff --git a/src/video_core/buffer_cache/buffer_cache.h b/src/video_core/buffer_cache/buffer_cache.h index b2af92ff81..a3d71b88f0 100644 --- a/src/video_core/buffer_cache/buffer_cache.h +++ b/src/video_core/buffer_cache/buffer_cache.h @@ -200,6 +200,14 @@ void BufferCache

::UnmapGPUMemory(size_t as_id, GPUVAddr gpu_addr, size_t size template void BufferCache

::WriteMemory(DAddr device_addr, u64 size, bool keep_page_gpu_data) { Nx2NoteListHostWrite(device_addr, size); + // TEST (EDEN_NX2_READ_CHECK): a newer value where an early download landed, as ClearDownload + // forgets one over GPU-modified bytes: without early downloads the write had taken the bytes + // out of the GPU-modified ones, and the read's synchronous flush leaves them. Landed bytes + // are no longer GPU-modified, so the write reaches no ClearDownload: written back to the + // landed value after an upload of another, they read as GPU bytes the download missed. + if (!early_landed.empty()) [[unlikely]] { + ForgetEarlyLanded(device_addr, size); + } // The written range lies within its pages: when they hold no GPU-written data, both lookups // below find nothing, so one lookup answers them. if (keep_page_gpu_data) { @@ -851,6 +859,14 @@ void BufferCache

::FinishReadDownloads(DAddr device_addr, u64 size) { template void BufferCache

::OutdateReadDownloads(DAddr device_addr, u64 size) { + // TEST (EDEN_NX2_READ_CHECK): the GPU's copy gets newer bytes where an early download landed, + // which no read compares. A query result copy (SyncValues) is no GPU write the cache records: + // a synchronous flush doesn't download it, and guest memory gets the result once the query + // cache writes it there. MarkWrittenBuffer's bytes are GPU-modified from now on, which + // TakeEarlyLanded leaves out, and every way out of that forgets them too. + if (!early_landed.empty()) [[unlikely]] { + ForgetEarlyLanded(device_addr, size); + } // The bytes stay GPU-modified when the downloads land, so a later read downloads them. if (!read_downloads.empty()) [[unlikely]] { for (ReadDownload& download : read_downloads) { @@ -1195,7 +1211,8 @@ void BufferCache

::WriteReadDownloads(u64 tick) { // TEST: what later reads' checks find of an earlier download where they kept its bytes. for (size_t later = 1; later < read_downloads.size(); ++later) { if (read_downloads[later].check) [[unlikely]] { - NoteLandedBytes(*read_downloads[later].check, download.write_ranges); + NoteLandedBytes(*read_downloads[later].check, download.write_ranges, + cpu_written); } } if (download.check) [[unlikely]] { @@ -1436,7 +1453,8 @@ void BufferCache

::ExplainReadCheckDifference(const ReadDownload& download, DA } template -void BufferCache

::NoteLandedBytes(ReadCheck& check, const Common::RangeSet& written) { +void BufferCache

::NoteLandedBytes(ReadCheck& check, const Common::RangeSet& written, + const Common::RangeSet& cpu_written) { if (check.kept.Empty()) { return; } @@ -1447,6 +1465,12 @@ void BufferCache

::NoteLandedBytes(ReadCheck& check, const Common::RangeSet& written); + /// A download landed `written`, but for the guest CPU writes collected then (`cpu_written`): + /// what guest memory holds now over a later check's `kept`. + void NoteLandedBytes(ReadCheck& check, const Common::RangeSet& written, + const Common::RangeSet& cpu_written); /// A DownloadMemoryForRead: its copies, and what is left of them to do once the host GPU has /// run them. From f9f3bc6394baf9430307029f89f7e238faf5d1b9 Mon Sep 17 00:00:00 2001 From: jpkto <255363096+jpkto@users.noreply.github.com> Date: Sun, 8 Nov 2026 14:57:13 +0100 Subject: [PATCH 066/105] TEST: explain Outdated read download check results ExplainReadCheckDifference explains a byte this download brings that differs (Differs). An Outdated byte, which an earlier download brought, was reported in one line that cannot tell the check's own artifacts from a GPU write the downloads missed or a corrupted host buffer, so the one byte of the Hyrule Field verify run of 611a09d4a0 could not be placed. Behind EDEN_NX2_READ_CHECK, the buffer cache now keeps a ring of the last 4096 changes to buffer bytes while a read download is on its way or early downloaded bytes wait for their read: GPU writes with their GpuWaitStats write site (MarkWrittenBuffer, DMACopy), GPU writes it records only by outdating downloads (query result copies), DMA clears, guest memory writes (WriteMemory), uploads (SynchronizeBuffer) and inline uploads. The ring is sized by its first event, so without the check the cache carries an empty vector and each site costs the flag test. Each download notes the ring's position at its copy, and each landed byte of a check where it came from: the early download that landed it before the read came (a), or the download on its way at the read that landed it (b), with that download's tick and positions. An Outdated byte gets a second line: the path and the earlier download's tick and kind, the changes over the byte since that download's copy (before or after it landed, and before or after this read's copy), whether a guest write or an upload came after it landed, the newest GPU-side guest memory write noted there (ReadCheckWrites), the tracker's CPU- and GPU-modified state of the page, whether the byte is GPU-modified in the cache, and whether a guest write is collected there now. Tests: read download harness, all cases pass. Both existing cases that report Outdated (one per path) print the line; with the previous commit's change taken out, the new cases print it with the guest writes and the upload, or the query result copy, that caused them. Co-Authored-By: Claude Opus 5.5 --- src/video_core/buffer_cache/buffer_cache.h | 143 +++++++++++++++++- .../buffer_cache/buffer_cache_base.h | 66 +++++++- 2 files changed, 200 insertions(+), 9 deletions(-) diff --git a/src/video_core/buffer_cache/buffer_cache.h b/src/video_core/buffer_cache/buffer_cache.h index a3d71b88f0..0534f2168b 100644 --- a/src/video_core/buffer_cache/buffer_cache.h +++ b/src/video_core/buffer_cache/buffer_cache.h @@ -208,6 +208,7 @@ void BufferCache

::WriteMemory(DAddr device_addr, u64 size, bool keep_page_gpu if (!early_landed.empty()) [[unlikely]] { ForgetEarlyLanded(device_addr, size); } + NoteReadCheckEvent(device_addr, size, ReadCheckEvent::GuestWrite); // TEST // The written range lies within its pages: when they hold no GPU-written data, both lookups // below find nothing, so one lookup answers them. if (keep_page_gpu_data) { @@ -559,6 +560,7 @@ void BufferCache

::RecordReadDownload(std::span> r boost::container::small_vector copies; ReadDownload download; download.early = early; + download.events_before = read_check_event_count; // TEST u64 total_size_bytes = 0; const auto add_copy = [&](BufferId buffer_id, DAddr buffer_addr, DAddr begin, DAddr end) { copies.push_back({buffer_id, BufferCopy{ @@ -683,7 +685,16 @@ void BufferCache

::RecordReadDownload(std::span> r }; boost::container::small_vector landed_copies; if (check && !early && !early_landed.empty()) [[unlikely]] { - const auto add_landed = [&](DAddr lower, DAddr upper, const u8* as_landed) { + const auto add_landed = [&](DAddr lower, DAddr upper, const u8* as_landed, + const EarlyLanded& entry) { + for (const auto& from : entry.from) { + if (from.begin < upper && lower < from.end) { + auto origin = from; + origin.begin = (std::max)(from.begin, lower); + origin.end = (std::min)(from.end, upper); + check->origins.push_back(origin); + } + } ForEachBufferInRange(lower, upper - lower, [&](BufferId buffer_id, Buffer& buffer) { const DAddr buffer_addr = buffer.CpuAddr(); const DAddr begin = (std::max)(lower, buffer_addr); @@ -859,6 +870,7 @@ void BufferCache

::FinishReadDownloads(DAddr device_addr, u64 size) { template void BufferCache

::OutdateReadDownloads(DAddr device_addr, u64 size) { + NoteReadCheckEvent(device_addr, size, ReadCheckEvent::Unrecorded); // TEST // TEST (EDEN_NX2_READ_CHECK): the GPU's copy gets newer bytes where an early download landed, // which no read compares. A query result copy (SyncValues) is no GPU write the cache records: // a synchronous flush doesn't download it, and guest memory gets the result once the query @@ -1097,6 +1109,7 @@ void BufferCache

::NoteEarlyLanded(const ReadDownload& download, if (early_landed.size() >= 4 * MaxReadBackPages) { early_landed.clear(); } + const u64 landed_at = read_check_event_count; download.clean_ranges.ForEach([&](DAddr start, DAddr end) { Common::RangeSet landed; landed.Add(start, end - start); @@ -1113,6 +1126,18 @@ void BufferCache

::NoteEarlyLanded(const ReadDownload& download, device_memory.ReadBlockUnsafe(from, entry.as_landed.data() + (from - page), to - from); entry.bytes.Add(from, to - from); + if (entry.from.size() >= 8) { + entry.from.erase(entry.from.begin()); + } + entry.from.push_back({ + .begin = from, + .end = to, + .tick = download.tick, + .copied = download.events_before, + .landed = landed_at, + .early = true, + .before_read = true, + }); } }); }); @@ -1135,7 +1160,7 @@ void BufferCache

::TakeEarlyLanded(DAddr device_addr, u64 size, Func&& func) { taken.Subtract(lower, upper - lower); }); taken.ForEach([&](DAddr lower, DAddr upper) { - func(lower, upper, it->second.as_landed.data() + (lower - page)); + func(lower, upper, it->second.as_landed.data() + (lower - page), it->second); it->second.bytes.Subtract(lower, upper - lower); }); if (it->second.bytes.Empty()) { @@ -1211,8 +1236,7 @@ void BufferCache

::WriteReadDownloads(u64 tick) { // TEST: what later reads' checks find of an earlier download where they kept its bytes. for (size_t later = 1; later < read_downloads.size(); ++later) { if (read_downloads[later].check) [[unlikely]] { - NoteLandedBytes(*read_downloads[later].check, download.write_ranges, - cpu_written); + NoteLandedBytes(*read_downloads[later].check, download, cpu_written); } } if (download.check) [[unlikely]] { @@ -1322,6 +1346,8 @@ void BufferCache

::CheckReadDownload(const ReadDownload& download, address, size, found, wanted, why); if (result == Result::Differs) { ExplainReadCheckDifference(download, address, size, found, wanted); + } else if (result == Result::Outdated) { + ExplainReadCheckOutdated(download, address, size); } }; std::vector guest; @@ -1453,11 +1479,110 @@ void BufferCache

::ExplainReadCheckDifference(const ReadDownload& download, DA } template -void BufferCache

::NoteLandedBytes(ReadCheck& check, const Common::RangeSet& written, +void BufferCache

::ExplainReadCheckOutdated(const ReadDownload& download, DAddr address, + u64 size) { + if constexpr (!IS_OPENGL) { + // The newest download noted as bringing it: early_landed before this read came (a), or + // one on its way at this read that landed before it (b). + const typename ReadCheck::Origin* origin = nullptr; + for (const auto& entry : download.check->origins) { + if (address >= entry.begin && address < entry.end) { + origin = &entry; + } + } + std::string path = "no download noted as bringing it"; + u64 since = download.events_before; + u64 landed_at = download.events_before; + if (origin != nullptr) { + path = fmt::format("{} {} download of tick {}", + origin->before_read ? "(a) landed before this read came, by" + : "(b) on its way at this read, landed by", + origin->early ? "an early" : "a read's", origin->tick); + since = origin->copied; + landed_at = origin->landed; + } + // What wrote it since that download's copy, oldest first. + static constexpr std::array kinds{ + "GPU write", "unrecorded GPU write (query result copy)", + "DMA clear", "guest write", + "upload", "inline upload", + }; + const u64 total = read_check_event_count; + const u64 ring = read_check_events.size(); + // No ring yet: no event was noted (NoteReadCheckEvent sizes it first). + const u64 first = + ring == 0 ? total : (std::max)(since, total > ring ? total - ring : u64{0}); + std::string events; + u64 listed = 0; + u64 uploads = 0; + bool guest_write_since = false; + bool upload_since = false; + for (u64 index = first; index < total; ++index) { + const ReadCheckEventEntry& event = read_check_events[index % ring]; + if (event.begin >= address + size || address >= event.end) { + continue; + } + const bool after_landing = event.sequence > landed_at; + const bool upload = + event.kind == ReadCheckEvent::Upload || event.kind == ReadCheckEvent::Inline; + uploads += upload ? 1 : 0; + upload_since = upload_since || (upload && after_landing); + guest_write_since = + guest_write_since || (event.kind == ReadCheckEvent::GuestWrite && after_landing); + if (listed++ >= 8) { + continue; + } + events += fmt::format( + "{}{}{} ({} it landed, {} this read's copy)", events.empty() ? "" : ", ", + kinds[static_cast(event.kind)], + event.kind == ReadCheckEvent::GpuWrite + ? fmt::format(" by {}", + GpuWaitStats::write_site_names[static_cast(event.site)]) + : std::string{}, + after_landing ? "after" : "before", + event.sequence > download.events_before ? "after" : "before"); + } + if (listed > 8) { + events += fmt::format(" and {} more", listed - 8); + } + const DAddr page = Common::AlignDown(address, DEVICE_PAGESIZE); + LOG_ERROR(HW_GPU, + "Read download check: {:#x}+{:#x} is outdated: {}; compared by {} download of " + "tick {} (now {}); since that copy over it: {}{}; a guest write since it " + "landed: {}; uploads since that copy: {}, since it landed: {}; newest GPU-side " + "guest memory write noted there: {}; page {:#x} in the tracker: CPU-modified " + "{}, GPU-modified {}; GPU-modified in the cache: {}; a guest CPU write " + "collected there now: {}", + address, size, path, download.early ? "an early" : "a read's", download.tick, + runtime.CurrentTick(), events.empty() ? "nothing noted" : events, + since < first ? " (older events lost)" : "", guest_write_since ? "yes" : "no", + uploads, upload_since ? "yes" : "no", + VideoCore::ReadCheckWrites::Find(address, size), page, + memory_tracker.IsRegionCpuModified(page, DEVICE_PAGESIZE) ? "yes" : "no", + memory_tracker.IsRegionGpuModified(page, DEVICE_PAGESIZE) ? "yes" : "no", + gpu_modified_ranges.Intersects(address, size) ? "yes" : "no", + PendingCpuWriteRanges().Intersects(address, size) ? "yes" : "no"); + } +} + +template +void BufferCache

::NoteLandedBytes(ReadCheck& check, const ReadDownload& download, const Common::RangeSet& cpu_written) { if (check.kept.Empty()) { return; } + const Common::RangeSet& written = download.write_ranges; + const auto note_origin = [&](DAddr start, DAddr end) { + check.origins.push_back({ + .begin = start, + .end = end, + .tick = download.tick, + .copied = download.events_before, + .landed = read_check_event_count, + .early = download.early, + .before_read = false, + }); + }; for (const BufferCopy& copy : check.copies) { const DAddr copy_addr = static_cast(copy.src_offset); u8* const landed = check.landed_bytes.data() + (copy.dst_offset - check.staging.offset); @@ -1465,6 +1590,7 @@ void BufferCache

::NoteLandedBytes(ReadCheck& check, const Common::RangeSet::DMACopy(GPUVAddr src_address, GPUVAddr dest_address, u64 am ClearDownload(*cpu_dest_address, amount); MarkVertexBuffersStale(*cpu_dest_address, amount); + NoteReadCheckEvent(*cpu_dest_address, amount, ReadCheckEvent::GpuWrite, + GpuWaitStats::WriteSite::DmaCopy); // TEST BufferId buffer_a; BufferId buffer_b; @@ -1607,6 +1735,7 @@ bool BufferCache

::DMAClear(GPUVAddr dst_address, u64 amount, u32 value) { FinishReadDownloads(*cpu_dst_address, size); ClearDownload(*cpu_dst_address, size); gpu_modified_ranges.Subtract(*cpu_dst_address, size); + NoteReadCheckEvent(*cpu_dst_address, size, ReadCheckEvent::DmaClear); // TEST // As an inline write: command list pages read the cleared words from guest memory. Nx2NoteListHostWrite(*cpu_dst_address, size); @@ -3227,6 +3356,7 @@ void BufferCache

::MarkWrittenBuffer(BufferId buffer_id, DAddr device_addr, u3 for (auto& ranges : pending_download_ranges) { ranges.Subtract(device_addr, size); } + NoteReadCheckEvent(device_addr, size, ReadCheckEvent::GpuWrite, site); // TEST OutdateReadDownloads(device_addr, size); NoteReadBackWrite(device_addr, size); memory_tracker.MarkRegionAsGpuModified(device_addr, size); @@ -3796,6 +3926,7 @@ bool BufferCache

::SynchronizeBuffer(Buffer& buffer, DAddr device_addr, u32 si const auto add_upload = [&](DAddr start, DAddr end) { if (start == end) return; const u64 range_size = end - start; + NoteReadCheckEvent(start, range_size, ReadCheckEvent::Upload); // TEST upload_copies.push_back(BufferCopy{ .src_offset = total_size_bytes, .dst_offset = start - buffer_start, @@ -3830,6 +3961,7 @@ bool BufferCache

::SynchronizeBuffer(Buffer& buffer, DAddr device_addr, u32 si u64 largest_copy = 0; DAddr buffer_start = buffer.CpuAddr(); for_each_upload_range([&](u64 device_addr_out, u64 range_size) { + NoteReadCheckEvent(device_addr_out, range_size, ReadCheckEvent::Upload); // TEST copies.push_back(BufferCopy{ .src_offset = total_size_bytes, .dst_offset = device_addr_out - buffer_start, @@ -3931,6 +4063,7 @@ void BufferCache

::InlineMemoryImplementation(DAddr dest_address, size_t copy_ ClearDownload(dest_address, copy_size); gpu_modified_ranges.Subtract(dest_address, copy_size); MarkVertexBuffersStale(dest_address, copy_size); + NoteReadCheckEvent(dest_address, copy_size, ReadCheckEvent::Inline); // TEST BufferId buffer_id = FindBuffer(dest_address, static_cast(copy_size), false); auto& buffer = slot_buffers[buffer_id]; diff --git a/src/video_core/buffer_cache/buffer_cache_base.h b/src/video_core/buffer_cache/buffer_cache_base.h index f674d299f1..cf68d5b243 100644 --- a/src/video_core/buffer_cache/buffer_cache_base.h +++ b/src/video_core/buffer_cache/buffer_cache_base.h @@ -845,6 +845,17 @@ class BufferCache : public VideoCommon::ChannelSetupCaches own; + /// The downloads that brought `landed`, newest last, to explain an Outdated byte. + struct Origin { + DAddr begin{}; + DAddr end{}; + u64 tick{}; + u64 copied{}; ///< read_check_events before its copy + u64 landed{}; ///< read_check_events before it landed + bool early{}; + bool before_read{}; ///< Landed before this read came (early_landed), not kept + }; + boost::container::small_vector origins; }; struct ReadCheckTotals { u64 every{}; ///< Tests: check 1 in `every` read downloads, over EDEN_NX2_READ_CHECK @@ -877,10 +888,6 @@ class BufferCache : public VideoCommon::ChannelSetupCaches& written, - const Common::RangeSet& cpu_written); /// A DownloadMemoryForRead: its copies, and what is left of them to do once the host GPU has /// run them. @@ -900,7 +907,12 @@ class BufferCache : public VideoCommon::ChannelSetupCaches& cpu_written); /// TEST (EDEN_NX2_READ_CHECK): compares guest memory with `download`'s check copy, right after /// the download landed. void CheckReadDownload(const ReadDownload& download, @@ -908,6 +920,50 @@ class BufferCache : public VideoCommon::ChannelSetupCaches read_check_events; + u64 read_check_event_count{}; + static constexpr size_t ReadCheckEventRing = 4096; + /// TEST (EDEN_NX2_READ_CHECK): something changed the GPU's copy or guest memory there. + void NoteReadCheckEvent(DAddr device_addr, u64 size, ReadCheckEvent kind, + GpuWaitStats::WriteSite site = GpuWaitStats::WriteSite::Other) { + if (!ReadChecks() || size == 0 || (read_downloads.empty() && early_landed.empty())) { + return; + } + if (read_check_events.empty()) [[unlikely]] { + read_check_events.resize(ReadCheckEventRing); + } + if (kind == ReadCheckEvent::Unrecorded && read_check_event_count != 0) { + // MarkWrittenBuffer outdates the downloads after noting its write. + const ReadCheckEventEntry& last = + read_check_events[(read_check_event_count - 1) % read_check_events.size()]; + if (last.kind == ReadCheckEvent::GpuWrite && last.begin == device_addr && + last.end == device_addr + size) { + return; + } + } + ++read_check_event_count; + read_check_events[(read_check_event_count - 1) % read_check_events.size()] = { + device_addr, device_addr + size, read_check_event_count, kind, site}; + } /// Oldest first, and written to guest memory in that order: none lands over newer bytes. std::deque read_downloads; /// All the read downloads' clean_ranges (they don't overlap): GPU-modified bytes already on @@ -921,6 +977,8 @@ class BufferCache : public VideoCommon::ChannelSetupCaches bytes; std::vector as_landed; ///< The page, DEVICE_PAGESIZE bytes + /// The downloads that landed them, newest last (at most 8): a check's Origin. + boost::container::small_vector from; }; std::unordered_map early_landed; /// Whether an early download serves this read: in check mode it flushes (and is compared). From d4e91e81de35f3932f4f765414de813d07f8e45b Mon Sep 17 00:00:00 2001 From: jpkto <255363096+jpkto@users.noreply.github.com> Date: Sun, 8 Nov 2026 15:34:32 +0100 Subject: [PATCH 067/105] vulkan: align a command chunk's data for every command it holds A Windows clang-cl build of this branch crashed in the command the ASTC decoder pass records (reported by lev92 on #805, with a minidump): a movaps on its BlockLinearSwizzle2DParams, whose alignas(16) members were at an address 8 mod 16. CommandChunk::Record aligns a command's offset within data to the command's own alignment, so data itself must be aligned at least as much. It was declared alignas(std::max_align_t), which is 8 on MSVC and clang-cl (16 with GCC on x86-64). Before, data followed two pointers, a size_t and a bool, and happened to start 16-aligned. The EDEN_NX2_GPU_COST site list (e11c916aca) put a 24-byte std::vector before it. That moved data to 8 mod 16, so every command with a 16-byte aligned type was constructed misaligned. data now asks for 16 bytes itself (or std::max_align_t, if that is more), and Record checks at compile time that no command needs more. Tested: every Vulkan file that records commands compiles with GCC and -fno-rtti, including vk_compute_pass.cpp with the ASTC decoder pass, so the new static_assert holds for every command type recorded there. MSVC builds already disable C4324 (structure padded for alignment). Co-Authored-By: Claude Opus 5.5 --- src/video_core/renderer_vulkan/vk_scheduler.h | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/video_core/renderer_vulkan/vk_scheduler.h b/src/video_core/renderer_vulkan/vk_scheduler.h index 5743679523..c1bcfe164f 100644 --- a/src/video_core/renderer_vulkan/vk_scheduler.h +++ b/src/video_core/renderer_vulkan/vk_scheduler.h @@ -411,6 +411,8 @@ class Scheduler { bool Record(T& command) { using FuncType = TypedCommand; static_assert(sizeof(FuncType) < sizeof(data), "Lambda is too large"); + // The offset is aligned within data, so data itself must be aligned at least as much. + static_assert(alignof(FuncType) <= DataAlignment, "Lambda needs more alignment"); command_offset = Common::AlignUp(command_offset, alignof(FuncType)); if (command_offset > sizeof(data) - sizeof(FuncType)) { @@ -447,7 +449,13 @@ class Scheduler { size_t command_offset = 0; bool submit = false; std::vector sites; - alignas(std::max_align_t) std::array data{}; + /// Commands are constructed in data at offsets aligned for their type. Some hold + /// alignas(16) members (the ASTC decoder pass's BlockLinearSwizzle2DParams), and + /// std::max_align_t is only 8 on MSVC and clang-cl, so data asks for 16 itself rather + /// than depending on where the members before it happen to end. + static constexpr size_t DataAlignment = + alignof(std::max_align_t) > 16 ? alignof(std::max_align_t) : 16; + alignas(DataAlignment) std::array data{}; }; template From a5cb08801f889bf43abc18390b16ff1a1b6d8b8e Mon Sep 17 00:00:00 2001 From: Burak Date: Sun, 8 Nov 2026 17:37:47 +0300 Subject: [PATCH 068/105] vulkan: let the image recycler keep images of a new allocator at a cleared one's address Clear leaves the allocator marked closed, so that images released during the rest of the teardown are destroyed. A Device made later in the same process can get the old allocator's address back, and the pool then stayed empty for it. Open, called after vmaCreateAllocator, takes the mark off. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01CqQ96AP4a2yY5zMzWNrFdE --- src/video_core/vulkan_common/vulkan_device.cpp | 1 + src/video_core/vulkan_common/vulkan_wrapper.cpp | 8 ++++++++ src/video_core/vulkan_common/vulkan_wrapper.h | 2 ++ 3 files changed, 11 insertions(+) diff --git a/src/video_core/vulkan_common/vulkan_device.cpp b/src/video_core/vulkan_common/vulkan_device.cpp index 60c739675b..75d2491372 100644 --- a/src/video_core/vulkan_common/vulkan_device.cpp +++ b/src/video_core/vulkan_common/vulkan_device.cpp @@ -840,6 +840,7 @@ Device::Device(VkInstance instance_, vk::PhysicalDevice physical_, VkSurfaceKHR }; vk::Check(vmaCreateAllocator(&allocator_info, &allocator)); + vk::ImageRecycler::Open(allocator); owns_static_pipeline_cache = surface != VkSurfaceKHR{}; LoadStaticPipelineCache(); diff --git a/src/video_core/vulkan_common/vulkan_wrapper.cpp b/src/video_core/vulkan_common/vulkan_wrapper.cpp index bdcaabf7d8..d0460be57b 100644 --- a/src/video_core/vulkan_common/vulkan_wrapper.cpp +++ b/src/video_core/vulkan_common/vulkan_wrapper.cpp @@ -730,6 +730,14 @@ void ImageRecycler::Shrink(u64 bytes) noexcept { TrimRecycler(state, RecyclerCap - bytes, std::chrono::steady_clock::now()); } +void ImageRecycler::Open(VmaAllocator allocator) noexcept { + RecyclerState& state = Recycler(); + std::scoped_lock lock{state.mutex}; + if (state.closed == allocator) { + state.closed = nullptr; + } +} + void ImageRecycler::Clear(VmaAllocator allocator) noexcept { RecyclerState& state = Recycler(); std::scoped_lock lock{state.mutex}; diff --git a/src/video_core/vulkan_common/vulkan_wrapper.h b/src/video_core/vulkan_common/vulkan_wrapper.h index 422b91cec0..49090d0a98 100644 --- a/src/video_core/vulkan_common/vulkan_wrapper.h +++ b/src/video_core/vulkan_common/vulkan_wrapper.h @@ -787,6 +787,8 @@ class ImageRecycler { VmaAllocation& allocation) noexcept; /// Destroys every kept image of this allocator and keeps nothing more (allocator teardown). static void Clear(VmaAllocator allocator) noexcept; + /// Keeps images of a new allocator again, though it has the address of a cleared one. + static void Open(VmaAllocator allocator) noexcept; /// Device memory the kept images hold (free memory to the texture cache's collector). [[nodiscard]] static u64 KeptBytes() noexcept; /// Drops the oldest kept images until at most `bytes` are kept. From d8988e7032d4ff02b305d39966ae6f0fd0139d31 Mon Sep 17 00:00:00 2001 From: jpkto <255363096+jpkto@users.noreply.github.com> Date: Sun, 8 Nov 2026 16:18:03 +0100 Subject: [PATCH 069/105] core: give device memory translations generations no other manager has A thread's translation cache (t_cache, thread_local and shared by every DeviceMemoryManager on the thread) names its manager by address and generation. The generation was a counter of each manager, starting at 1 and bumped by Map and Unmap. A manager made where a destroyed one was started at the same generation the old one had after its first Map. The thread then kept using the old manager's host pointers, into memory the old DeviceMemory had freed or another one now holds. A test fixture on the stack does this for every test case, and so would a System made again in the same process. This branch's CI hit it at d4e91e81de. Six buffer_cache_ports cases read guest memory wrong, two of them with every compared byte off, while the same cases passed in the read download harness. There, each test's DeviceMemory happened to get the same address back. Keeping every test's DeviceMemory alive in the harness, so that each gets a new address, reproduces two of the failures. Generations now come from one counter for all managers of a type, so no manager's translations ever look valid to another. Tested: read download harness, 14311 checks, 0 failures, both as it is and with each test's DeviceMemory at a new address (2 failures before this change). gpu_device_memory_manager.cpp and device_memory.cpp compile. Co-Authored-By: Claude Opus 5.5 --- src/core/device_memory_manager.h | 10 +++++++++- src/core/device_memory_manager.inc | 4 ++-- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/src/core/device_memory_manager.h b/src/core/device_memory_manager.h index 12cd0e602c..c1aa524c10 100644 --- a/src/core/device_memory_manager.h +++ b/src/core/device_memory_manager.h @@ -245,7 +245,15 @@ class DeviceMemoryManager { tracked_entries.GetUnchecked(page_index).cpu_backing_address = address | (asid.id << asid_start_bit); } - std::atomic translation_generation{1}; ///< Bumped by Map and Unmap + /// Generations come from one counter shared by every manager. A thread's translations name + /// their manager by address, and a manager made where a destroyed one was (a test fixture + /// on the stack, a System made again in the same process) would otherwise start with the + /// same generation and take the old manager's host pointers for its own. + static u64 NextTranslationGeneration() { + static std::atomic next{1}; + return next.fetch_add(1, std::memory_order_relaxed); + } + std::atomic translation_generation{NextTranslationGeneration()}; ///< New at Map and Unmap using CounterType = u8; using CounterAtomicType = std::atomic_uint8_t; static constexpr size_t subentries = 8 / sizeof(CounterType); diff --git a/src/core/device_memory_manager.inc b/src/core/device_memory_manager.inc index 40ff9ef5f3..1b6ecbc4e8 100644 --- a/src/core/device_memory_manager.inc +++ b/src/core/device_memory_manager.inc @@ -310,7 +310,7 @@ void DeviceMemoryManager::Map(DAddr address, VAddr virtual_address, size } impl->multi_dev_address.Register(new_dev, start_id); } - translation_generation.fetch_add(1, std::memory_order_release); + translation_generation.store(NextTranslationGeneration(), std::memory_order_release); if (track) { TrackContinuityImpl(address, virtual_address, size, asid); } @@ -346,7 +346,7 @@ void DeviceMemoryManager::Unmap(DAddr address, size_t size) { base_dev = new_start | MULTI_FLAG; } } - translation_generation.fetch_add(1, std::memory_order_release); + translation_generation.store(NextTranslationGeneration(), std::memory_order_release); } template From f01be636def436e5a46da393a96d008698520051 Mon Sep 17 00:00:00 2001 From: jpkto <255363096+jpkto@users.noreply.github.com> Date: Sun, 8 Nov 2026 16:48:20 +0100 Subject: [PATCH 070/105] TEST: read download check: leave out landed bytes an upload took from the guest The first verify run of the merged branch (d8988e7032, Hyrule Field) reported 55 bytes Outdated in 339,618 downloads, with the explanation line from f9f3bc6394. Every one was path (a): early landed bytes, guest memory equal to the snapshot taken at landing, and the check's copy of the buffer different. Every explanation also read "upload (after it landed, before this read's copy)" and "a guest write since it landed: no". An early download's landed bytes are no longer GPU-modified, so a guest write there may never reach WriteMemory, and the buffer cache doesn't hear of it. The next upload sends that write to the buffer. If the guest then writes the landed value back, guest memory equals the snapshot while the buffer holds the write the guest overwrote. A synchronous flush would not download those bytes at all, so the guest has what it would have without early downloads. The check was wrong, not the downloads. An upload now forgets the early landed bytes whose guest value differs from the snapshot (ForgetRewrittenEarlyLanded). Bytes still equal to it are kept: the buffer gets the landed value back, and the read still compares them. This changes only EDEN_NX2_READ_CHECK state, and costs one test of early_landed per upload range otherwise. Tests: a new buffer_cache_ports case, "Read download check leaves out early downloaded bytes an upload replaced". Without the change it reports "0x20040+0x10 is outdated: (a) ... upload (after it landed, before this read's copy) ... a guest write since it landed: no", the shape of the user's lines. The read download harness passes with and without moving each test's DeviceMemory: 14,348 checks, 0 failures. Co-Authored-By: Claude Opus 5.5 --- src/tests/video_core/buffer_cache_ports.cpp | 35 +++++++++++++++++ src/video_core/buffer_cache/buffer_cache.h | 38 +++++++++++++++++++ .../buffer_cache/buffer_cache_base.h | 11 ++++++ 3 files changed, 84 insertions(+) diff --git a/src/tests/video_core/buffer_cache_ports.cpp b/src/tests/video_core/buffer_cache_ports.cpp index 246ff590e2..b4aa66d376 100644 --- a/src/tests/video_core/buffer_cache_ports.cpp +++ b/src/tests/video_core/buffer_cache_ports.cpp @@ -294,6 +294,10 @@ struct BufferCacheTestAccess { static void Upload(Cache& cache, BufferId id, DAddr address, u32 size) { (void)cache.SynchronizeBuffer(cache.slot_buffers[id], address, size); } + /// A guest write the buffer cache doesn't hear of, on a page the tracker counts as written. + static void MarkCpuModified(Cache& cache, DAddr address, u64 size) { + cache.memory_tracker.MarkRegionAsCpuModified(address, size); + } static void Uniform(Cache& cache, BufferCacheChannelInfo& channel, BufferId id, DAddr address, u32 size) { cache.channel_state = &channel; @@ -1935,3 +1939,34 @@ TEST_CASE("Read download check leaves out early downloaded bytes a query result REQUIRE(Access::ReadChecks(*f.cache).mismatched == 0); REQUIRE(Access::ReadChecks(*f.cache).uncovered == 0); } + +TEST_CASE("Read download check leaves out early downloaded bytes an upload replaced", + "[buffer_ports]") { + Fixture f; + const Nx2TitleScope nx2_title; + const auto id = Access::Add(*f.cache, 0x20000, 0x10000); + auto& buffer = Access::Get(*f.cache, id); + Access::CheckEveryRead(*f.cache); + for (u8 frame = 1; frame <= 2; ++frame) { + WriteByDispatch(f, id, 0x20000, 0x100, frame); + ReadWord(f, 0x20010); + } + WriteByDispatch(f, id, 0x20000, 0x100, 3); + f.runtime.Complete(f.runtime.submitted_tick); + f.cache->NoteWorkRecorded(true); + REQUIRE(Access::ReadDownloads(*f.cache) == 0); + // The guest writes 16 of the landed bytes where the buffer cache doesn't hear of it, and a + // draw uploads them. Then it writes the landed value back, which no upload has taken yet. + std::memset(f.bytes + 0x10040, 0xee, 0x10); + Access::MarkCpuModified(*f.cache, 0x20040, 0x10); + Access::Upload(*f.cache, id, 0x20000, 0x1000); + REQUIRE(buffer.bytes[0x40] == 0xee); + std::memset(f.bytes + 0x10040, 3, 0x10); + // A synchronous flush at the read leaves them to the guest: they are not GPU-modified. + REQUIRE_FALSE(ReadWord(f, 0x20010).preemtive); + REQUIRE(f.bytes[0x10040] == 3); + REQUIRE(f.bytes[0x10010] == 3); + REQUIRE(Access::ReadChecks(*f.cache).compared != 0); + REQUIRE(Access::ReadChecks(*f.cache).mismatched == 0); + REQUIRE(Access::ReadChecks(*f.cache).uncovered == 0); +} diff --git a/src/video_core/buffer_cache/buffer_cache.h b/src/video_core/buffer_cache/buffer_cache.h index f6e200d598..67ac2e028a 100644 --- a/src/video_core/buffer_cache/buffer_cache.h +++ b/src/video_core/buffer_cache/buffer_cache.h @@ -1169,6 +1169,42 @@ void BufferCache

::TakeEarlyLanded(DAddr device_addr, u64 size, Func&& func) { } } +template +void BufferCache

::ForgetRewrittenEarlyLanded(DAddr device_addr, u64 size) { + std::array guest; + for (DAddr page = Common::AlignDown(device_addr, DEVICE_PAGESIZE); page < device_addr + size; + page += DEVICE_PAGESIZE) { + const auto it = early_landed.find(page); + if (it == early_landed.end()) { + continue; + } + EarlyLanded& entry = it->second; + Common::RangeSet rewritten; + entry.bytes.ForEachInRange(device_addr, size, [&](DAddr lower, DAddr upper) { + const u64 length = upper - lower; + device_memory.ReadBlockUnsafe(lower, guest.data(), length); + const u8* const landed = entry.as_landed.data() + (lower - page); + for (u64 i = 0; i < length;) { + if (guest[i] == landed[i]) { + ++i; + continue; + } + u64 end = i + 1; + while (end < length && guest[end] != landed[end]) { + ++end; + } + rewritten.Add(lower + i, end - i); + i = end; + } + }); + rewritten.ForEach( + [&](DAddr lower, DAddr upper) { entry.bytes.Subtract(lower, upper - lower); }); + if (entry.bytes.Empty()) { + early_landed.erase(it); + } + } +} + template void BufferCache

::ForgetEarlyLanded(DAddr device_addr, u64 size) { for (DAddr page = Common::AlignDown(device_addr, DEVICE_PAGESIZE); page < device_addr + size; @@ -3987,6 +4023,7 @@ bool BufferCache

::SynchronizeBuffer(Buffer& buffer, DAddr device_addr, u32 si if (start == end) return; const u64 range_size = end - start; NoteReadCheckEvent(start, range_size, ReadCheckEvent::Upload); // TEST + ForgetUploadedEarlyLanded(start, range_size); // TEST upload_copies.push_back(BufferCopy{ .src_offset = total_size_bytes, .dst_offset = start - buffer_start, @@ -4022,6 +4059,7 @@ bool BufferCache

::SynchronizeBuffer(Buffer& buffer, DAddr device_addr, u32 si DAddr buffer_start = buffer.CpuAddr(); for_each_upload_range([&](u64 device_addr_out, u64 range_size) { NoteReadCheckEvent(device_addr_out, range_size, ReadCheckEvent::Upload); // TEST + ForgetUploadedEarlyLanded(device_addr_out, range_size); // TEST copies.push_back(BufferCopy{ .src_offset = total_size_bytes, .dst_offset = device_addr_out - buffer_start, diff --git a/src/video_core/buffer_cache/buffer_cache_base.h b/src/video_core/buffer_cache/buffer_cache_base.h index 67a7605e1b..cda5aaac60 100644 --- a/src/video_core/buffer_cache/buffer_cache_base.h +++ b/src/video_core/buffer_cache/buffer_cache_base.h @@ -1000,6 +1000,17 @@ class BufferCache : public VideoCommon::ChannelSetupCaches from; }; std::unordered_map early_landed; + /// TEST (EDEN_NX2_READ_CHECK): an upload sends guest memory's bytes to the buffer. Where they + /// are no longer what an early download landed, the guest wrote there since, and landed bytes + /// are no longer GPU-modified, so that write may never have reached WriteMemory. The buffer + /// then holds it while guest memory may hold the landed value again: no read compares those + /// bytes, as a synchronous flush would not download them. + void ForgetUploadedEarlyLanded(DAddr device_addr, u64 size) { + if (!early_landed.empty()) [[unlikely]] { + ForgetRewrittenEarlyLanded(device_addr, size); + } + } + void ForgetRewrittenEarlyLanded(DAddr device_addr, u64 size); /// Whether an early download serves this read: in check mode it flushes (and is compared). [[nodiscard]] bool EarlyDownloadServes(DAddr device_addr, u64 size) const; void NoteEarlyLanded(const ReadDownload& download, const Common::RangeSet& cpu_written); From 894a9e838692fdbd5b7c3dfec78e1ab366f48b20 Mon Sep 17 00:00:00 2001 From: jpkto <255363096+jpkto@users.noreply.github.com> Date: Sun, 8 Nov 2026 17:35:00 +0100 Subject: [PATCH 071/105] gpu_split: count fence-flagged releases in memory once their fence ran The first verify run of the merged branch (d8988e7032) passed 739,427 acquires in 6.5 minutes of Hyrule Field, and 74,721 of them were not met after the drain (EDEN_NX2_DRAIN_CHECK). Before the merge it was 0 in 4.6 million. Castle Town showed the same, 41,193 of 512,537. Upstream #337 (673783fa6b) keeps fence-flagged query reports pending until their fence runs: FenceManager::SignalFence no longer skips their fence or runs the operation at once. So a release with IsAFence reaches memory only when the fencing thread runs that fence, not when the back runs it. Most Switch 2 releases are fence-flagged: the puller's 4-byte and SEM_EXECUTE releases, QMD plain and four-word releases, the copy engine's, short 3D and one-word compute reports. The front's semaphore view still counted every release in memory once the back had run it (QueuedSemaphoreWriteAt), and passed acquires on a release still queued (MetByQueued) that had not landed after the drain. The front now notes how each release lands (GpuSplit::Landing). A fenced one counts as in memory only once its fence ran. The query cache's operation tells the split so, with the back entry the report came in (BackEntry, NoteFencedWrite). Fences run their operations in order, so one landing also covers every entry before it. Entry tags carry the split's start count, so a late fence of an earlier start lands nothing of the next. While a fenced release may still land: - AskView passes a greater-or-equal acquire only if the word meets it in memory now and the release meets it too. Otherwise the acquire suspends, and the release latches the waiter when it lands, or a poll finds the word in memory. - A write that lands as the back runs it (a reduction, a maximum, a 16-byte release), sent after a fenced release to the same word, leaves the word's value unknown until that one landed. Either may come last. - QMD and compute report reductions and maxima take no base from it (FrontQmdView::WordAt, KeplerCompute::FrontNoteReport): the back reads the word when it runs them, before or after the landing. Counter reports land with a fence as well (SyncOperation), and now count the same way. Releases that land when the back runs them are unchanged. TEST: the drain check report now splits "not met after the drain" by how the view passed (memory, queued, positioned wait). Tests: a new maxwell_fast_method case, "Split counts a fenced semaphore write in memory once its fence ran, not before". The split harness runs it against the real back thread: 55 checks, 20 of 20 runs, and 10 of 10 pinned to one core. The QMD view harness passes, with its fake back landing fenced releases as it runs them: 236,291 notes checked, 0 released otherwise. The other harnesses pass too. Co-Authored-By: Claude Opus 5.5 --- src/tests/video_core/maxwell_fast_method.cpp | 59 ++++++++++++++++++ src/video_core/dma_pusher.cpp | 7 ++- src/video_core/engines/kepler_compute.cpp | 14 +++-- src/video_core/engines/maxwell_3d.cpp | 10 ++- src/video_core/engines/puller.cpp | 49 ++++++++++----- src/video_core/front_qmd.h | 13 ++-- src/video_core/gpu_split.cpp | 35 ++++++++--- src/video_core/gpu_split.h | 64 +++++++++++++++++--- src/video_core/nx2_drain_check.h | 12 ++-- src/video_core/query_cache/query_cache.h | 11 +++- 10 files changed, 223 insertions(+), 51 deletions(-) diff --git a/src/tests/video_core/maxwell_fast_method.cpp b/src/tests/video_core/maxwell_fast_method.cpp index b9816c04d4..9c693e8db4 100644 --- a/src/tests/video_core/maxwell_fast_method.cpp +++ b/src/tests/video_core/maxwell_fast_method.cpp @@ -1035,6 +1035,65 @@ TEST_CASE("Split remembers the semaphore writes its back has not run yet", REQUIRE_FALSE(split.QueuedSemaphoreWriteAt(0x2000).has_value()); } +TEST_CASE("Split counts a fenced semaphore write in memory once its fence ran, not before", + "[video_core][gpu_split_dependencies]") { + using Access = VideoCommon::GpuSplitTestAccess; + using Landing = VideoCommon::GpuSplit::Landing; + Fixture f; + f.system.Initialize(); + VideoCommon::GpuSplit split; + Access::Start(split, f.system); + SCOPE_EXIT { + split.Stop(); + }; + // A fence-flagged report lands once the fence after it ran (QueryCacheBase::CounterReport), + // which tells with the entry the back gave it. + split.Defer([] {}); + std::optional entry; + split.Defer([&split, &entry] { entry = split.BackEntry(); }); + split.NoteSemaphoreWrite(0x1000, 21, true, false, Landing::Fenced); + split.Drain(VideoCommon::DrainSite::Other); + REQUIRE(entry.has_value()); + REQUIRE(*entry > 0); + REQUIRE_FALSE(split.BackEntry().has_value()); + auto queued = split.QueuedSemaphoreWriteAt(0x1000); + REQUIRE(queued.has_value()); + REQUIRE(queued->fenced); + REQUIRE(queued->payload == 21); + // A write that lands as the back runs it, sent before that fence ran: the word holds either. + split.Defer([] {}); + split.NoteSemaphoreWrite(0x1000, 22, true); + split.Drain(VideoCommon::DrainSite::Other); + queued = split.QueuedSemaphoreWriteAt(0x1000); + REQUIRE(queued.has_value()); + REQUIRE(queued->fenced); + REQUIRE_FALSE(queued->known); + // A fence of an entry before it lands nothing of it; its own lands it. + split.NoteFencedWrite(*entry - 1); + REQUIRE(split.QueuedSemaphoreWriteAt(0x1000).has_value()); + split.NoteFencedWrite(*entry); + REQUIRE_FALSE(split.QueuedSemaphoreWriteAt(0x1000).has_value()); + // Once it landed, a write after it holds the word once the back ran it. + split.Defer([] {}); + split.NoteSemaphoreWrite(0x1000, 23, true); + queued = split.QueuedSemaphoreWriteAt(0x1000); + REQUIRE(queued.has_value()); + REQUIRE_FALSE(queued->fenced); + REQUIRE(queued->payload == 23); + split.Drain(VideoCommon::DrainSite::Other); + REQUIRE_FALSE(split.QueuedSemaphoreWriteAt(0x1000).has_value()); + // A later entry's fence lands the fenced writes of the entries before it. + split.Defer([] {}); + split.NoteSemaphoreWrite(0x2000, 24, true, false, Landing::Fenced); + std::optional later; + split.Defer([&split, &later] { later = split.BackEntry(); }); + split.Drain(VideoCommon::DrainSite::Other); + REQUIRE(split.QueuedSemaphoreWriteAt(0x2000).has_value()); + REQUIRE(later.has_value()); + split.NoteFencedWrite(*later); + REQUIRE_FALSE(split.QueuedSemaphoreWriteAt(0x2000).has_value()); +} + TEST_CASE("Split waits for a semaphore write's position, not for the work sent after it", "[video_core][gpu_split_dependencies]") { using Access = VideoCommon::GpuSplitTestAccess; diff --git a/src/video_core/dma_pusher.cpp b/src/video_core/dma_pusher.cpp index f4bead1fbc..59ab57f19f 100644 --- a/src/video_core/dma_pusher.cpp +++ b/src/video_core/dma_pusher.cpp @@ -770,12 +770,15 @@ void DmaPusher::NoteEngineWrites(Engines::EngineTypes type, u32 method) { if (!device) { return; } - split.NoteSemaphoreWrite(*device, dma_registers[MaxwellDMA::SemaphoreMethod + 2], true); + // Fence-flagged reports: they land once their fence ran. + constexpr auto landing = VideoCommon::GpuSplit::Landing::Fenced; + split.NoteSemaphoreWrite(*device, dma_registers[MaxwellDMA::SemaphoreMethod + 2], true, false, + landing); VideoCommon::Nx2AcquireStats::NoteRelease(*device, VideoCommon::Nx2AcquireStats::Releaser::Copy); if (writes.semaphore.size > sizeof(u32)) { // The payload as 64 bits (a timestamp follows). - split.NoteSemaphoreWrite(*device + 4, 0, true); + split.NoteSemaphoreWrite(*device + 4, 0, true, false, landing); } } diff --git a/src/video_core/engines/kepler_compute.cpp b/src/video_core/engines/kepler_compute.cpp index d2fbed3a9f..f8e33c4336 100644 --- a/src/video_core/engines/kepler_compute.cpp +++ b/src/video_core/engines/kepler_compute.cpp @@ -1881,8 +1881,9 @@ void KeplerCompute::FrontNoteReport(VideoCommon::GpuSplit& split, const FrontSta const u32 op = (control >> 9) & 7; std::optional base; if (!from_macro) { + // A fenced release lands once its fence ran, before or after the reduction. if (const auto queued = split.QueuedSemaphoreWriteAt(*device)) { - if (queued->known && !queued->at_least && + if (queued->known && !queued->at_least && !queued->fenced && queued->position >= split.LastWriterPosition()) { base = queued->payload; } @@ -1894,10 +1895,15 @@ void KeplerCompute::FrontNoteReport(VideoCommon::GpuSplit& split, const FrontSta base.has_value()); return; } - split.NoteSemaphoreWrite(*device, fs.registers[0x6c2], true); - if (((control >> 28) & 1) == 0) { + // One word: a fence-flagged report, which lands once its fence ran. Else a payload report + // with a timestamp, written as the back runs it. + const bool one_word = ((control >> 28) & 1) != 0; + const auto landing = + one_word ? VideoCommon::GpuSplit::Landing::Fenced : VideoCommon::GpuSplit::Landing::AtRun; + split.NoteSemaphoreWrite(*device, fs.registers[0x6c2], true, false, landing); + if (!one_word) { // 64 bits (a timestamp follows). - split.NoteSemaphoreWrite(*device + 4, 0, true); + split.NoteSemaphoreWrite(*device + 4, 0, true, false, landing); } } diff --git a/src/video_core/engines/maxwell_3d.cpp b/src/video_core/engines/maxwell_3d.cpp index ca8045f7a7..4ef23067b4 100644 --- a/src/video_core/engines/maxwell_3d.cpp +++ b/src/video_core/engines/maxwell_3d.cpp @@ -1502,12 +1502,18 @@ void Maxwell3D::FrontNoteReport(VideoCommon::GpuSplit& split, const FrontState& return; } const bool payload = report.query.report == Regs::ReportSemaphore::Report::Payload; - split.NoteSemaphoreWrite(*device, report.payload, payload); + // As QueryCacheBase::CounterReport writes it: a payload report without a fence flag as the + // back runs it, a short release (fence-flagged) or a counter once a fence ran. + const bool fence_flagged = operation == Regs::ReportSemaphore::Operation::Release && + report.query.short_query != 0; + const auto landing = payload && !fence_flagged ? VideoCommon::GpuSplit::Landing::AtRun + : VideoCommon::GpuSplit::Landing::Fenced; + split.NoteSemaphoreWrite(*device, report.payload, payload, false, landing); VideoCommon::Nx2AcquireStats::NoteRelease(*device, VideoCommon::Nx2AcquireStats::Releaser::Report3D); if (report.query.short_query == 0) { // 64 bits (a timestamp follows). - split.NoteSemaphoreWrite(*device + 4, 0, payload); + split.NoteSemaphoreWrite(*device + 4, 0, payload, false, landing); } } diff --git a/src/video_core/engines/puller.cpp b/src/video_core/engines/puller.cpp index 65fa1c1377..3f70939949 100644 --- a/src/video_core/engines/puller.cpp +++ b/src/video_core/engines/puller.cpp @@ -86,29 +86,37 @@ void ReleaseFences(DmaPusher& dma_pusher) { VideoCommon::DrainSite::FenceRelease); } -// A release was just sent to the back (as Writes::Noted): tell the front what it writes -// (CheckWithoutDrain). -void NoteRelease(DmaPusher& dma_pusher, GPUVAddr address, u32 payload, bool four_words) { +// A release with `properties` was just sent to the back (as Writes::Noted): tell the front what +// it writes (CheckWithoutDrain). +void NoteRelease(DmaPusher& dma_pusher, GPUVAddr address, u32 payload, + VideoCommon::QueryPropertiesFlags properties) { auto& split = VideoCommon::GpuSplit::Get(); if (!split.Active()) { return; } + const bool four_words = True(properties & VideoCommon::QueryPropertiesFlags::HasTimeout); + // A fence-flagged report lands once the fence after it ran (QueryCacheBase::CounterReport). + const auto landing = True(properties & VideoCommon::QueryPropertiesFlags::IsAFence) + ? VideoCommon::GpuSplit::Landing::Fenced + : VideoCommon::GpuSplit::Landing::AtRun; split.NoteWrite(dma_pusher.memory_manager, address, four_words ? 16 : sizeof(u32)); const auto device = dma_pusher.memory_manager.GpuToCpuAddress(address); if (!device) { return; } - split.NoteSemaphoreWrite(*device, payload, true); + split.NoteSemaphoreWrite(*device, payload, true, false, landing); VideoCommon::Nx2AcquireStats::NoteRelease(*device, VideoCommon::Nx2AcquireStats::Releaser::Puller); if (four_words) { // The payload as 64 bits (a timestamp follows). - split.NoteSemaphoreWrite(*device + 4, 0, true); + split.NoteSemaphoreWrite(*device + 4, 0, true, false, landing); } } -// Whether the back writes a payload release to memory when it runs it (CounterReport): below -// High GPU accuracy, with fences that run their operations at once (FenceManager::SignalFence). +// Whether the back writes a payload report without a fence flag to memory when it runs it +// (CounterReport): below High GPU accuracy, with fences that run their operations at once +// (FenceManager::SignalFence). Fence-flagged reports land once their fence ran +// (GpuSplit::Landing::Fenced). bool ReleasesLandAtOnce() { const bool delay_fence = Settings::IsGPUFenceBehaviorDefault() @@ -118,8 +126,9 @@ bool ReleasesLandAtOnce() { } // Whether the front decides Switch 2 acquires from its view of the semaphore words (AskView) -// instead of draining the split: the back writes each release to memory as it runs it, and the -// front sees every release it sends, its macros' included. +// instead of draining the split: the back writes each release to memory as it runs it, or once +// its fence ran (GpuSplit::Landing), and the front sees every release it sends, its macros' +// included. bool ViewApplies() { const auto& split = VideoCommon::GpuSplit::Get(); return split.Active() && split.FrontMacros() && ReleasesLandAtOnce(); @@ -154,8 +163,16 @@ ViewAnswer AskView(DmaPusher& dma_pusher, const Puller::PendingAcquire& acquire, // and copy work sent before it may still release the word without the front's knowing (QMD // releases): a value that meets it stands once the back ran that work. const bool settled = acquire.WaitsToReach() || split.Ran(acquire.writers_before); - // The last release the front sent to the word decides while the back hasn't run it. + // The last release the front sent to the word decides while it may not be in memory. if (const auto queued = split.QueuedSemaphoreWriteAt(device)) { + if (queued->fenced) { + // It lands once a fence ran, at no position the front can wait for. A greater-or-equal + // acquire the word meets now and once it landed stands either way (the words only + // advance); else the landing latches the waiter, or a poll finds it in memory. + const bool landed_meets = + queued->known && acquire.WaitsToReach() && acquire.Matches(queued->payload); + return {landed_meets && acquire.Matches(memory()) ? Kind::Met : Kind::NotMet}; + } if (!queued->known) { return {Kind::Unknown, queued->position}; } @@ -352,8 +369,8 @@ bool Puller::CanResumeAcquire(DmaPusher& dma_pusher) { // With delayed fence operations (Balanced or Accurate fences, accurate GPU), a release // queued after the acquire released the fences waits in the caches for the next fence. With // every channel waiting none comes: release them again now and then while waiting. (The - // semaphore view, PollWithoutDrain, applies only while releases land at once: never with - // delayed fence operations.) + // semaphore view, PollWithoutDrain, applies only while releases without a fence flag land at + // once: never with delayed fence operations. Fence-flagged ones signal their own fence.) const bool delayed = Settings::IsGPUFenceBehaviorDefault() ? Settings::IsGPULevelHigh() : Settings::IsGPUFenceBehaviorBalanced() || @@ -550,8 +567,7 @@ void Puller::ProcessSemaphoreTriggerMethod(DmaPusher& dma_pusher) { payload, 0); }, VideoCommon::GpuSplit::Writes::Noted); - NoteRelease(dma_pusher, sequence_address, payload, - properties == VideoCommon::QueryPropertiesFlags::HasTimeout); + NoteRelease(dma_pusher, sequence_address, payload, properties); } else if (g_nx2_ampere) { // Switch 2 waits on semaphores the CPU (or another channel) releases, e.g. once it has // written the commands this channel runs next: suspend the channel until the value is @@ -690,7 +706,8 @@ void Puller::ProcessSemaphoreRelease(DmaPusher& dma_pusher) { VideoCommon::QueryPropertiesFlags::IsAFence, payload, 0); }, VideoCommon::GpuSplit::Writes::Noted); - NoteRelease(dma_pusher, sequence_address, payload, false); + NoteRelease(dma_pusher, sequence_address, payload, + VideoCommon::QueryPropertiesFlags::IsAFence); } void Puller::ProcessSemaphoreAcquire(DmaPusher& dma_pusher) { @@ -722,7 +739,7 @@ void Puller::ProcessSemaphoreExecute(DmaPusher& dma_pusher, u32 argument) { VideoCommon::QueryPropertiesFlags::IsAFence, payload, 0); }, VideoCommon::GpuSplit::Writes::Noted); - NoteRelease(dma_pusher, address, payload, false); + NoteRelease(dma_pusher, address, payload, VideoCommon::QueryPropertiesFlags::IsAFence); return; } // Acquires: never block the GPU thread on them; they are checked for the warning below only. diff --git a/src/video_core/front_qmd.h b/src/video_core/front_qmd.h index c99d99eb4d..75a74e0d23 100644 --- a/src/video_core/front_qmd.h +++ b/src/video_core/front_qmd.h @@ -572,9 +572,10 @@ class FrontQmdView { split.NoteSemaphoreWrite(*device, value, base.has_value()); check(*device, value, base.has_value(), false); } else if (four_words) { - // The payload as 64 bits, a timestamp after it. - split.NoteSemaphoreWrite(*device, payload, true); - split.NoteSemaphoreWrite(*device + 4, 0, true); + // The payload as 64 bits, a timestamp after it. Fence-flagged reports, like the plain + // release below: they land once their fence ran. + split.NoteSemaphoreWrite(*device, payload, true, false, GpuSplit::Landing::Fenced); + split.NoteSemaphoreWrite(*device + 4, 0, true, false, GpuSplit::Landing::Fenced); check(*device, payload, true, false); check(*device + 4, 0, true, false); } else if (maximum) { @@ -583,7 +584,7 @@ class FrontQmdView { split.NoteSemaphoreWrite(*device, value, true, !base.has_value()); check(*device, value, true, !base.has_value()); } else { - split.NoteSemaphoreWrite(*device, payload, true); + split.NoteSemaphoreWrite(*device, payload, true, false, GpuSplit::Landing::Fenced); check(*device, payload, true, false); } } @@ -603,7 +604,9 @@ class FrontQmdView { // The last write sent to the word's granule, placed or not. const u64 target = memory.WritesTarget(split, address, sizeof(u32)); if (const auto queued = split.QueuedSemaphoreWriteAt(device)) { - if (queued->known && !queued->at_least && queued->position >= target) { + // A fenced one lands once its fence ran, before or after the back reads the word. + if (queued->known && !queued->at_least && !queued->fenced && + queued->position >= target) { return queued->payload; } return std::nullopt; diff --git a/src/video_core/gpu_split.cpp b/src/video_core/gpu_split.cpp index c6e5563baa..1a5624182c 100644 --- a/src/video_core/gpu_split.cpp +++ b/src/video_core/gpu_split.cpp @@ -143,9 +143,9 @@ void GpuSplit::MaybeStart(Core::System& system_) { profile_publishes = Nx2Profile::Enabled(); // TEST log_methods = std::getenv("EDEN_NX2_ACQUIRE_STATS") != nullptr; // TEST ring = std::make_unique(RingWords); + ++starts; // Before the back starts: it tags the entries of fenced writes with it (EntryTag) back_thread = std::jthread([this](std::stop_token stop_token) { BackLoop(stop_token); }); active = true; - ++starts; const char* jit = std::getenv("EDEN_NX2_MME_JIT"); front_macros = mode == '2' && (jit == nullptr || jit[0] != '0'); LOG_INFO(HW_GPU, "GPU thread split: engine methods run on a second thread{}", @@ -169,6 +169,8 @@ void GpuSplit::Stop() { write_position = 0; last_writer_position = 0; last_sync_position = 0; + last_entry_start = 0; + back_entry_start = 0; semaphore_writes.clear(); semaphore_writes_limit = 1024; last_any_write = 0; @@ -297,27 +299,39 @@ bool GpuSplit::WaitForSyncedWriters(DrainSite site) { return WaitFor(std::min({last_writer_position, last_sync_position, write_position}), site); } -void GpuSplit::NoteSemaphoreWrite(u64 device_address, u32 payload, bool known, bool at_least) { +void GpuSplit::NoteSemaphoreWrite(u64 device_address, u32 payload, bool known, bool at_least, + Landing landing) { if (semaphore_writes.size() >= semaphore_writes_limit) { - // The words of releases the back ran hold their values in memory. - const u64 done = consumed.load(std::memory_order_acquire); + // The words of releases in memory hold their values. std::erase_if(semaphore_writes, - [done](const auto& entry) { return entry.second.position <= done; }); + [this](const auto& entry) { return InMemory(entry.second); }); semaphore_writes_limit = std::max(1024, semaphore_writes.size() * 2); } - semaphore_writes.insert_or_assign(device_address & ~u64{3}, - SemaphoreWrite{write_position, payload, known, at_least}); + // The release is in the last entry sent, or one before (a QMD signal valued late). + SemaphoreWrite write{write_position, EntryTag(last_entry_start), payload, + known, at_least, landing == Landing::Fenced}; + const u64 word = device_address & ~u64{3}; + if (const auto it = semaphore_writes.find(word); + !write.fenced && it != semaphore_writes.end() && it->second.fenced && + fenced_landed.load(std::memory_order_acquire) <= it->second.entry) { + // A fenced write sent before may land after this one: until it did, the word holds + // either. + write.entry = it->second.entry; + write.known = false; + write.at_least = false; + write.fenced = true; + } + semaphore_writes.insert_or_assign(word, write); } std::optional GpuSplit::QueuedSemaphoreWriteAt( u64 device_address) const { const auto it = semaphore_writes.find(device_address & ~u64{3}); - if (it == semaphore_writes.end() || - it->second.position <= consumed.load(std::memory_order_acquire)) { + if (it == semaphore_writes.end() || InMemory(it->second)) { return std::nullopt; } return QueuedSemaphoreWrite{it->second.payload, it->second.known, it->second.at_least, - it->second.position}; + it->second.fenced, it->second.position}; } bool GpuSplit::WritersPending() const { @@ -746,6 +760,7 @@ void GpuSplit::BackLoop(std::stop_token stop_token) { u64 last_store = read_position; while (read_position != available) { const size_t index = read_position & RingMask; + back_entry_start = read_position; const size_t size = Execute(&ring[index], std::min(available - read_position, RingWords - index)); read_position += size != 0 ? size : RingWords - index; diff --git a/src/video_core/gpu_split.h b/src/video_core/gpu_split.h index 3fe3e3ce8b..c3a36e583a 100644 --- a/src/video_core/gpu_split.h +++ b/src/video_core/gpu_split.h @@ -112,6 +112,23 @@ class GpuSplit { [[nodiscard]] static bool OnBackThread() { return on_back_thread; } + /// Back thread: the entry it runs, as NoteFencedWrite takes it; elsewhere nothing. + [[nodiscard]] std::optional BackEntry() const { + if (!on_back_thread) { + return std::nullopt; + } + return EntryTag(back_entry_start); + } + /// Any thread: the operation of a report the back sent in `entry` (BackEntry) ran with its + /// fence, so the report is in memory, and those of the entries before it: fences run their + /// operations in order (QueryCacheBase::CounterReport). + void NoteFencedWrite(u64 entry) { + u64 landed = fenced_landed.load(std::memory_order_relaxed); + while (landed <= entry && + !fenced_landed.compare_exchange_weak(landed, entry + 1, std::memory_order_release, + std::memory_order_relaxed)) { + } + } /// Called on the back before it sleeps waiting for work (the rasterizer submits deferred work). void SetIdleHook(void (*hook)(void*), void* argument) { idle_hook = hook; @@ -326,18 +343,29 @@ class GpuSplit { /// the writers a command list fetched now is ordered after. Returns whether any of it was /// still queued. `site`: whose wait it is, for EDEN_NX2_GPU_PROFILE. bool WaitForSyncedWriters(DrainSite site = DrainSite::ListWriters); - /// A semaphore release was just sent: once the back ran it, the word at `device_address` holds - /// `payload`, or a value the front can't tell (`known` false: a reduction, a counter report), - /// or `payload` at least (`at_least`: a maximum with a value the front can't tell). - void NoteSemaphoreWrite(u64 device_address, u32 payload, bool known, bool at_least = false); + /// When a release the back runs reaches memory. + enum class Landing : u8 { + AtRun, ///< As the back runs it: payload reports without a fence, reductions, maxima + Fenced, ///< Once the fence signaled after it ran (QueryCacheBase::CounterReport): + ///< fence-flagged reports, counter reports + }; + /// A semaphore release was just sent: once it landed (`landing`), the word at + /// `device_address` holds `payload`, or a value the front can't tell (`known` false: a + /// reduction, a counter report), or `payload` at least (`at_least`: a maximum with a value the + /// front can't tell). + void NoteSemaphoreWrite(u64 device_address, u32 payload, bool known, bool at_least = false, + Landing landing = Landing::AtRun); struct QueuedSemaphoreWrite { u32 payload; bool known; bool at_least; ///< Known only as a lower bound (unsigned) - u64 position; ///< Once the back ran this (Ran), the word holds the write + /// It lands once a fence ran, at no position the front can wait for; or an earlier write + /// that does may still land after it. + bool fenced; + u64 position; ///< Once the back ran this (Ran), the word holds the write, unless `fenced` }; - /// The last write sent to the semaphore word at `device_address`, if the back may not have run - /// it yet. + /// The last write sent to the semaphore word at `device_address`, if it may not be in memory + /// yet. [[nodiscard]] std::optional QueuedSemaphoreWriteAt( u64 device_address) const; /// Whether compute or copy work sent so far is still queued (EDEN_NX2_DRAIN_CHECK). @@ -523,6 +551,7 @@ class GpuSplit { if (profile_publishes && write_position == published_local) [[unlikely]] { NoteFirstUnpublished(); // TEST } + last_entry_start = write_position; write_position += words; if (PublishDue() && !publish_held) { PublishNow(); @@ -609,11 +638,15 @@ class GpuSplit { u64 write_position{}; u64 last_writer_position{}; u64 last_sync_position{}; + u64 last_entry_start{}; ///< Where the last entry sent starts struct SemaphoreWrite { u64 position; ///< write_position once it was sent + /// `fenced`: the entry (EntryTag) whose fenced write must have landed (NoteFencedWrite) + u64 entry; u32 payload; bool known; bool at_least; + bool fenced; }; /// The front's NoteSemaphoreWrite, by word address. std::unordered_map semaphore_writes; @@ -645,6 +678,20 @@ class GpuSplit { // Back side. Tegra::Engines::EngineInterface* back_engine{}; + u64 back_entry_start{}; ///< Where the entry the back runs starts (BackEntry) + + /// An entry's position as fenced writes name it: with this start of the split in the high + /// bits, so a fence of an earlier start that runs late lands nothing of this one (positions + /// stay far below 2^48). + [[nodiscard]] u64 EntryTag(u64 position) const { + return (u64{starts} << 48) | position; + } + /// Whether the word holds `write` (as far as no other write changed it since): the back ran + /// it, and a fenced one landed. + [[nodiscard]] bool InMemory(const SemaphoreWrite& write) const { + return write.position <= consumed.load(std::memory_order_acquire) && + (!write.fenced || fenced_landed.load(std::memory_order_acquire) > write.entry); + } static GpuSplit instance; static inline thread_local bool on_back_thread{}; @@ -660,6 +707,9 @@ class GpuSplit { // always touched together by both threads, so they share a block. alignas(128) std::atomic published{}; alignas(128) std::atomic consumed{}; + /// One past the last entry (EntryTag) whose fenced writes landed (NoteFencedWrite): the + /// fencing thread writes it, the front reads it. + alignas(128) std::atomic fenced_landed{}; /// A position the front waits for (WaitFor): the back stores `consumed` once past it. alignas(128) std::atomic wait_target{~u64{0}}; alignas(128) std::atomic back_waiting{}; diff --git a/src/video_core/nx2_drain_check.h b/src/video_core/nx2_drain_check.h index efb3326eb7..2534984ed6 100644 --- a/src/video_core/nx2_drain_check.h +++ b/src/video_core/nx2_drain_check.h @@ -435,11 +435,15 @@ inline void MaybeReport() { const u64 by_memory = take(state.view_passes[base]); const u64 by_queued = take(state.view_passes[base + 1]); const u64 waited = take(state.view_passes[base + 2]); - const u64 wrong = take(state.view_wrong[base]) + take(state.view_wrong[base + 1]) + - take(state.view_wrong[base + 2]); + const u64 wrong_memory = take(state.view_wrong[base]); + const u64 wrong_queued = take(state.view_wrong[base + 1]); + const u64 wrong_waited = take(state.view_wrong[base + 2]); return fmt::format("{} (by memory {}, by a release still queued {}, after a positioned " - "wait {}), not met after the drain {}", - by_memory + by_queued + waited, by_memory, by_queued, waited, wrong); + "wait {}), not met after the drain {} (of those by memory {}, queued " + "{}, after a wait {})", + by_memory + by_queued + waited, by_memory, by_queued, waited, + wrong_memory + wrong_queued + wrong_waited, wrong_memory, wrong_queued, + wrong_waited); }; const auto waits = [&](bool poll) { const u64 count = take(state.view_waits[poll ? 1 : 0]); diff --git a/src/video_core/query_cache/query_cache.h b/src/video_core/query_cache/query_cache.h index 4feda99da8..7f3a4fe591 100644 --- a/src/video_core/query_cache/query_cache.h +++ b/src/video_core/query_cache/query_cache.h @@ -21,6 +21,7 @@ #include "common/settings.h" #include "video_core/engines/maxwell_3d.h" #include "video_core/gpu.h" +#include "video_core/gpu_split.h" #include "video_core/host1x/gpu_device_memory_manager.h" #include "video_core/memory_manager.h" #include "video_core/nx2.h" @@ -282,9 +283,17 @@ void QueryCacheBase::CounterReport(GPUVAddr addr, QueryType counter_type }; u8* pointer = impl->device_memory.template GetPointer(cpu_addr); u8* pointer_timestamp = impl->device_memory.template GetPointer(cpu_addr + 8); + // The GPU thread split's back entry this report came in: the front counts the report in + // memory once its operation ran with a fence (GpuSplit::NoteFencedWrite). + const std::optional split_entry = VideoCommon::GpuSplit::Get().BackEntry(); std::function operation([this, streamer, query_base = query, query_location, pointer, pointer_timestamp, cpu_addr, addr, - asid = gpu_memory->GetID(), counter_type] { + asid = gpu_memory->GetID(), counter_type, split_entry] { + SCOPE_EXIT { + if (split_entry) { + VideoCommon::GpuSplit::Get().NoteFencedWrite(*split_entry); + } + }; if (True(query_base->flags & QueryFlagBits::IsInvalidated)) { if (const u64 trace = QueryTrace::Sample(QueryTrace::Event::Discard)) { LOG_INFO(HW_GPU, From 969cc058fedebec127b1d137387327d52842eb85 Mon Sep 17 00:00:00 2001 From: DjouWish Date: Sun, 8 Nov 2026 18:40:30 +0100 Subject: [PATCH 072/105] decoders: cap the unswizzle threads by the spare physical cores At most 4 and a quarter of the host threads, and only the physical cores past the 6 the guest cores and the GPU threads keep busy. On a 6-core Ryzen 5 5500 (#599) the 3 workers competed with them: no gain and slower movies. An 8-core Ryzen 7 5800X now gets 2. The threads start with the texture cache instead of at the first large unswizzle. Co-Authored-By: Claude Opus 5.5 --- src/video_core/texture_cache/texture_cache.h | 1 + src/video_core/textures/decoders.cpp | 27 +++++++++++++++++++- src/video_core/textures/decoders.h | 3 +++ 3 files changed, 30 insertions(+), 1 deletion(-) diff --git a/src/video_core/texture_cache/texture_cache.h b/src/video_core/texture_cache/texture_cache.h index 93db205836..b4221c8c5c 100644 --- a/src/video_core/texture_cache/texture_cache.h +++ b/src/video_core/texture_cache/texture_cache.h @@ -56,6 +56,7 @@ using namespace Common::Literals; template TextureCache

::TextureCache(Runtime& runtime_, Tegra::MaxwellDeviceMemoryManager& device_memory_) : runtime{runtime_}, device_memory{device_memory_} { + Tegra::Texture::PrepareUnswizzleThreads(); debug_stats = std::getenv("EDEN_NX2_TC_LOG") != nullptr && Common::Log::IsLevelEnabled(Common::Log::Class::HW_GPU, Common::Log::Level::Debug); diff --git a/src/video_core/textures/decoders.cpp b/src/video_core/textures/decoders.cpp index 7d7bf60676..2683eab325 100644 --- a/src/video_core/textures/decoders.cpp +++ b/src/video_core/textures/decoders.cpp @@ -5,10 +5,12 @@ // SPDX-License-Identifier: GPL-2.0-or-later #include +#include #include #include #include #include +#include #include #include #include @@ -19,6 +21,8 @@ #include "common/assert.h" #include "common/bit_util.h" #include "common/div_ceil.h" +#include "common/logging.h" +#include "common/cpu_features.h" #include "common/thread.h" #include "video_core/gpu.h" #include "video_core/textures/decoders.h" @@ -65,11 +69,28 @@ class LinePool { } private: + /// At most 4 and a quarter of the host threads, and only the physical cores past the 6 the + /// guest cores and the GPU threads keep busy: on a 6-core Ryzen 5 5500, 3 workers competed + /// with them (no gain, slower movies). EDEN_NX2_UNSWIZZLE_THREADS sets the count (tests). + static u32 WorkerCount() { + if (const char* const text = std::getenv("EDEN_NX2_UNSWIZZLE_THREADS")) { + return (std::min)(static_cast(std::max(0, std::atoi(text))), 8U); + } + const u32 logical = std::thread::hardware_concurrency(); + u32 physical = logical / 2; + if (const auto cores = Common::GetProcessorCount(); cores && *cores > 0) { + physical = static_cast(*cores); + } + const u32 spare = physical > 6 ? physical - 6 : 0; + return (std::min)({4U, logical / 4, spare}); + } + LinePool() { - const u32 count = (std::min)(4U, std::thread::hardware_concurrency() / 4); + const u32 count = WorkerCount(); for (u32 i = 0; i < count; ++i) { workers.emplace_back([this](std::stop_token stop) { Loop(stop); }); } + LOG_INFO(HW_GPU, "Unswizzle helper threads: {}", count); } ~LinePool() { @@ -310,6 +331,10 @@ void Swizzle(std::span output, std::span input, u32 bytes_per_pixe } // Anonymous namespace +void PrepareUnswizzleThreads() { + [[maybe_unused]] const bool enabled = LinePool::Get().Enabled(); +} + void UnswizzleTexture(std::span output, std::span input, u32 bytes_per_pixel, u32 width, u32 height, u32 depth, u32 block_height, u32 block_depth, u32 stride_alignment, bool native_tiling) { diff --git a/src/video_core/textures/decoders.h b/src/video_core/textures/decoders.h index 6151967869..843c5bdf43 100644 --- a/src/video_core/textures/decoders.h +++ b/src/video_core/textures/decoders.h @@ -27,6 +27,9 @@ constexpr u32 SWIZZLE_X_BITS = 0b100101111; constexpr u32 SWIZZLE_Y_BITS = 0b011010000; /// Unswizzles a block linear texture into linear memory. +/// Starts the threads large unswizzles are split over now, instead of at the first one. +void PrepareUnswizzleThreads(); + void UnswizzleTexture(std::span output, std::span input, u32 bytes_per_pixel, u32 width, u32 height, u32 depth, u32 block_height, u32 block_depth, u32 stride_alignment = 1, bool native_tiling = false); From 57bc26a42b5b8f9e9bc4a407fb3a33938eed7e97 Mon Sep 17 00:00:00 2001 From: DjouWish Date: Sun, 8 Nov 2026 18:40:59 +0100 Subject: [PATCH 073/105] vk_query_cache: wait on each run's newest query before the availability copy #679's wait on the newest query, kept with the availability resolve on AMD's proprietary driver: the newest result is copied with the wait, the rest without it after a barrier, so fewer results come in late and resolve to "visible". On top of #805 on an RX 9070 XT, Ocarina of Time no longer fell below 30 FPS while running, steadier in motion. Co-Authored-By: Claude Opus 5.5 --- .../renderer_vulkan/vk_query_cache.cpp | 33 +++++++++++++++++-- 1 file changed, 30 insertions(+), 3 deletions(-) diff --git a/src/video_core/renderer_vulkan/vk_query_cache.cpp b/src/video_core/renderer_vulkan/vk_query_cache.cpp index b342fd1b33..5ef67a221e 100644 --- a/src/video_core/renderer_vulkan/vk_query_cache.cpp +++ b/src/video_core/renderer_vulkan/vk_query_cache.cpp @@ -332,8 +332,9 @@ class SamplesStreamer : public BaseStreamer { // occluded object drawn instead of hidden (the scene vanished) or a stall (9 -> 22 // FPS in Kokiri Forest). The availability word is what decides, not what the copy // writes for an unavailable query, which the Vulkan spec leaves undefined - // (approach from Savulska's test on #573). Waiting on the newest query instead - // (#679) gave 54 FPS under the Deku tree on an RX 9070 XT, this 56-58. + // (approach from Savulska's test on #573). Each run still waits on its newest query + // first (#679), so fewer results come in late: steadier frame times in motion on an + // RX 9070 XT. const bool resolve_availability = availability_pass.has_value(); const VkDeviceSize span_slots = span_end - span_start; const VkDeviceSize availability_offset = base_offset * 2; @@ -363,8 +364,34 @@ class SamplesStreamer : public BaseStreamer { } for (const auto& [start, end] : merged) { if (resolve_availability) { + const size_t last = end - 1; + const VkDeviceSize last_offset = + availability_offset + (last - span_start) * 16; cmdbuf.CopyQueryPoolResults( - query_pool, static_cast(start), static_cast(end - start), + query_pool, static_cast(last), 1, availability_buffer, + last_offset, 16, + VK_QUERY_RESULT_64_BIT | VK_QUERY_RESULT_WITH_AVAILABILITY_BIT | + VK_QUERY_RESULT_WAIT_BIT); + if (last == start) { + continue; + } + // Orders the unwaited copy after the waited one. + const VkBufferMemoryBarrier waited_barrier{ + .sType = VK_STRUCTURE_TYPE_BUFFER_MEMORY_BARRIER, + .pNext = nullptr, + .srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT, + .dstAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT, + .srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, + .dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, + .buffer = availability_buffer, + .offset = last_offset, + .size = 16, + }; + cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_TRANSFER_BIT, + VK_PIPELINE_STAGE_TRANSFER_BIT, 0, + waited_barrier); + cmdbuf.CopyQueryPoolResults( + query_pool, static_cast(start), static_cast(last - start), availability_buffer, availability_offset + (start - span_start) * 16, 16, VK_QUERY_RESULT_64_BIT | VK_QUERY_RESULT_WITH_AVAILABILITY_BIT); } else if (!wait_on_last || end - start == 1) { From 802c5e45925e6da2691f7a543da6b53a9fe9d89b Mon Sep 17 00:00:00 2001 From: jpkto <255363096+jpkto@users.noreply.github.com> Date: Sun, 8 Nov 2026 19:38:35 +0100 Subject: [PATCH 074/105] gpu_split: decide acquires on a word with a pending fenced release by memory 894a9e8386 kept a fence-flagged release out of memory until its fence ran. For an Equal or Mask acquire on such a word, AskView then always suspended, even when the word in memory already held the value. The drain path reads memory and passes there: the release has not landed yet. Once a newer release to the word lands, the word may never hold that value again, so the view could leave the channel waiting where the drain path, and nx2 itself, go on. A pending fenced release now lets memory decide, as after a drain: - A greater-or-equal acquire passes when memory meets it now. The words only advance, so the writes sent before still meet it. - Any other acquire, or one on a release whose value the front can't tell, first waits until the back ran the release. Then the writes sent to the word before it are in memory, and memory decides. Otherwise the acquire suspends. The release latches the waiter when it lands, or a poll finds the value in memory. Releases that land when the back runs them are decided as before. TEST: a fenced release that lands while the drain check drains can change the word the view read, as it can on the drain path itself. The check counts those passes apart ("changed by a fenced release landing during the drain"), so "not met after the drain" still has to stay 0. Found by a review of 894a9e8386, while looking into the 2-3 s Equal acquire waits two testers reported during host GPU stalls. Those waits are the fence latency #337 brings: their words did not hold the value yet. This change is about the other case. Co-Authored-By: Claude Opus 5.5 --- src/video_core/engines/puller.cpp | 51 ++++++++++++++++++++----------- src/video_core/nx2_drain_check.h | 18 ++++++++--- 2 files changed, 47 insertions(+), 22 deletions(-) diff --git a/src/video_core/engines/puller.cpp b/src/video_core/engines/puller.cpp index 3f70939949..a19c4f1dd6 100644 --- a/src/video_core/engines/puller.cpp +++ b/src/video_core/engines/puller.cpp @@ -164,15 +164,16 @@ ViewAnswer AskView(DmaPusher& dma_pusher, const Puller::PendingAcquire& acquire, // releases): a value that meets it stands once the back ran that work. const bool settled = acquire.WaitsToReach() || split.Ran(acquire.writers_before); // The last release the front sent to the word decides while it may not be in memory. - if (const auto queued = split.QueuedSemaphoreWriteAt(device)) { - if (queued->fenced) { - // It lands once a fence ran, at no position the front can wait for. A greater-or-equal - // acquire the word meets now and once it landed stands either way (the words only - // advance); else the landing latches the waiter, or a poll finds it in memory. - const bool landed_meets = - queued->known && acquire.WaitsToReach() && acquire.Matches(queued->payload); - return {landed_meets && acquire.Matches(memory()) ? Kind::Met : Kind::NotMet}; + const auto queued = split.QueuedSemaphoreWriteAt(device); + if (queued && queued->fenced) { + // It lands once a fence ran, at no position the front can wait for: memory decides, as + // after a drain, and the landing latches the waiter or a poll finds the word. Writes sent + // to the word before it are in memory once the back ran it. A greater-or-equal acquire + // memory meets now meets it then too (the words only advance). + if (!split.Ran(queued->position) && !(queued->known && acquire.WaitsToReach())) { + return {Kind::Unknown, queued->position}; } + } else if (queued) { if (!queued->known) { return {Kind::Unknown, queued->position}; } @@ -219,6 +220,25 @@ bool MetAfterDrain(DmaPusher& dma_pusher, const Puller::PendingAcquire& acquire, (waiter != nullptr && Nx2SemaphoreWaiters::Get().Latched(waiter)); } +// TEST (EDEN_NX2_DRAIN_CHECK): whether a fence-flagged release to the word at `device` may still +// land. One that lands during the drain can change the word the view read, as it can on the +// drain path itself. +bool FencedWritePending(u64 device) { + const auto queued = VideoCommon::GpuSplit::Get().QueuedSemaphoreWriteAt(device); + return queued && queued->fenced; +} + +// TEST: checks a pass of the view against the drain path: whether the drained word meets it, and +// whether a fenced release that landed during the drain changed it. +bool PassStands(DmaPusher& dma_pusher, const Puller::PendingAcquire& acquire, const void* waiter, + u64 device, bool poll, VideoCommon::Nx2DrainCheck::ViewPass how) { + const bool fenced = FencedWritePending(device); + const bool met = MetAfterDrain(dma_pusher, acquire, waiter); + VideoCommon::Nx2DrainCheck::NoteViewPass(poll, how, met, + !met && fenced && !FencedWritePending(device)); + return met; +} + // TEST: counts the view's decision on an acquire (EDEN_NX2_ACQUIRE_STATS) and checks it against // the drain path (EDEN_NX2_DRAIN_CHECK): a pass the drained word doesn't meet suspends instead, // a suspension the drained word meets passes. @@ -240,13 +260,12 @@ ViewAnswer CheckAcquire(DmaPusher& dma_pusher, const Puller::PendingAcquire& acq Nx2MayBeGpuWritten(device, sizeof(u32)), answer.kind == Kind::Flush); } if (Check::g_enabled) { - const bool met = MetAfterDrain(dma_pusher, acquire, nullptr); if (pass) { - Check::NoteViewPass(false, how, met); - if (!met) { + if (!PassStands(dma_pusher, acquire, nullptr, device, false, how)) { return {Kind::NotMet}; } } else { + const bool met = MetAfterDrain(dma_pusher, acquire, nullptr); Check::NoteViewWait(false, met); if (met) { return {Kind::Met}; @@ -258,13 +277,11 @@ ViewAnswer CheckAcquire(DmaPusher& dma_pusher, const Puller::PendingAcquire& acq // TEST: whether the view's resume of a poll stands (EDEN_NX2_DRAIN_CHECK). bool PollPassStands(DmaPusher& dma_pusher, const Puller::PendingAcquire& acquire, - const void* waiter, VideoCommon::Nx2DrainCheck::ViewPass how) { + const void* waiter, u64 device, VideoCommon::Nx2DrainCheck::ViewPass how) { if (!VideoCommon::Nx2DrainCheck::g_enabled) { return true; } - const bool met = MetAfterDrain(dma_pusher, acquire, waiter); - VideoCommon::Nx2DrainCheck::NoteViewPass(true, how, met); - return met; + return PassStands(dma_pusher, acquire, waiter, device, true, how); } // TEST: a suspended acquire resumes (EDEN_NX2_ACQUIRE_STATS). @@ -408,14 +425,14 @@ bool Puller::PollWithoutDrain(DmaPusher& dma_pusher, u64 device) { const ViewAnswer answer = AskView(dma_pusher, acquire, device); switch (answer.kind) { case Kind::Met: - if (!PollPassStands(dma_pusher, acquire, this, ViewPass::Memory)) { + if (!PollPassStands(dma_pusher, acquire, this, device, ViewPass::Memory)) { checked = true; break; } NoteResumed(this, acquire, Resume::Memory); return true; case Kind::MetByQueued: - if (!PollPassStands(dma_pusher, acquire, this, ViewPass::Queued)) { + if (!PollPassStands(dma_pusher, acquire, this, device, ViewPass::Queued)) { checked = true; break; } diff --git a/src/video_core/nx2_drain_check.h b/src/video_core/nx2_drain_check.h index 2534984ed6..755402afc1 100644 --- a/src/video_core/nx2_drain_check.h +++ b/src/video_core/nx2_drain_check.h @@ -63,6 +63,9 @@ struct State { /// not meet. std::array, 6> view_passes{}; std::array, 6> view_wrong{}; + /// Passes the drained word didn't meet because a fenced release landed during the drain: the + /// drain path itself may read either value then. + std::array, 6> view_raced{}; /// Acquires the view suspended and polls it kept waiting (by poll), and of those the ones /// the drained word met. std::array, 2> view_waits{}; @@ -129,13 +132,15 @@ enum class ViewPass : u32 { }; /// The view passed an acquire (or resumed a poll, `poll`); whether the word read after a drain -/// met it too. -inline void NoteViewPass(bool poll, ViewPass how, bool met_after_drain) { +/// met it too, and if not, whether a fenced release that landed during the drain changed it +/// (`raced`). +inline void NoteViewPass(bool poll, ViewPass how, bool met_after_drain, bool raced) { const size_t index = (poll ? 3U : 0U) + static_cast(how); State& state = Get(); state.view_passes[index].fetch_add(1, std::memory_order_relaxed); if (!met_after_drain) { - state.view_wrong[index].fetch_add(1, std::memory_order_relaxed); + (raced ? state.view_raced : state.view_wrong)[index].fetch_add(1, + std::memory_order_relaxed); } } @@ -438,12 +443,15 @@ inline void MaybeReport() { const u64 wrong_memory = take(state.view_wrong[base]); const u64 wrong_queued = take(state.view_wrong[base + 1]); const u64 wrong_waited = take(state.view_wrong[base + 2]); + const u64 raced = take(state.view_raced[base]) + take(state.view_raced[base + 1]) + + take(state.view_raced[base + 2]); return fmt::format("{} (by memory {}, by a release still queued {}, after a positioned " "wait {}), not met after the drain {} (of those by memory {}, queued " - "{}, after a wait {})", + "{}, after a wait {}), changed by a fenced release landing during the " + "drain {}", by_memory + by_queued + waited, by_memory, by_queued, waited, wrong_memory + wrong_queued + wrong_waited, wrong_memory, wrong_queued, - wrong_waited); + wrong_waited, raced); }; const auto waits = [&](bool poll) { const u64 count = take(state.view_waits[poll ? 1 : 0]); From 0758309bcce8d662662f8e4c815f983fba2dd781 Mon Sep 17 00:00:00 2001 From: jpkto <255363096+jpkto@users.noreply.github.com> Date: Sun, 8 Nov 2026 21:23:24 +0100 Subject: [PATCH 075/105] gpu_thread: publish what the front sent before it sleeps on suspended channels With no command queued and channels suspended, the front yields or sleeps (SleepForOneTick, or 1 ms) without publishing. The words it sent since its last publish wait out the sleep while an idle back waits for them. On nx2 alone the front drains for most acquires, and each drain publishes first, so the poll rarely holds anything. This branch removes those drains, which exposes the sleep. Publish does nothing when nothing new was sent, and no PublishHold is open there: holds only cover the front's QMD handling. Found and measured by hellasleeper108 in #805. At the Lon Lon Ranch spot (about 2,370 draws a frame) the frame rate went from 44.2 +- 4.0 to 48.0 +- 1.7 fps, late publishes from 180-230 to about 78 ms/s, and none were late by 1 ms or more. On nx2 alone it makes no difference. Co-authored-by: hellasleeper108 <216483438+hellasleeper108@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 --- src/video_core/gpu_thread.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/video_core/gpu_thread.cpp b/src/video_core/gpu_thread.cpp index 442099eae0..8cebc81d81 100644 --- a/src/video_core/gpu_thread.cpp +++ b/src/video_core/gpu_thread.cpp @@ -104,6 +104,11 @@ void ThreadManager::StartThread(VideoCore::RendererBase& renderer, Core::Fronten if (!scheduler.HasPendingChannels()) { continue; } + // Words sent since the last publish would wait out the sleep below while the + // back runs out of work: hand them over first. + if (split.Active()) { + split.Publish(); + } // The CPU usually releases the semaphore soon after: poll for half a // millisecond, then in short sleeps. Ocarina of Time waits here ~330 times a // second and its present waits for the work behind them: with 1 ms sleeps the From 34621dcd6baf7546d68d6d37955c501a14232ca3 Mon Sep 17 00:00:00 2001 From: jpkto <255363096+jpkto@users.noreply.github.com> Date: Sun, 8 Nov 2026 21:44:56 +0100 Subject: [PATCH 076/105] gpu: take the applied writes out of applying_writes on every exit InvalidateGPUCache publishes the address of its local writes vector in applying_writes and took it out again only after OnCacheInvalidations returned. If that call threw, the vector was gone while its address stayed, and the next ForEachPendingCpuWrite read freed memory. A scope guard now takes it out on every exit. Nothing else changes. Found by CodeRabbit's review of #805. Co-Authored-By: Claude Opus 5.5 --- src/video_core/gpu.cpp | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/video_core/gpu.cpp b/src/video_core/gpu.cpp index dabcf7ff17..2dc8c80cb6 100644 --- a/src/video_core/gpu.cpp +++ b/src/video_core/gpu.cpp @@ -13,6 +13,7 @@ #include #include "common/assert.h" +#include "common/scope_exit.h" #include "common/settings.h" #include "common/settings_enums.h" #include "core/core.h" @@ -127,9 +128,13 @@ struct GPU::Impl { } applying_writes.push_back(&writes); } + // Also if OnCacheInvalidations throws: ForEachPendingCpuWrite must not reach `writes` + // once it is gone. + SCOPE_EXIT { + std::scoped_lock lock{applying_writes_mutex}; + std::erase(applying_writes, &writes); + }; renderer->ReadRasterizer()->OnCacheInvalidations(writes); - std::scoped_lock lock{applying_writes_mutex}; - std::erase(applying_writes, &writes); } void ForEachPendingCpuWrite(const std::function& callback) { From b2304fa55ae1568abc98b6798df7bacfebb31d27 Mon Sep 17 00:00:00 2001 From: Copilot App <223556219+Copilot@users.noreply.github.com> Date: Mon, 9 Nov 2026 08:57:41 -0300 Subject: [PATCH 077/105] texture_cache: don't prepare the null image for views of empty descriptors With #805's memoized textures, PrepareImageView ran for remembered views of empty texture descriptors. Their image is the null image (slot 0, format Invalid, starting CpuModified), so a memo hit refreshed it: TrackImage at address 0, then UploadImageContents aborted in BitsPerBlock(Invalid) about 73 s after boot, while Ocarina of Time loaded its save. Fix found by @danieljesus in the comments of #805. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- src/video_core/texture_cache/texture_cache.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/video_core/texture_cache/texture_cache.h b/src/video_core/texture_cache/texture_cache.h index cfb3dc5b3c..2025f77712 100644 --- a/src/video_core/texture_cache/texture_cache.h +++ b/src/video_core/texture_cache/texture_cache.h @@ -4136,7 +4136,8 @@ void TextureCache

::PrepareImageView(ImageViewId image_view_id, bool is_modifi return; } const ImageViewBase& image_view = slot_image_views[image_view_id]; - if (image_view.IsBuffer()) { + // Views of empty descriptors point at the null image: there is nothing to upload or track. + if (image_view.IsBuffer() || image_view.image_id == NULL_IMAGE_ID) { return; } PrepareImage(image_view.image_id, is_modification, invalidate); From 74a38bc34cdadd299624129a72552a24d6057b0d Mon Sep 17 00:00:00 2001 From: thibaut Date: Sun, 8 Nov 2026 18:26:19 +0100 Subject: [PATCH 078/105] query_cache: keep fence-flagged payload releases synchronized at low accuracy with Default fences. That covers the payload releases channels sync on too: the front's acquires then wait for the host GPU, about 1,700 delayed fences a second in Ocarina of Time, and the game fell from ~48 to ~32 fps standing at Lon Lon Ranch (Windows, RTX 4090). A payload release writes a constant, so it stays synchronized as before #337: its fence may be skipped and the query freed at once. Reports of GPU counters, whose values depend on the work before them, keep #337's behaviour. Co-Authored-By: Claude Opus 5.5 --- src/video_core/query_cache/query_cache.h | 36 ++++++++++++++++++------ 1 file changed, 28 insertions(+), 8 deletions(-) diff --git a/src/video_core/query_cache/query_cache.h b/src/video_core/query_cache/query_cache.h index 7f3a4fe591..611b35d7c0 100644 --- a/src/video_core/query_cache/query_cache.h +++ b/src/video_core/query_cache/query_cache.h @@ -283,11 +283,21 @@ void QueryCacheBase::CounterReport(GPUVAddr addr, QueryType counter_type }; u8* pointer = impl->device_memory.template GetPointer(cpu_addr); u8* pointer_timestamp = impl->device_memory.template GetPointer(cpu_addr + 8); + // A fence-flagged payload release writes a constant, so at low accuracy with Default fences + // it stays synchronized as before #337: its fence may be skipped and the query freed at once. + // Only reports of GPU counters wait for their fence to run. + const bool is_synced = is_fence && counter_type == QueryType::Payload && + (Settings::IsGPUFenceBehaviorDefault() + ? !Settings::IsGPULevelHigh() + : !Settings::IsGPUFenceBehaviorBalanced() && + !Settings::IsGPUFenceBehaviorAccurate()); // The GPU thread split's back entry this report came in: the front counts the report in - // memory once its operation ran with a fence (GpuSplit::NoteFencedWrite). - const std::optional split_entry = VideoCommon::GpuSplit::Get().BackEntry(); - std::function operation([this, streamer, query_base = query, query_location, - pointer, pointer_timestamp, cpu_addr, addr, + // memory once its operation ran with a fence (GpuSplit::NoteFencedWrite). A synced release's + // operation doesn't run in fence order, so it must not count earlier reports as landed. + const std::optional split_entry = + is_synced ? std::nullopt : VideoCommon::GpuSplit::Get().BackEntry(); + std::function operation([this, is_synced, streamer, query_base = query, + query_location, pointer, pointer_timestamp, cpu_addr, addr, asid = gpu_memory->GetID(), counter_type, split_entry] { SCOPE_EXIT { if (split_entry) { @@ -302,7 +312,9 @@ void QueryCacheBase::CounterReport(GPUVAddr addr, QueryType counter_type trace, asid, addr, cpu_addr, static_cast(counter_type), static_cast(query_base->flags)); } - impl->pending_unregister.push_back(query_location); + if (!is_synced) [[likely]] { + impl->pending_unregister.push_back(query_location); + } return; } if (False(query_base->flags & QueryFlagBits::IsFinalValueSynced)) [[unlikely]] { @@ -315,7 +327,9 @@ void QueryCacheBase::CounterReport(GPUVAddr addr, QueryType counter_type } LOG_ERROR(HW_GPU, "Query report value not synchronized. Consider increasing GPU accuracy."); - impl->pending_unregister.push_back(query_location); + if (!is_synced) [[likely]] { + impl->pending_unregister.push_back(query_location); + } return; } query_base->value += streamer->GetAmendValue(); @@ -342,12 +356,18 @@ void QueryCacheBase::CounterReport(GPUVAddr addr, QueryType counter_type wide ? query_base->value : static_cast(query_base->value), static_cast(query_base->flags)); } - impl->pending_unregister.push_back(query_location); + if (!is_synced) [[likely]] { + impl->pending_unregister.push_back(query_location); + } }); if (is_fence) { - VideoCore::g_signaling_query_report_fence = true; + VideoCore::g_signaling_query_report_fence = !is_synced; impl->rasterizer.SignalFence(std::move(operation)); VideoCore::g_signaling_query_report_fence = false; + if (is_synced) { + streamer->Free(new_query_id); + return; + } } else { if (!Settings::IsGPULevelHigh() && counter_type == QueryType::Payload) { FinishReads(cpu_addr, has_timestamp ? 16 : 4); From 630416afe11580956792d9104c9402ba947f04b1 Mon Sep 17 00:00:00 2001 From: thibaut Date: Mon, 9 Nov 2026 09:06:13 +0100 Subject: [PATCH 079/105] query_cache: drop the [[likely]] hints on the synced checks (docs/Coding.md) Co-Authored-By: Claude Opus 5.5 --- src/video_core/query_cache/query_cache.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/video_core/query_cache/query_cache.h b/src/video_core/query_cache/query_cache.h index 611b35d7c0..c151fa0e37 100644 --- a/src/video_core/query_cache/query_cache.h +++ b/src/video_core/query_cache/query_cache.h @@ -312,7 +312,7 @@ void QueryCacheBase::CounterReport(GPUVAddr addr, QueryType counter_type trace, asid, addr, cpu_addr, static_cast(counter_type), static_cast(query_base->flags)); } - if (!is_synced) [[likely]] { + if (!is_synced) { impl->pending_unregister.push_back(query_location); } return; @@ -327,7 +327,7 @@ void QueryCacheBase::CounterReport(GPUVAddr addr, QueryType counter_type } LOG_ERROR(HW_GPU, "Query report value not synchronized. Consider increasing GPU accuracy."); - if (!is_synced) [[likely]] { + if (!is_synced) { impl->pending_unregister.push_back(query_location); } return; @@ -356,7 +356,7 @@ void QueryCacheBase::CounterReport(GPUVAddr addr, QueryType counter_type wide ? query_base->value : static_cast(query_base->value), static_cast(query_base->flags)); } - if (!is_synced) [[likely]] { + if (!is_synced) { impl->pending_unregister.push_back(query_location); } }); From f837a6640a0072c76e4af915e27fb56cc39dcba2 Mon Sep 17 00:00:00 2001 From: Copilot App <223556219+Copilot@users.noreply.github.com> Date: Mon, 9 Nov 2026 09:06:40 -0300 Subject: [PATCH 080/105] gpu_split: synced payload releases land as the back runs them (#874 with #805) With #874, a fence-flagged payload release below High GPU accuracy with Default fences runs its CounterReport operation at once instead of with its fence. As jpkto described in #874 for whichever of the two merges second: - its operation doesn't call NoteFencedWrite: it no longer runs in fence order and would count earlier counter reports that still wait for their fence as landed (the split entry is captured only when !is_synced); - the front notes those releases as Landing::AtRun under the same condition (GpuSplit::FencedPayloadLanding): the puller's, the copy engine's, QMD plain and four-word, short 3D and one-word compute reports. Counter reports stay Fenced. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- src/video_core/dma_pusher.cpp | 5 +++-- src/video_core/engines/kepler_compute.cpp | 8 ++++---- src/video_core/engines/maxwell_3d.cpp | 8 +++++--- src/video_core/engines/puller.cpp | 4 ++-- src/video_core/front_qmd.h | 12 +++++++----- src/video_core/gpu_split.cpp | 9 +++++++++ src/video_core/gpu_split.h | 4 ++++ 7 files changed, 34 insertions(+), 16 deletions(-) diff --git a/src/video_core/dma_pusher.cpp b/src/video_core/dma_pusher.cpp index 3fdcf994cd..01f9aa03f6 100644 --- a/src/video_core/dma_pusher.cpp +++ b/src/video_core/dma_pusher.cpp @@ -771,8 +771,9 @@ void DmaPusher::NoteEngineWrites(Engines::EngineTypes type, u32 method) { if (!device) { return; } - // Fence-flagged reports: they land once their fence ran. - constexpr auto landing = VideoCommon::GpuSplit::Landing::Fenced; + // Fence-flagged payload releases: they land once their fence ran, or as the back runs them + // where CounterReport keeps them synchronized. + const auto landing = VideoCommon::GpuSplit::FencedPayloadLanding(); split.NoteSemaphoreWrite(*device, dma_registers[MaxwellDMA::SemaphoreMethod + 2], true, false, landing); VideoCommon::Nx2AcquireStats::NoteRelease(*device, diff --git a/src/video_core/engines/kepler_compute.cpp b/src/video_core/engines/kepler_compute.cpp index f8e33c4336..999b03c614 100644 --- a/src/video_core/engines/kepler_compute.cpp +++ b/src/video_core/engines/kepler_compute.cpp @@ -1895,11 +1895,11 @@ void KeplerCompute::FrontNoteReport(VideoCommon::GpuSplit& split, const FrontSta base.has_value()); return; } - // One word: a fence-flagged report, which lands once its fence ran. Else a payload report - // with a timestamp, written as the back runs it. + // One word: a fence-flagged payload report (GpuSplit::FencedPayloadLanding). Else a payload + // report with a timestamp, written as the back runs it. const bool one_word = ((control >> 28) & 1) != 0; - const auto landing = - one_word ? VideoCommon::GpuSplit::Landing::Fenced : VideoCommon::GpuSplit::Landing::AtRun; + const auto landing = one_word ? VideoCommon::GpuSplit::FencedPayloadLanding() + : VideoCommon::GpuSplit::Landing::AtRun; split.NoteSemaphoreWrite(*device, fs.registers[0x6c2], true, false, landing); if (!one_word) { // 64 bits (a timestamp follows). diff --git a/src/video_core/engines/maxwell_3d.cpp b/src/video_core/engines/maxwell_3d.cpp index 71346a4ed2..3f2839b712 100644 --- a/src/video_core/engines/maxwell_3d.cpp +++ b/src/video_core/engines/maxwell_3d.cpp @@ -1608,11 +1608,13 @@ void Maxwell3D::FrontNoteReport(VideoCommon::GpuSplit& split, const FrontState& } const bool payload = report.query.report == Regs::ReportSemaphore::Report::Payload; // As QueryCacheBase::CounterReport writes it: a payload report without a fence flag as the - // back runs it, a short release (fence-flagged) or a counter once a fence ran. + // back runs it, a short (fence-flagged) payload release as FencedPayloadLanding says, a + // counter once a fence ran. const bool fence_flagged = operation == Regs::ReportSemaphore::Operation::Release && report.query.short_query != 0; - const auto landing = payload && !fence_flagged ? VideoCommon::GpuSplit::Landing::AtRun - : VideoCommon::GpuSplit::Landing::Fenced; + const auto landing = !payload ? VideoCommon::GpuSplit::Landing::Fenced + : fence_flagged ? VideoCommon::GpuSplit::FencedPayloadLanding() + : VideoCommon::GpuSplit::Landing::AtRun; split.NoteSemaphoreWrite(*device, report.payload, payload, false, landing); VideoCommon::Nx2AcquireStats::NoteRelease(*device, VideoCommon::Nx2AcquireStats::Releaser::Report3D); diff --git a/src/video_core/engines/puller.cpp b/src/video_core/engines/puller.cpp index 1636a934cb..682b543512 100644 --- a/src/video_core/engines/puller.cpp +++ b/src/video_core/engines/puller.cpp @@ -96,9 +96,9 @@ void NoteRelease(DmaPusher& dma_pusher, GPUVAddr address, u32 payload, return; } const bool four_words = True(properties & VideoCommon::QueryPropertiesFlags::HasTimeout); - // A fence-flagged report lands once the fence after it ran (QueryCacheBase::CounterReport). + // A fence-flagged payload release lands as GpuSplit::FencedPayloadLanding says. const auto landing = True(properties & VideoCommon::QueryPropertiesFlags::IsAFence) - ? VideoCommon::GpuSplit::Landing::Fenced + ? VideoCommon::GpuSplit::FencedPayloadLanding() : VideoCommon::GpuSplit::Landing::AtRun; split.NoteWrite(dma_pusher.memory_manager, address, four_words ? 16 : sizeof(u32)); const auto device = dma_pusher.memory_manager.GpuToCpuAddress(address); diff --git a/src/video_core/front_qmd.h b/src/video_core/front_qmd.h index 75a74e0d23..3ecba92758 100644 --- a/src/video_core/front_qmd.h +++ b/src/video_core/front_qmd.h @@ -572,10 +572,11 @@ class FrontQmdView { split.NoteSemaphoreWrite(*device, value, base.has_value()); check(*device, value, base.has_value(), false); } else if (four_words) { - // The payload as 64 bits, a timestamp after it. Fence-flagged reports, like the plain - // release below: they land once their fence ran. - split.NoteSemaphoreWrite(*device, payload, true, false, GpuSplit::Landing::Fenced); - split.NoteSemaphoreWrite(*device + 4, 0, true, false, GpuSplit::Landing::Fenced); + // The payload as 64 bits, a timestamp after it. Fence-flagged payload releases, like + // the plain release below (GpuSplit::FencedPayloadLanding). + const auto landing = GpuSplit::FencedPayloadLanding(); + split.NoteSemaphoreWrite(*device, payload, true, false, landing); + split.NoteSemaphoreWrite(*device + 4, 0, true, false, landing); check(*device, payload, true, false); check(*device + 4, 0, true, false); } else if (maximum) { @@ -584,7 +585,8 @@ class FrontQmdView { split.NoteSemaphoreWrite(*device, value, true, !base.has_value()); check(*device, value, true, !base.has_value()); } else { - split.NoteSemaphoreWrite(*device, payload, true, false, GpuSplit::Landing::Fenced); + split.NoteSemaphoreWrite(*device, payload, true, false, + GpuSplit::FencedPayloadLanding()); check(*device, payload, true, false); } } diff --git a/src/video_core/gpu_split.cpp b/src/video_core/gpu_split.cpp index c10008157b..3649d97e46 100644 --- a/src/video_core/gpu_split.cpp +++ b/src/video_core/gpu_split.cpp @@ -300,6 +300,15 @@ bool GpuSplit::WaitForSyncedWriters(DrainSite site) { return WaitFor(std::min({last_writer_position, last_sync_position, write_position}), site); } +GpuSplit::Landing GpuSplit::FencedPayloadLanding() { + // As QueryCacheBase::CounterReport decides is_synced. + const bool synced = Settings::IsGPUFenceBehaviorDefault() + ? !Settings::IsGPULevelHigh() + : !Settings::IsGPUFenceBehaviorBalanced() && + !Settings::IsGPUFenceBehaviorAccurate(); + return synced ? Landing::AtRun : Landing::Fenced; +} + void GpuSplit::NoteSemaphoreWrite(u64 device_address, u32 payload, bool known, bool at_least, Landing landing) { if (semaphore_writes.size() >= semaphore_writes_limit) { diff --git a/src/video_core/gpu_split.h b/src/video_core/gpu_split.h index f11b6843a8..d6c096f2b6 100644 --- a/src/video_core/gpu_split.h +++ b/src/video_core/gpu_split.h @@ -353,6 +353,10 @@ class GpuSplit { Fenced, ///< Once the fence signaled after it ran (QueryCacheBase::CounterReport): ///< fence-flagged reports, counter reports }; + /// When a fence-flagged payload release reaches memory. Below High GPU accuracy with Default + /// fences, CounterReport keeps it synchronized (is_synced): its operation runs as the back + /// runs it, without waiting for the fence. + static Landing FencedPayloadLanding(); /// A semaphore release was just sent: once it landed (`landing`), the word at /// `device_address` holds `payload`, or a value the front can't tell (`known` false: a /// reduction, a counter report), or `payload` at least (`at_least`: a maximum with a value the From 762b7cd5c0111c16f6fefff17714e2ac3ee2c11f Mon Sep 17 00:00:00 2001 From: Zortos Date: Sun, 8 Nov 2026 20:44:06 +0100 Subject: [PATCH 081/105] vulkan: free compute IR before a synchronous driver compile (precache memory) (#878) * vulkan: free compute IR before a synchronous driver compile Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * vulkan: release the SPIR-V allocation too, not just its contents Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --------- Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> (cherry picked from commit fd4ce24c9b9390e660be345b467521e7fb08dfd3) --- src/shader_recompiler/object_pool.h | 7 +++ src/tests/CMakeLists.txt | 1 + src/tests/video_core/shader_object_pool.cpp | 56 +++++++++++++++++++ .../renderer_vulkan/vk_pipeline_cache.cpp | 13 ++++- 4 files changed, 75 insertions(+), 2 deletions(-) create mode 100644 src/tests/video_core/shader_object_pool.cpp diff --git a/src/shader_recompiler/object_pool.h b/src/shader_recompiler/object_pool.h index c39d3527ea..4d3d337d29 100644 --- a/src/shader_recompiler/object_pool.h +++ b/src/shader_recompiler/object_pool.h @@ -23,6 +23,13 @@ class ObjectPool { return std::construct_at(Memory(), std::forward(args)...); } + /// Destroys the objects and returns their memory; ReleaseContents keeps it for reuse + void Free() { + chunks.clear(); + chunks.shrink_to_fit(); + node = &chunks.emplace_back(new_chunk_size); + } + void ReleaseContents() { if (chunks.empty()) { return; diff --git a/src/tests/CMakeLists.txt b/src/tests/CMakeLists.txt index df335287bf..fedeae91c9 100644 --- a/src/tests/CMakeLists.txt +++ b/src/tests/CMakeLists.txt @@ -104,6 +104,7 @@ add_executable(tests video_core/ampere_tennis_tessellation.cpp video_core/pipeline_cache_files.cpp video_core/shader_precache.cpp + video_core/shader_object_pool.cpp video_core/storage_binding_capacity.cpp video_core/native_indirect_reset.cpp video_core/native_qmd.cpp diff --git a/src/tests/video_core/shader_object_pool.cpp b/src/tests/video_core/shader_object_pool.cpp new file mode 100644 index 0000000000..2a72917258 --- /dev/null +++ b/src/tests/video_core/shader_object_pool.cpp @@ -0,0 +1,56 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include + +#include "shader_recompiler/object_pool.h" + +namespace { +struct Counted { + static inline int live = 0; + + explicit Counted(int value_) : value{value_} { + ++live; + } + ~Counted() { + --live; + } + + int value; +}; +} // Anonymous namespace + +TEST_CASE("ObjectPool::Free destroys every object across chunks", "[shader]") { + Counted::live = 0; + { + Shader::ObjectPool pool{4}; + for (int index = 0; index < 10; ++index) { + REQUIRE(pool.Create(index)->value == index); + } + REQUIRE(Counted::live == 10); + pool.Free(); + REQUIRE(Counted::live == 0); + } + REQUIRE(Counted::live == 0); +} + +TEST_CASE("ObjectPool stays usable after Free", "[shader]") { + Counted::live = 0; + { + Shader::ObjectPool pool{4}; + for (int index = 0; index < 6; ++index) { + (void)pool.Create(index); + } + pool.Free(); + Counted* const first{pool.Create(100)}; + for (int index = 0; index < 8; ++index) { + (void)pool.Create(index); + } + REQUIRE(first->value == 100); + REQUIRE(Counted::live == 9); + pool.ReleaseContents(); + REQUIRE(Counted::live == 0); + (void)pool.Create(7); + REQUIRE(Counted::live == 1); + } + REQUIRE(Counted::live == 0); +} diff --git a/src/video_core/renderer_vulkan/vk_pipeline_cache.cpp b/src/video_core/renderer_vulkan/vk_pipeline_cache.cpp index 341f7f1813..d78c4ec710 100644 --- a/src/video_core/renderer_vulkan/vk_pipeline_cache.cpp +++ b/src/video_core/renderer_vulkan/vk_pipeline_cache.cpp @@ -2386,7 +2386,7 @@ ComputePipelineCacheEntry PipelineCache::CreateComputePipeline( max_shared_memory / 1024); program.shared_memory_size = max_shared_memory; } - const std::vector code{EmitSPIRV(profile, program)}; + std::vector code{EmitSPIRV(profile, program)}; device.SaveShader(code); vk::ShaderModule spv_module{BuildShader(device, code)}; @@ -2411,9 +2411,18 @@ ComputePipelineCacheEntry PipelineCache::CreateComputePipeline( spv_module.SetObjectNameEXT(name.c_str()); } Common::ThreadWorker* const thread_worker{build_in_parallel ? &workers : nullptr}; + const Shader::Info info{std::move(program.info)}; + if (!thread_worker) { + // The driver compiles on this thread. Precache and boot workers would otherwise hold their + // IR through the compile, up to hundreds of MiB for one large shader. + program = {}; + code = std::vector{}; + pools.block.Free(); + pools.inst.Free(); + } return {.pipeline = std::make_unique( device, scheduler, vulkan_pipeline_cache, descriptor_pool, guest_descriptor_queue, - descriptor_buffer_ring, thread_worker, statistics, &shader_notify, program.info, + descriptor_buffer_ring, thread_worker, statistics, &shader_notify, info, std::move(spv_module), key.unique_hash), .null_pointer_slots = env.IsNativeProgram() ? env.NullStoragePointerSlots() : std::vector>{}, From 05bea620e49adea4a119e7f9da958f0c576c8d67 Mon Sep 17 00:00:00 2001 From: Zortos Date: Sun, 8 Nov 2026 20:44:55 +0100 Subject: [PATCH 082/105] nx2 video: pick libdav1d for AV1 explicitly, cap its threads, log the decoder choice (#879) * nx2 video: decode AV1 with libdav1d on a capped thread count and log the decoder choice FFmpeg's own av1 decoder only decodes through a GPU hwaccel, so AV1 asks for libdav1d by name instead of relying on FFmpeg's registration order. dav1d gets half the host threads (2 to 8) instead of one per host thread. HardwareContext only creates the GPU devices the decoder has a hardware config for. With NVDEC emulation on GPU (the first-run default), an AV1 movie no longer creates CUDA, D3D11VA, DXVA2, D3D12VA and Vulkan devices before falling back to libdav1d. Each nx2 decoder logs one line: GPU or CPU, the decoder, its threads, why and how long it took to open. The vdec trace adds decode_us per input. * tests: skip the AV1 decode test when FFmpeg has no libdav1d (cherry picked from commit 64b962aa1e94cbffc1c491dda46e9297babbd4ce) --- src/core/hle/service/nx2_gsv.h | 8 +- src/tests/CMakeLists.txt | 1 + src/tests/video_core/nx2_video_av1.cpp | 141 ++++++++++++++++++++ src/video_core/host1x/ffmpeg.cpp | 11 +- src/video_core/host1x/nx2_video_decoder.cpp | 36 ++++- 5 files changed, 190 insertions(+), 7 deletions(-) create mode 100644 src/tests/video_core/nx2_video_av1.cpp diff --git a/src/core/hle/service/nx2_gsv.h b/src/core/hle/service/nx2_gsv.h index be88d6fcb4..9efbd35e34 100644 --- a/src/core/hle/service/nx2_gsv.h +++ b/src/core/hle/service/nx2_gsv.h @@ -3187,18 +3187,22 @@ class INx2VideoSession final : public ServiceFramework { decoder = std::make_unique(codec); } pictures.clear(); + const auto decode_start = std::chrono::steady_clock::now(); decoder->Decode(input.data, input.pts, pictures); if (input.end) { decoder->Drain(pictures); } + const auto decode_us = std::chrono::duration_cast( + std::chrono::steady_clock::now() - decode_start) + .count(); lk.lock(); m_decoded_frames += pictures.size(); if (Nx2VideoTraceEnabled()) { LOG_INFO(Service, "nx2:vdec trace decode syncpoint={} codec={} open={} pts={} eos={} " - "pictures={} decoded_frames={} discarded_by_flush={}", + "pictures={} decoded_frames={} discarded_by_flush={} decode_us={}", m_syncpoint, decoder->Name(), decoder->IsOpen(), input.pts, input.end, - pictures.size(), m_decoded_frames, m_flush); + pictures.size(), m_decoded_frames, m_flush, decode_us); } if (m_flush) { continue; diff --git a/src/tests/CMakeLists.txt b/src/tests/CMakeLists.txt index fedeae91c9..18014b004a 100644 --- a/src/tests/CMakeLists.txt +++ b/src/tests/CMakeLists.txt @@ -111,6 +111,7 @@ add_executable(tests video_core/fast_draw.cpp video_core/gpu_command_safety.cpp video_core/syncpoint_actions.cpp + video_core/nx2_video_av1.cpp video_core/nx2_video_render.cpp video_core/ampere_matrix.cpp video_core/spirv_software_matrix.cpp diff --git a/src/tests/video_core/nx2_video_av1.cpp b/src/tests/video_core/nx2_video_av1.cpp new file mode 100644 index 0000000000..b636815193 --- /dev/null +++ b/src/tests/video_core/nx2_video_av1.cpp @@ -0,0 +1,141 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include +#include +#include + +#include + +#include "common/scope_exit.h" +#include "common/settings.h" +#include "video_core/host1x/ffmpeg.h" +#include "video_core/host1x/nx2_video_decoder.h" + +namespace { +using Tegra::Host1x::Nx2Picture; +using Tegra::Host1x::Nx2PictureFormat; +using Tegra::Host1x::Nx2VideoDecoder; + +// Four 64x48 temporal units of FFmpeg's testsrc2 encoded by libaom (8-bit 4:2:0, limited range): +// a key frame and three inter frames. +constexpr std::array Av1Frame0{ + 0x12, 0x00, 0x0a, 0x0a, 0x00, 0x00, 0x00, 0x02, 0xaf, 0xf7, 0x9b, 0x5f, 0x20, 0x08, 0x32, 0xbb, + 0x06, 0x10, 0x00, 0xab, 0x00, 0x28, 0xa2, 0x8a, 0x24, 0x10, 0x40, 0xdd, 0x4a, 0xd1, 0x0d, 0x57, + 0x48, 0x9a, 0x76, 0x9f, 0xde, 0x5c, 0x71, 0x1d, 0x97, 0x40, 0x0d, 0xf2, 0x3b, 0x30, 0x75, 0x40, + 0x66, 0xec, 0xe4, 0x30, 0xb6, 0xdb, 0xdf, 0x4c, 0xf4, 0x1b, 0x62, 0x0d, 0x6b, 0x8a, 0x83, 0xca, + 0x97, 0xdf, 0x88, 0x01, 0x67, 0xf1, 0xf7, 0x61, 0x01, 0x4f, 0x55, 0x89, 0x53, 0x3c, 0x1c, 0x3c, + 0xe8, 0x49, 0x40, 0x18, 0xd1, 0x66, 0x06, 0x5a, 0xb8, 0xb1, 0xb4, 0x3b, 0xc8, 0xc8, 0x9d, 0x65, + 0x9b, 0xbc, 0x51, 0x45, 0x26, 0x1f, 0xaf, 0x6d, 0xe0, 0x55, 0x38, 0x89, 0xe6, 0x15, 0x17, 0xca, + 0x3c, 0xcf, 0xaf, 0x85, 0xa4, 0x36, 0xc0, 0xce, 0xfb, 0x6e, 0x5d, 0x94, 0x53, 0x64, 0xb7, 0x49, + 0x0b, 0x19, 0xdf, 0x24, 0x4e, 0xce, 0xe8, 0x3f, 0x75, 0x5c, 0x49, 0x0f, 0x0b, 0xa5, 0x12, 0x9c, + 0x87, 0xc2, 0x7e, 0x7a, 0xfc, 0x53, 0x55, 0x46, 0xa2, 0xc7, 0x17, 0x4c, 0x84, 0x5d, 0xf0, 0x85, + 0x6b, 0x1e, 0x5a, 0xa9, 0xb3, 0x98, 0xfd, 0x8a, 0xdb, 0x06, 0xd3, 0x1b, 0x1d, 0x59, 0xbc, 0x42, + 0xcc, 0x81, 0xd6, 0x70, 0x21, 0x0a, 0xa5, 0x05, 0xe6, 0x62, 0x8d, 0x77, 0x6a, 0xad, 0x17, 0xf1, + 0x84, 0x1e, 0x8e, 0x9b, 0xb3, 0xbd, 0x40, 0xa5, 0x8c, 0x19, 0x9e, 0x95, 0x47, 0xf4, 0xb3, 0x95, + 0x10, 0xdc, 0x83, 0xc5, 0x2d, 0x91, 0xa3, 0xd5, 0xfc, 0x6e, 0x2c, 0x0e, 0x39, 0x91, 0xd1, 0xfb, + 0x30, 0x0b, 0xa5, 0xaf, 0xe2, 0x95, 0xb7, 0xfd, 0x2d, 0x07, 0x16, 0x0c, 0x2d, 0x7d, 0x8b, 0x20, + 0xc3, 0x26, 0xbf, 0x56, 0xa3, 0xd4, 0x45, 0xc5, 0x63, 0x22, 0xb6, 0xb4, 0x80, 0x5b, 0x99, 0x11, + 0x11, 0x30, 0xac, 0x04, 0xa3, 0x17, 0xc0, 0xf3, 0x25, 0xda, 0x8a, 0xd8, 0x41, 0x9c, 0x9f, 0x46, + 0xf9, 0x3e, 0x5f, 0x94, 0xd3, 0xb4, 0x6d, 0x41, 0x64, 0x5f, 0x21, 0xd2, 0xf7, 0xe2, 0x2b, 0x7e, + 0x6b, 0xe2, 0x0f, 0x7e, 0x7b, 0xe7, 0x01, 0x9b, 0x88, 0xfc, 0x0c, 0xac, 0xd2, 0x51, 0xf1, 0x7a, + 0xbe, 0x32, 0xee, 0xc1, 0xa9, 0xb0, 0x98, 0xcd, 0x7b, 0x5b, 0x17, 0xa8, 0xfe, 0x47, 0xd6, 0x34, + 0xa3, 0x04, 0xf5, 0xd5, 0x44, 0xc4, 0x6d, 0xab, 0x94, 0x80, 0x58, 0xd1, 0x3f, 0xeb, 0xbd, 0x86, + 0x97, 0xa8, 0x7c, 0x60, 0x64, 0xe1, 0x79, 0x66, 0xe3, 0x1e, 0x29, 0x61, 0xe3, 0x42, 0x0e, 0xe1, + 0xe4, 0xb2, 0x74, 0x6a, 0x86, 0x7d, 0xd0, 0x26, 0xf5, 0xc3, 0x78, 0xbf, 0xc3, 0xca, 0x32, 0x49, + 0xe0, 0x7f, 0xfe, 0xfa, 0xa7, 0xbe, 0x7a, 0x11, 0x3a, 0x1d, 0x94, 0x5a, 0x1b, 0x40, 0xbe, 0xe0, + 0x3f, 0xb0, 0x0a, 0xa2, 0x65, 0xa0, 0x91, 0xd1, 0xeb, 0x24, 0x8c, 0x90, 0x39, 0x70, 0xb4, 0x92, + 0x55, 0xd2, 0xe6, 0xda, 0xd3, 0xaf, 0x47, 0xcf, 0xdf, 0x46, 0xc7, 0x08, 0x11, 0x61, 0x50, 0x81, + 0xb3, 0xe6, 0x35, 0x31, 0x9d, 0x9d, 0x3e, 0xd4, 0x26, 0x5c, 0x56, 0x40, 0x64, 0x40, 0xd2, 0xb0, + 0x67, 0x3e, 0x9e, 0xd2, 0xce, 0x17, 0x8c, 0x76, 0x78, 0xfc, 0xf5, 0xef, 0x71, 0x36, 0x3c, 0xf5, + 0xf3, 0xd1, 0x4a, 0x48, 0xc5, 0x88, 0x59, 0xe7, 0xa9, 0xa1, 0xf6, 0x39, 0xf0, 0x1d, 0x8a, 0xa2, + 0x45, 0xfb, 0xc2, 0xaa, 0xbc, 0x35, 0x50, 0x25, 0x57, 0xc6, 0x14, 0xc1, 0xac, 0x97, 0x4e, 0x42, + 0x69, 0xcd, 0x12, 0x20, 0x79, 0x2e, 0x1f, 0x03, 0x3f, 0x7e, 0x3c, 0x9c, 0x3d, 0xb0, 0x1e, 0xdb, + 0xcc, 0x0f, 0xc8, 0x1a, 0x69, 0x36, 0xc8, 0x8f, 0xaf, 0x3c, 0x93, 0x4a, 0xc7, 0x02, 0x57, 0x45, + 0xda, 0x49, 0xee, 0x7c, 0x2a, 0x46, 0x3c, 0x13, 0xd6, 0x24, 0x64, 0xc9, 0x0a, 0x79, 0xf1, 0xcc, + 0x2c, 0x30, 0x7a, 0xe8, 0x34, 0xe6, 0x0f, 0x6a, 0x9e, 0x51, 0x6c, 0xd1, 0x8d, 0x6e, 0x29, 0xe5, + 0x17, 0xc8, 0xed, 0x9a, 0x13, 0xf9, 0x86, 0x0a, 0x26, 0xcf, 0x92, 0x92, 0x68, 0xce, 0x12, 0xa8, + 0xe5, 0xde, 0xef, 0xe4, 0x5c, 0x31, 0xed, 0x0d, 0xb5, 0x36, 0x08, 0xbf, 0x33, 0x4a, 0x80, 0x38, + 0x14, 0x14, 0x99, 0x6d, 0xa9, 0xa1, 0x9b, 0xd6, 0xa1, 0x7a, 0x25, 0x92, 0x24, 0x01, 0x8d, 0x44, + 0x2f, 0x1f, 0x9b, 0xa2, 0x11, 0xc2, 0x1d, 0x24, 0xe2, 0xb7, 0x91, 0xab, 0xba, 0xeb, 0xd9, 0xb0, + 0x3c, 0xd1, 0x6f, 0x01, 0x76, 0xee, 0x45, 0x02, 0xa8, 0x11, 0xf1, 0x8d, 0xad, 0x4c, 0xfe, 0x87, + 0xa1, 0xc7, 0x90, 0x05, 0x22, 0x30, 0x88, 0xf2, 0xf4, 0xdb, 0x8f, 0x38, 0x77, 0x89, 0x65, 0xf4, + 0xc3, 0x1f, 0xc2, 0x3b, 0x26, 0xce, 0xf3, 0xc7, 0x9d, 0x68, 0x0a, 0xac, 0x7e, 0x2c, 0x93, 0xbc, + 0xc3, 0x29, 0xf2, 0xcd, 0xe9, 0x75, 0x69, 0x0b, 0xfe, 0x86, 0xa9, 0x7a, 0x92, 0xfd, 0x8a, 0xf8, + 0x3e, 0x83, 0x02, 0xaf, 0xca, 0x68, 0x2c, 0x57, 0x6a, 0x0e, 0x42, 0x20, 0x44, 0x43, 0xc8, 0xcf, + 0x78, 0x8b, 0xde, 0xe7, 0xd6, 0xe7, 0xd3, 0x9b, 0x71, 0xe6, 0xb4, 0xd8, 0x8b, 0x9a, 0x35, 0xd6, + 0x07, 0xe6, 0x96, 0xc1, 0x2a, 0x45, 0xef, 0xb9, 0xed, 0xfd, 0x2e, 0xe2, 0xfd, 0x90, 0x9c, 0x19, + 0x45, 0xa1, 0x14, 0xa4, 0x92, 0x89, 0x5e, 0x1c, 0x69, 0x9b, 0x27, 0x66, 0xdc, 0x43, 0xd7, 0x1b, + 0xc1, 0xb2, 0x6f, 0xea, 0xfc, 0x25, 0xc2, 0x33, 0xf0, 0xad, 0x89, 0x2d, 0xa2, 0xb8, 0x1a, 0x31, + 0xd9, 0x9a, 0xf1, 0xa1, 0x1c, 0xf9, 0x80, 0x7c, 0x1d, 0xde, 0xb1, 0xa4, 0x63, 0x7a, 0x2e, 0x90, + 0xbd, 0x4a, 0xc0, 0xe2, 0xf7, 0x7a, 0x45, 0x98, 0x09, 0xcd, 0x7f, 0x7c, 0x0a, 0xb8, 0x46, 0x5d, + 0x97, 0xec, 0xbb, 0x3c, 0x76, 0x7f, 0x12, 0x9a, 0x77, 0x9f, 0x37, 0x09, 0xcf, 0x60, 0x1d, 0xbf, + 0xd0, 0x68, 0xe5, 0xe9, 0x28, 0xa7, 0x4a, 0x3d, 0x85, 0x3f, 0x9f, 0x21, 0x06, 0xfa, 0xa1, 0xca, + 0x52, 0x2f, 0xf0, 0x99, 0x73, 0x99, 0x08, 0xda, 0xa4, 0x33, 0x5f, 0x05, 0xd2, 0x8e, 0x97, 0x7f, + 0xf8, 0x41, 0xbc, 0x33, 0xc0, 0xc8, 0x7d, 0xe3, 0x89, 0xcc, 0x25, 0xac, +}; +constexpr std::array Av1Frame1{ + 0x12, 0x00, 0x32, 0x12, 0x30, 0x03, 0x80, 0x80, 0xfd, 0xf8, 0x06, + 0xe4, 0x01, 0x04, 0x10, 0x40, 0x2c, 0x34, 0x82, 0x00, 0x97, 0x84, +}; +constexpr std::array Av1Frame2{ + 0x12, 0x00, 0x32, 0x12, 0x30, 0x04, 0x01, 0x05, 0x7d, 0xf8, 0x06, + 0xe4, 0x01, 0x04, 0x10, 0x40, 0x2c, 0x34, 0x82, 0x00, 0x93, 0xc0, +}; +constexpr std::array Av1Frame3{ + 0x12, 0x00, 0x32, 0x12, 0x30, 0x06, 0x02, 0x09, 0xfd, 0xf8, 0x06, + 0xe4, 0x01, 0x04, 0x10, 0x40, 0x2c, 0x34, 0x82, 0x00, 0x8f, 0x80, +}; +} // namespace + +TEST_CASE("NX2 video decodes AV1 on the CPU with libdav1d whatever NVDEC emulation is set to", + "[video_core][nx2_video_av1]") { + if (avcodec_find_decoder_by_name("libdav1d") == nullptr) { + SKIP("FFmpeg has no libdav1d (YUZU_USE_EXTERNAL_FFMPEG builds H.264, VP8, VP9 only)"); + } + const auto emulation = Settings::values.nvdec_emulation.GetValue(); + SCOPE_EXIT { + Settings::values.nvdec_emulation.SetValue(emulation); + }; + const std::array, 4> frames{Av1Frame0, Av1Frame1, Av1Frame2, Av1Frame3}; + for (const auto setting : {Settings::NvdecEmulation::Cpu, Settings::NvdecEmulation::Gpu}) { + Settings::values.nvdec_emulation.SetValue(setting); + Nx2VideoDecoder decoder{"video/av01"}; + REQUIRE(decoder.IsOpen()); + std::vector pictures; + for (u64 i = 0; i < frames.size(); ++i) { + decoder.Decode(frames[i], i * 33'333, pictures); + } + decoder.Drain(pictures); + REQUIRE(pictures.size() == frames.size()); + for (u64 i = 0; i < pictures.size(); ++i) { + const Nx2Picture& picture = pictures[i]; + CHECK(picture.pts == i * 33'333); + REQUIRE(picture.width == 64); + REQUIRE(picture.height == 48); + REQUIRE_FALSE(picture.frame->IsHardwareDecoded()); + REQUIRE(picture.frame->GetPixelFormat() == AV_PIX_FMT_YUV420P); + REQUIRE(picture.frame->GetFrame()->color_range != AVCOL_RANGE_JPEG); + + // The compositor reads NV12: luma rows as decoded, then Cb/Cr interleaved. + constexpr u32 pitch = 80; + constexpr u32 chroma_offset = pitch * 48; + std::vector nv12(chroma_offset + pitch * 24, 0xcd); + decoder.Render(picture, Nx2PictureFormat::Nv12, 64, 48, pitch, chroma_offset, nv12); + for (u32 y = 0; y < 48; ++y) { + const u8* luma = picture.frame->GetPlane(0) + y * picture.frame->GetStride(0); + REQUIRE(std::equal(luma, luma + 64, nv12.begin() + y * pitch)); + } + for (u32 y = 0; y < 24; ++y) { + const u8* u = picture.frame->GetPlane(1) + y * picture.frame->GetStride(1); + const u8* v = picture.frame->GetPlane(2) + y * picture.frame->GetStride(2); + for (u32 x = 0; x < 32; ++x) { + REQUIRE(nv12[chroma_offset + y * pitch + x * 2] == u[x]); + REQUIRE(nv12[chroma_offset + y * pitch + x * 2 + 1] == v[x]); + } + } + } + REQUIRE_FALSE(std::equal(pictures[0].frame->GetPlane(0), + pictures[0].frame->GetPlane(0) + 64, + pictures[0].frame->GetPlane(0) + 1)); + } +} diff --git a/src/video_core/host1x/ffmpeg.cpp b/src/video_core/host1x/ffmpeg.cpp index 1af8307adc..7355f797d4 100644 --- a/src/video_core/host1x/ffmpeg.cpp +++ b/src/video_core/host1x/ffmpeg.cpp @@ -186,6 +186,10 @@ Decoder::Decoder(Tegra::Host1x::NvdecCommon::VideoCodec codec) { } } #endif + if (!m_codec && av_codec == AV_CODEC_ID_AV1) { + // FFmpeg's own av1 decoder only decodes through a GPU hwaccel. + m_codec = avcodec_find_decoder_by_name("libdav1d"); + } if (!m_codec) { m_codec = avcodec_find_decoder(av_codec); } @@ -200,7 +204,6 @@ bool Decoder::SupportsDecodingOnDevice(AVPixelFormat* out_pix_fmt, AVHWDeviceTyp } if (config->methods & AV_CODEC_HW_CONFIG_METHOD_HW_DEVICE_CTX && config->device_type == type) { - LOG_INFO(HW_GPU, "Using {} GPU decoder", av_hwdevice_get_type_name(type)); *out_pix_fmt = config->pix_fmt; return true; } @@ -237,11 +240,13 @@ bool HardwareContext::InitializeForDecoder(DecoderContext& decoder_context, cons continue; } - if (!this->InitializeWithType(type)) { + // Creating a device is slow (CUDA, Vulkan): only create the ones the decoder can use. + if (!decoder.SupportsDecodingOnDevice(&hw_pix_fmt, type)) { continue; } - if (decoder.SupportsDecodingOnDevice(&hw_pix_fmt, type)) { + if (this->InitializeWithType(type)) { + LOG_INFO(HW_GPU, "Using {} GPU decoder", av_hwdevice_get_type_name(type)); decoder_context.InitializeHardwareDecoder(*this, hw_pix_fmt); return true; } diff --git a/src/video_core/host1x/nx2_video_decoder.cpp b/src/video_core/host1x/nx2_video_decoder.cpp index 86a800274e..336a9cd535 100644 --- a/src/video_core/host1x/nx2_video_decoder.cpp +++ b/src/video_core/host1x/nx2_video_decoder.cpp @@ -1,10 +1,13 @@ // SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project // SPDX-License-Identifier: GPL-3.0-or-later +#include #include +#include #include #include #include +#include #include "common/logging.h" #include "common/settings.h" @@ -18,6 +21,7 @@ extern "C" { #pragma GCC diagnostic ignored "-Wconversion" #endif +#include #include #if defined(__GNUC__) || defined(__clang__) @@ -46,6 +50,12 @@ std::optional CodecFromMime(std::string_view mime) { return std::nullopt; } +// dav1d defaults to a thread per host thread. A 1080p60 movie needs a few, and the emulator's +// own threads need the rest. +int Dav1dThreads() { + return static_cast(std::clamp(std::thread::hardware_concurrency() / 2, 2U, 8U)); +} + } // namespace bool IsH264Stream(std::span data) { @@ -129,6 +139,7 @@ struct Nx2VideoDecoder::Impl { Nx2VideoDecoder::Nx2VideoDecoder(std::string_view mime) : impl{std::make_unique()} { impl->name = std::string{mime}; + const auto start = std::chrono::steady_clock::now(); const auto codec = CodecFromMime(mime); if (!codec) { LOG_ERROR(HW_GPU, "nx2 video: no decoder for codec \"{}\"", mime); @@ -140,12 +151,33 @@ Nx2VideoDecoder::Nx2VideoDecoder(std::string_view mime) : impl{std::make_unique< return; } impl->context.emplace(*impl->decoder); - if (Settings::values.nvdec_emulation.GetValue() == Settings::NvdecEmulation::Gpu) { + AVCodecContext* const context = impl->context->GetCodecContext(); + const std::string_view decoder_name = impl->decoder->GetCodec()->name; + if (decoder_name == "libdav1d") { + context->thread_count = Dav1dThreads(); + } + const bool gpu = Settings::values.nvdec_emulation.GetValue() == Settings::NvdecEmulation::Gpu; + if (gpu) { impl->hardware.emplace(); impl->hardware->InitializeForDecoder(*impl->context, *impl->decoder); } impl->open = impl->context->OpenContext(*impl->decoder); - LOG_INFO(HW_GPU, "nx2 video: decoder for \"{}\" {}", mime, impl->open ? "open" : "failed to open"); + const auto ms = std::chrono::duration_cast( + std::chrono::steady_clock::now() - start) + .count(); + if (!impl->open) { + LOG_ERROR(HW_GPU, "nx2 video: {} failed to open for \"{}\"", decoder_name, mime); + } else if (context->hw_device_ctx) { + const auto* device = + reinterpret_cast(context->hw_device_ctx->data); + LOG_INFO(HW_GPU, "nx2 video: \"{}\" decodes on the GPU ({}) with {}, opened in {} ms", mime, + av_hwdevice_get_type_name(device->type), decoder_name, ms); + } else { + LOG_INFO(HW_GPU, + "nx2 video: \"{}\" decodes on the CPU with {}, {} threads ({}), opened in {} ms", + mime, decoder_name, context->thread_count, + gpu ? "no usable GPU decoder" : "NVDEC emulation is CPU", ms); + } } Nx2VideoDecoder::~Nx2VideoDecoder() = default; From 8c42de076198279abaa27c1e8fc3d5fe4d42c419 Mon Sep 17 00:00:00 2001 From: Zortos Date: Sun, 8 Nov 2026 21:31:10 +0100 Subject: [PATCH 083/105] shader_recompiler: remove static clip_distance_written data race in SPIR-V emit (#902) Assisted-by: Gemini (Section 6 clip_distance_written race and eviction comment) (cherry picked from commit 0dfaf624bafc85122dde2a9eada17aad6834bbb2) Co-authored-by: AManWithAPlan Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> (cherry picked from commit 3153d96b97bc501b558441a4242ce31508e3a5c1) --- .../backend/spirv/emit_spirv_context_get_set.cpp | 7 ------- .../backend/spirv/emit_spirv_special.cpp | 8 +++----- src/shader_recompiler/backend/spirv/spirv_emit_context.h | 2 -- src/video_core/renderer_vulkan/vk_pipeline_cache.cpp | 5 +++-- 4 files changed, 6 insertions(+), 16 deletions(-) diff --git a/src/shader_recompiler/backend/spirv/emit_spirv_context_get_set.cpp b/src/shader_recompiler/backend/spirv/emit_spirv_context_get_set.cpp index 5b2f2e604c..130f0b0fd9 100644 --- a/src/shader_recompiler/backend/spirv/emit_spirv_context_get_set.cpp +++ b/src/shader_recompiler/backend/spirv/emit_spirv_context_get_set.cpp @@ -430,13 +430,6 @@ void EmitSetAttribute(EmitContext& ctx, IR::Attribute attr, Id value, [[maybe_un value = ctx.OpBitcast(output->type, value); } - static constexpr IR::Attribute cd0 = IR::Attribute::ClipDistance0; - static constexpr IR::Attribute cd7 = IR::Attribute::ClipDistance7; - - if (attr >= cd0 && attr <= cd7) { - const u32 idx = (u32) attr - (u32) cd0; - clip_distance_written.set(idx); - } ctx.OpStore(output->pointer, value); } diff --git a/src/shader_recompiler/backend/spirv/emit_spirv_special.cpp b/src/shader_recompiler/backend/spirv/emit_spirv_special.cpp index 5767f4e1c0..a8c7a6bca0 100644 --- a/src/shader_recompiler/backend/spirv/emit_spirv_special.cpp +++ b/src/shader_recompiler/backend/spirv/emit_spirv_special.cpp @@ -170,11 +170,9 @@ void EmitPrologue(EmitContext& ctx) { if (Sirit::ValidId(ctx.clip_distances)) { for (u32 i = 0; i < ctx.profile.max_user_clip_distances; ++i) { - if (!clip_distance_written.test(i)) { - const Id idx = ctx.Const(i); - const Id element = ctx.OpAccessChain(ctx.output_f32, ctx.clip_distances, idx); - ctx.OpStore(element, ctx.Const(0.0f)); - } + const Id idx = ctx.Const(i); + const Id element = ctx.OpAccessChain(ctx.output_f32, ctx.clip_distances, idx); + ctx.OpStore(element, ctx.Const(0.0f)); } } } diff --git a/src/shader_recompiler/backend/spirv/spirv_emit_context.h b/src/shader_recompiler/backend/spirv/spirv_emit_context.h index 9a16182cd5..bd3a9d5a6e 100644 --- a/src/shader_recompiler/backend/spirv/spirv_emit_context.h +++ b/src/shader_recompiler/backend/spirv/spirv_emit_context.h @@ -20,8 +20,6 @@ namespace Shader::Backend::SPIRV { -static std::bitset<8> clip_distance_written; - using Sirit::Id; class VectorTypes { diff --git a/src/video_core/renderer_vulkan/vk_pipeline_cache.cpp b/src/video_core/renderer_vulkan/vk_pipeline_cache.cpp index d78c4ec710..0e42e8e25c 100644 --- a/src/video_core/renderer_vulkan/vk_pipeline_cache.cpp +++ b/src/video_core/renderer_vulkan/vk_pipeline_cache.cpp @@ -1676,7 +1676,7 @@ void PipelineCache::TickFrame() { void PipelineCache::EvictUnusedPipelines() { // Building a pipeline again on MoltenVK means a Metal compile mid-game, far dearer than the - // 2-3 ms below, so none are evicted there. + // rebuild cost below, so none are evicted there. if (device.IsMoltenVK()) { return; } @@ -1688,7 +1688,8 @@ void PipelineCache::EvictUnusedPipelines() { } // Each built pipeline holds ~0.45 MB of driver memory and none was ever destroyed, so it grew // with every place a session went (title 1: 1,500 on its title screen alone). One unused for - // this long is destroyed; it takes 2-3 ms to build again from the driver's pipeline cache. + // this long is destroyed; rebuilding it later requires a full GPU-thread shader translation + // (~8.5 ms each) plus a driver pipeline cache hit. constexpr u64 EvictAfterFrames = 9000; if (frame_count <= EvictAfterFrames) { return; From ec32c7a8e1b21a1ca82f56237745b5ecaaf235f4 Mon Sep 17 00:00:00 2001 From: atk3001 <59624138+atk3001@users.noreply.github.com> Date: Sun, 8 Nov 2026 22:26:26 +0100 Subject: [PATCH 084/105] texture_cache: don't prepare the null image on a texture memo hit (Ocarina of Time) (#910) ResolveMemoizedTextures prepared every memoized view, including NULL_IMAGE_VIEW_ID, which VisitImageView never prepares. The null image has an invalid format and the default CpuModified flag, so a memo hit on a draw with a null texture tried to upload it and ended in UNREACHABLE() in BitsPerBlock. Ocarina of Time crashed this way loading a save. Fixes #904 Co-authored-by: Claude Opus 5.5 (cherry picked from commit c2e293ec88d931ecd1c82299289f582c23136ab7) --- src/video_core/texture_cache/texture_cache.h | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/video_core/texture_cache/texture_cache.h b/src/video_core/texture_cache/texture_cache.h index 1649fc68cc..77a90a348a 100644 --- a/src/video_core/texture_cache/texture_cache.h +++ b/src/video_core/texture_cache/texture_cache.h @@ -692,7 +692,10 @@ bool TextureCache

::ResolveMemoizedTextures(SampledTextureMemo& memo, const SampledTextureMemo::Entry& entry = memo.entries[i]; views[i] = ImageViewInOut{.index = entry.view_index, .blacklist = false, .id = entry.view}; samplers[i] = entry.sampler; - PrepareImageView(entry.view, false, false); + // As in VisitImageView: the null image has nothing to upload and no valid format. + if (entry.view != NULL_IMAGE_VIEW_ID) { + PrepareImageView(entry.view, false, false); + } } if (has_deleted_images) { // Preparing an image deleted others: the full path resolves the views again. From 6a049eb9cdffeb254c479e25e7ef485046389a33 Mon Sep 17 00:00:00 2001 From: Zortos Date: Sun, 8 Nov 2026 23:29:10 +0100 Subject: [PATCH 085/105] common: make Log2Ceil(0) the smallest bucket instead of wrapping (#918) Log2Floor(0) wraps to 0xFFFFFFFF, so Log2Ceil(0) shifted by that much and returned a huge index. A zero-size staging request (the null image's upload in the crash #910 fixed) then read StagingBufferPool's 64-entry cache far out of range. Log2Ceil now returns 0 for 0. Refs #904, #910 (cherry picked from commit 281477eac7e17a071c80fee15489f0dccc2ad991) --- src/common/bit_util.h | 5 +++++ src/tests/CMakeLists.txt | 1 + src/tests/common/bit_util.cpp | 25 +++++++++++++++++++++++++ 3 files changed, 31 insertions(+) create mode 100644 src/tests/common/bit_util.cpp diff --git a/src/common/bit_util.h b/src/common/bit_util.h index ce916e6162..a85f3a94b7 100644 --- a/src/common/bit_util.h +++ b/src/common/bit_util.h @@ -34,9 +34,14 @@ template return T(MostSignificantBit(value)); } +/// The exponent of the smallest power of two that is at least value. 0 for 0: a zero-size +/// request fits the smallest bucket, where Log2Floor(0) would wrap and index far out of range. template requires std::is_integral_v [[nodiscard]] constexpr T Log2Ceil(const T value) { + if (value == T(0)) { + return T(0); + } const T log2_f = Log2Floor(value); return T(log2_f + T((value ^ (T(1ULL) << log2_f)) != T(0ULL))); } diff --git a/src/tests/CMakeLists.txt b/src/tests/CMakeLists.txt index 18014b004a..aeb49f2a70 100644 --- a/src/tests/CMakeLists.txt +++ b/src/tests/CMakeLists.txt @@ -120,6 +120,7 @@ add_executable(tests video_core/spirv_nan_safe_min_max.cpp video_core/virtual_range_cache.cpp common/bit_field.cpp + common/bit_util.cpp common/cityhash.cpp common/container_hash.cpp common/core_pinning.cpp diff --git a/src/tests/common/bit_util.cpp b/src/tests/common/bit_util.cpp new file mode 100644 index 0000000000..21e6d36d8c --- /dev/null +++ b/src/tests/common/bit_util.cpp @@ -0,0 +1,25 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later + +#include + +#include "common/bit_util.h" +#include "common/common_types.h" + +TEST_CASE("Log2Ceil rounds up to the next power of two", "[common][bit_util]") { + REQUIRE(Common::Log2Ceil(1) == 0); + REQUIRE(Common::Log2Ceil(2) == 1); + REQUIRE(Common::Log2Ceil(3) == 2); + REQUIRE(Common::Log2Ceil(4096) == 12); + REQUIRE(Common::Log2Ceil(4097) == 13); + REQUIRE(Common::Log2Ceil(0x80000000U) == 31); + REQUIRE(Common::Log2Ceil(u64{1} << 63) == 63); +} + +TEST_CASE("Log2Ceil of zero is the smallest bucket", "[common][bit_util]") { + // A zero-size staging request (the null image's upload) indexed the 64-entry staging + // cache at 0xFFFFFFFF. + STATIC_REQUIRE(Common::Log2Ceil(0) == 0); + STATIC_REQUIRE(Common::Log2Ceil(0) == 0); + STATIC_REQUIRE(Common::Log2Ceil(0) == 0); +} From 8014ba8449f6ff6b8e752fc539a4704b4ad9a407 Mon Sep 17 00:00:00 2001 From: Zortos Date: Mon, 9 Nov 2026 00:41:10 +0100 Subject: [PATCH 086/105] video_core: give GetSubmappedDeviceRange's min one type (macOS build) (#932) size is std::size_t and address_space_size u64. They are the same type on Linux and Windows but not on macOS (unsigned long and unsigned long long), where the min from #858 did not compile. The result is at most size, so it fits the std::size_t parameter it is passed as. Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> (cherry picked from commit cbb512ee7a9de9d9dbe2f865ec2e468888503214) --- src/video_core/memory_manager.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/video_core/memory_manager.cpp b/src/video_core/memory_manager.cpp index 560482dd74..90f4ef116f 100644 --- a/src/video_core/memory_manager.cpp +++ b/src/video_core/memory_manager.cpp @@ -982,8 +982,8 @@ boost::container::small_vector, 32> MemoryManager::GetSubmappedDeviceRange(GPUVAddr gpu_addr, std::size_t size) const { boost::container::small_vector, 32> result{}; if (IsWithinGPUAddressRange(gpu_addr)) { - GetSubmappedRangeImpl(gpu_addr, (std::min)(size, address_space_size - gpu_addr), - result); + GetSubmappedRangeImpl( + gpu_addr, (std::min)(size, address_space_size - gpu_addr), result); } return result; } From d9d8e1178951752a48a5bfad89f898e9ec9c80bf Mon Sep 17 00:00:00 2001 From: Zortos Date: Mon, 9 Nov 2026 08:41:24 +0100 Subject: [PATCH 087/105] common: reserve the last page table level from the range's last byte (#957) MultiLevelPageTable::ReserveRange took (start + size) >> first_level_shift as the last first-level index, inclusive. A range ending exactly at the top of the address space (which MemoryManager::PageTableOp accepts) indexes one past first_level_map: it reads past the vector and, when that word is null, AllocateLevel writes past it. ASan reports it in the maxwell_fast_method test that maps the last GPU page. Take the index of the range's last byte instead; an empty range reserves nothing. Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> (cherry picked from commit cee0c2dbc0a8e7c568712b170b4bdb63b7efb23b) --- src/common/multi_level_page_table.inc | 5 ++++- src/tests/CMakeLists.txt | 1 + src/tests/common/multi_level_page_table.cpp | 16 ++++++++++++++++ 3 files changed, 21 insertions(+), 1 deletion(-) create mode 100644 src/tests/common/multi_level_page_table.cpp diff --git a/src/common/multi_level_page_table.inc b/src/common/multi_level_page_table.inc index 79125e8e0b..30d9ea169d 100644 --- a/src/common/multi_level_page_table.inc +++ b/src/common/multi_level_page_table.inc @@ -51,8 +51,11 @@ MultiLevelPageTable::~MultiLevelPageTable() noexcept { template void MultiLevelPageTable::ReserveRange(u64 start, std::size_t size) { + if (size == 0) { + return; + } const u64 new_start = start >> first_level_shift; - const u64 new_end = (start + size) >> first_level_shift; + const u64 new_end = (start + size - 1) >> first_level_shift; for (u64 i = new_start; i <= new_end; i++) if (!first_level_map[i]) AllocateLevel(i); diff --git a/src/tests/CMakeLists.txt b/src/tests/CMakeLists.txt index aeb49f2a70..6254aabb4b 100644 --- a/src/tests/CMakeLists.txt +++ b/src/tests/CMakeLists.txt @@ -132,6 +132,7 @@ add_executable(tests common/host_memory_failure.cpp common/host_memory_test_access.h core/host_memory_failure.cpp + common/multi_level_page_table.cpp common/page_table.cpp common/param_package.cpp common/range_map.cpp diff --git a/src/tests/common/multi_level_page_table.cpp b/src/tests/common/multi_level_page_table.cpp new file mode 100644 index 0000000000..7397f2c98b --- /dev/null +++ b/src/tests/common/multi_level_page_table.cpp @@ -0,0 +1,16 @@ +// SPDX-License-Identifier: GPL-3.0-or-later +#include +#include "common/multi_level_page_table.h" + +TEST_CASE("MultiLevelPageTable reserves ranges that end at the top of the address space", + "[multi_level_page_table]") { + Common::MultiLevelPageTable table{32, 4, 12}; + constexpr u64 last_page = (1ULL << 20) - 1; + table.ReserveRange((1ULL << 32) - 0x1000, 0x1000); + table[last_page] = 7; + REQUIRE(table[last_page] == 7); + table.ReserveRange(0, 1ULL << 32); + table[0] = 3; + REQUIRE(table[0] == 3); + REQUIRE(table[last_page] == 7); +} From dc9f79d7f22e888c497ff20af853656b1c25414b Mon Sep 17 00:00:00 2001 From: Zortos Date: Mon, 9 Nov 2026 08:41:52 +0100 Subject: [PATCH 088/105] file_sys: harden PFS0, RomFS, NCA and XCI parsers against malformed files (#947) * vfs: return 0 from VectorVfsFile::Read for an offset past the end VectorVfsFile::Read underflowed the remaining size for an offset past the end and copied out of bounds. Ported from failaip12's Eglantine PR #462; only the vfs_vector.cpp hunk is taken, the commit's audio, scratch buffer and zbic hunks are unrelated. (cherry picked from commit 6375dc49bf6a453192d5dd2948053557c831cc6f) * tests: cover VectorVfsFile reads past the end Check that VectorVfsFile::Read returns 0 for an offset at or past the end. Ported from failaip12's Eglantine PR #462; only the vfs_vector test is taken, the commit's usage tracker test belongs to an unrelated change. (cherry picked from commit 9932bd647286591a7ebdc3e9e7366a42893493b0) * common: let ASan builds link zbic's bundled zstd zstd.c declares the ASan poisoning functions itself, which gives them C++ linkage inside zbic_compression.cpp and breaks ASan links (the tests target fails with undefined __asan_poison_memory_region). Declaring them first through the sanitizer header keeps C linkage. Ported from failaip12's Eglantine PR #462; only the zbic_compression.cpp hunk is taken. (cherry picked from commit 6375dc49bf6a453192d5dd2948053557c831cc6f) * vfs: size ReadBytes buffers by the bytes the file holds ReadBytes allocated the requested size before reading, so a size taken from a file header (a PFS0 or RomFS table) could ask for terabytes. Clamp the buffer to what lies between the offset and the end of the file; the returned bytes are the same as before. * file_sys: bound PFS0 tables and entry names Check that the entry and string tables fit in the file before reading them, that each name offset lies inside the string table, and read the name with strnlen up to the table end instead of strlen. Iterate entries with a u32, since a u16 index never reached a count above 65535. * file_sys: walk RomFS tables without recursion and reject loops ExtractRomFS recursed once per directory level and followed sibling and child links with no visited check, so a link back to an earlier entry looped until memory ran out and a long chain overflowed the stack. Walk the tree with an explicit stack, mark each entry offset when it is read, cap the depth at what a 0x300-character Horizon path allows, and return nullptr for a table that runs past the file, an entry past its table, a revisited entry or a named root. * fssystem: check NCA FS header ranges and layer parameters NCA FS header offsets and sizes are signed and untrusted, but the driver compared plain sums against the storage size, so a negative size or an overflowing sum passed and later sized an allocation or an OffsetVfsFile. Check every range with an overflow-safe helper, and check the hash regions before they size the SHA-256/SHA3 buffer. Also reject what Atmosphere asserts or never meets in signed content: SHA-256 blocks smaller than a hash, integrity block orders a shift or a u32 cannot hold, fewer than three integrity layers, layer counts outside the hash target table, FS headers or NCA headers the file is too short to hold, and misaligned patch tables. IntegrityVerificationStorage::Read now returns 0 for an out-of-range read instead of asserting, and zeroes only the requested bytes past the data rather than up to the block end, which wrote past the caller's buffer on short reads. CreateAesXtsStorage returned the storage it had just moved into the alignment wrapper, so every AES-XTS section opened as a null storage. * file_sys: reject XCI images without a secure partition or HFS0 An HFS0 offset past the end of the image underflowed the root partition size, and a root partition without a secure entry handed NSP a null file. * tests: cover malformed PFS0, RomFS, NCA and XCI inputs Each case builds its input from scratch, with no game data: the shapes fuzzing found (name offsets past the string table, tables larger than the file, RomFS link loops and deep chains, SHA-256 and integrity layer parameters, negative patch table sizes, XCI images without a secure partition, short integrity reads past the data) and a valid input of the same shape that must still parse. * fuzz: add libFuzzer harnesses for the game-file parsers LICHII_FUZZERS (off by default, clang only) instruments the build with ASan and UBSan and builds three harnesses: raw PFS0/RomFS, a single NCA (alone, as an update with a missing base, and wrapped in an NSP), and a whole NSP or XCI. The NCA harnesses use made-up key area keys in a scratch keys directory, so plaintext NCAs reach the FS header and storage parsers without real keys. docs/Fuzzing.md covers building and running them. Based on the harnesses EGLA-84 used to fuzz the .nx loader (#866). * fssystem: bound buffered NCA tables by the bytes the NCAs store The indirect and AES-CTR-EX tables and the SHA-256/SHA3 hash layers are read into memory whole. Their ranges were only checked against the section storage, but sparse and indirect layers give that storage a virtual size taken from header fields, so a 1.5 KiB NCA could ask for a 372 GiB buffer. The patch tables are stored in the NCA itself and the hashes in it or the NCA it patches, so reject anything larger than those bodies. The hash buffer also caches two hash blocks, and 2 * hash_block_size overflowed s32 for a 1 GiB block. Compute it in size_t, require the hash layer to fit in one block as HierarchicalSha256Storage assumes, and reject blocks larger than the stored bytes rounded up to a power of two. * fssystem: drop the unused hash cache from SHA-256/SHA3 buffers The hash storages only use the first hash_region.size bytes of the buffer, so the two cache blocks were never read. Allocate the hash layer alone. That removes the reason to bound the block size by the file size, which rejected valid small NCAs whose hash block is larger than the whole file. (cherry picked from commit 59e3d227a10385ba02246aa74100096efd12bfa1) --- CMakeLists.txt | 12 + docs/Fuzzing.md | 24 + docs/Options.md | 2 + docs/README.md | 1 + src/CMakeLists.txt | 4 + src/common/zbic_compression.cpp | 9 + src/core/file_sys/card_image.cpp | 14 +- ...rchical_integrity_verification_storage.cpp | 6 +- ...ssystem_integrity_verification_storage.cpp | 25 +- .../fssystem_nca_file_system_driver.cpp | 152 ++++-- .../file_sys/fssystem/fssystem_nca_header.h | 25 +- .../file_sys/fssystem/fssystem_nca_reader.cpp | 6 +- src/core/file_sys/partition_filesystem.cpp | 35 +- src/core/file_sys/romfs.cpp | 123 +++-- src/core/file_sys/vfs/vfs.cpp | 8 +- src/core/file_sys/vfs/vfs_vector.cpp | 3 + src/fuzz/CMakeLists.txt | 8 + src/fuzz/fuzz_common.h | 110 ++++ src/fuzz/fuzz_nca.cpp | 54 ++ src/fuzz/fuzz_package.cpp | 40 ++ src/fuzz/fuzz_partition_romfs.cpp | 21 + src/fuzz/stubs.cpp | 19 + src/tests/CMakeLists.txt | 2 + src/tests/core/file_sys_parsers.cpp | 493 ++++++++++++++++++ src/tests/core/vfs_vector.cpp | 25 + 25 files changed, 1082 insertions(+), 139 deletions(-) create mode 100644 docs/Fuzzing.md create mode 100644 src/fuzz/CMakeLists.txt create mode 100644 src/fuzz/fuzz_common.h create mode 100644 src/fuzz/fuzz_nca.cpp create mode 100644 src/fuzz/fuzz_package.cpp create mode 100644 src/fuzz/fuzz_partition_romfs.cpp create mode 100644 src/fuzz/stubs.cpp create mode 100644 src/tests/core/file_sys_parsers.cpp create mode 100644 src/tests/core/vfs_vector.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index b254665981..8472bc4edb 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -84,6 +84,18 @@ cmake_dependent_option(YUZU_USE_BUNDLED_QT "Download bundled Qt binaries" "${MSV option(ENABLE_DEBUG_TOOLS "Enable debugging tools (maxwell disassembler, SPIRV translator, etc)" OFF) option(ENABLE_WERROR "Enable -Werror diagnostics" ON) +option(LICHII_FUZZERS "Build libFuzzer harnesses for the game-file parsers (clang only; instruments everything with ASan and UBSan)" OFF) +if (LICHII_FUZZERS) + if (NOT CMAKE_CXX_COMPILER_ID MATCHES "Clang" OR MSVC) + message(FATAL_ERROR "LICHII_FUZZERS needs clang with libFuzzer") + endif() + add_compile_options( + $<$:-fsanitize=address,undefined,fuzzer-no-link> + $<$:-fno-sanitize-recover=undefined> + $<$:-fno-omit-frame-pointer>) + add_link_options(-fsanitize=address,undefined) +endif() + # Handle specific -Werror downgrades safely across platforms if(ENABLE_WERROR) if(CMAKE_COMPILER_IS_GNUCC OR CMAKE_COMPILER_IS_GNUCXX OR CMAKE_CXX_COMPILER_ID MATCHES "Clang") diff --git a/docs/Fuzzing.md b/docs/Fuzzing.md new file mode 100644 index 0000000000..4d5e4fe4fe --- /dev/null +++ b/docs/Fuzzing.md @@ -0,0 +1,24 @@ +# Fuzzing the game-file parsers + +`-DLICHII_FUZZERS=ON` builds three libFuzzer harnesses in `src/fuzz/` and compiles the whole build with AddressSanitizer and UndefinedBehaviorSanitizer. It needs clang with its compiler-rt runtimes (`libclang-rt--dev` on Ubuntu). Use a separate build directory, since every object is instrumented. + +| Harness | Input | +|---|---| +| `fuzz_partition_romfs` | Raw PFS0/HFS0 partition, then the same bytes as a raw RomFS image | +| `fuzz_nca` | One NCA, on its own, as an update NCA with a missing base, and inside an NSP | +| `fuzz_package` | A whole NSP, then the same bytes as an XCI | + +NCA headers are normally encrypted. The NCA reader also accepts a plaintext header (magic `NCA3` at 0x200) and sections with encryption type None, so fuzz NCA inputs in that form. `fuzz_nca` and `fuzz_package` point the key manager at a scratch keys directory with made-up key area keys, so plaintext NCAs pass the key check. They never read or write your own keys. + +```sh +cmake -S . -B build-fuzz -G Ninja -DCMAKE_BUILD_TYPE=RelWithDebInfo \ + -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++ \ + -DLICHII_FUZZERS=ON -DENABLE_WERROR=OFF -DENABLE_QT=OFF -DYUZU_CMD=OFF -DYUZU_ROOM=OFF +cmake --build build-fuzz --target fuzz_partition_romfs fuzz_nca fuzz_package +mkdir -p corpus/nca && ASAN_OPTIONS=detect_leaks=0 \ + build-fuzz/bin/fuzz_nca -rss_limit_mb=3072 -malloc_limit_mb=3072 -timeout=25 corpus/nca seeds/ +``` + +Seed the corpus with small files you made yourself, never with game dumps. A crash writes `crash-` to the working directory. To replay it, pass the file to the harness: `build-fuzz/bin/fuzz_nca crash-`. Turn each fixed crash into a case in `src/tests/core/file_sys_parsers.cpp` that builds the input from scratch. + +With `-DYUZU_TESTS=ON`, the same build directory also builds `tests` under the sanitizers. diff --git a/docs/Options.md b/docs/Options.md index 4363e10324..ac53cde060 100644 --- a/docs/Options.md +++ b/docs/Options.md @@ -48,6 +48,8 @@ These options control dependencies. - Only available on Linux - `ENABLE_DEBUG_TOOLS` (OFF) Enables debugging and development tools, see [tools](../tools/README.md). - `ENABLE_WERROR` (ON) Enables warnings as errors (-Werror). +- `LICHII_FUZZERS` (OFF) Builds the libFuzzer harnesses for the game-file parsers and instruments the whole build with ASan and UBSan, see [Fuzzing](./Fuzzing.md). + - Clang only ### Flavors diff --git a/docs/README.md b/docs/README.md index 3bc196234a..1a6a964ff1 100644 --- a/docs/README.md +++ b/docs/README.md @@ -13,6 +13,7 @@ This contains documentation created by developers, build instructions, guideline - **[Dependencies](./Deps.md)** - **[Debug Guidelines](./Debug.md)** - **[RenderDoc](./Debug.md#renderdoc)** +- **[Fuzzing the game-file parsers](./Fuzzing.md)** - **[CPM - CMake Package Manager](./CPMUtil.md)** - **[Platform-Specific Caveats](./Caveats.md)** - **[The NVIDIA SM86 (Maxwell) GPU](./NvidiaGpu.md)** diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 384a34c20a..92ea5b16c7 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -292,6 +292,10 @@ if (YUZU_TESTS) add_subdirectory(tests) endif() +if (LICHII_FUZZERS) + add_subdirectory(fuzz) +endif() + if (YUZU_CMD) add_subdirectory(yuzu_cmd) set_target_properties(yuzu-cmd PROPERTIES OUTPUT_NAME "eden-cli") diff --git a/src/common/zbic_compression.cpp b/src/common/zbic_compression.cpp index fed0c5be9c..28ca5ba5f8 100644 --- a/src/common/zbic_compression.cpp +++ b/src/common/zbic_compression.cpp @@ -18,6 +18,15 @@ #endif #include "zstd.h" +// zstd.c declares the ASan poisoning functions itself, which gives them C++ linkage here and +// breaks ASan links. Declaring them first through the sanitizer header keeps C linkage. +#if defined(__SANITIZE_ADDRESS__) +#include +#elif defined(__has_feature) +#if __has_feature(address_sanitizer) +#include +#endif +#endif #define g_ZSTD_threading_useless_symbol g_ZSTD_zbic_threading_useless_symbol #include "zstd.c" #undef g_ZSTD_threading_useless_symbol diff --git a/src/core/file_sys/card_image.cpp b/src/core/file_sys/card_image.cpp index 593c561689..515073c4a6 100644 --- a/src/core/file_sys/card_image.cpp +++ b/src/core/file_sys/card_image.cpp @@ -40,6 +40,11 @@ XCI::XCI(VirtualFile file_, u64 program_id, size_t program_index) return; } + if (header.hfs_offset > file->GetSize()) { + status = Loader::ResultStatus::ErrorBadXCIHeader; + return; + } + PartitionFilesystem main_hfs(std::make_shared( file, file->GetSize() - header.hfs_offset, header.hfs_offset)); @@ -58,9 +63,12 @@ XCI::XCI(VirtualFile file_, u64 program_id, size_t program_index) partitions_raw[static_cast(partition)] = std::move(raw); } - secure_partition = std::make_shared( - main_hfs.GetFile(partition_names[static_cast(XCIPartition::Secure)]), - program_id, program_index); + auto secure_raw = main_hfs.GetFile(partition_names[static_cast(XCIPartition::Secure)]); + if (secure_raw == nullptr) { + status = Loader::ResultStatus::ErrorXCIMissingPartition; + return; + } + secure_partition = std::make_shared(std::move(secure_raw), program_id, program_index); ncas = secure_partition->GetNCAsCollapsed(); program = diff --git a/src/core/file_sys/fssystem/fssystem_hierarchical_integrity_verification_storage.cpp b/src/core/file_sys/fssystem/fssystem_hierarchical_integrity_verification_storage.cpp index f4ec53ad40..4db25924de 100644 --- a/src/core/file_sys/fssystem/fssystem_hierarchical_integrity_verification_storage.cpp +++ b/src/core/file_sys/fssystem/fssystem_hierarchical_integrity_verification_storage.cpp @@ -20,8 +20,10 @@ Result HierarchicalIntegrityVerificationStorage::Initialize( const HierarchicalIntegrityVerificationInformation& info, HierarchicalStorageInformation storage, int max_data_cache_entries, int max_hash_cache_entries, s8 buffer_level) { - // Validate preconditions. - ASSERT(IntegrityMinLayerCount <= info.max_layers && info.max_layers <= IntegrityMaxLayerCount); + // Validate the layer count. With fewer than three layers there is no level between the master + // hash and the data for the loops below to set up. + R_UNLESS(IntegrityMinLayerCount < info.max_layers && info.max_layers <= IntegrityMaxLayerCount, + ResultInvalidNcaHierarchicalIntegrityVerificationLayerCount); // Set member variables. m_max_layers = info.max_layers; diff --git a/src/core/file_sys/fssystem/fssystem_integrity_verification_storage.cpp b/src/core/file_sys/fssystem/fssystem_integrity_verification_storage.cpp index 15bb609110..4d5636e4dc 100644 --- a/src/core/file_sys/fssystem/fssystem_integrity_verification_storage.cpp +++ b/src/core/file_sys/fssystem/fssystem_integrity_verification_storage.cpp @@ -63,24 +63,23 @@ size_t IntegrityVerificationStorage::Read(u8* buffer, size_t size, size_t offset ASSERT(buffer != nullptr); // Validate the offset. - s64 data_size = m_data_storage->GetSize(); - ASSERT(offset <= static_cast(data_size)); + const size_t data_size = m_data_storage->GetSize(); + if (offset > data_size) { + return 0; + } // Validate the access range. - ASSERT(R_SUCCEEDED(IStorage::CheckAccessRange( - offset, size, Common::AlignUp(data_size, static_cast(m_verification_block_size))))); + if (R_FAILED(IStorage::CheckAccessRange( + offset, size, Common::AlignUp(data_size, static_cast(m_verification_block_size))))) { + return 0; + } // Determine the read extents. size_t read_size = size; - if (static_cast(offset + read_size) > data_size) { - // Determine the padding sizes. - s64 padding_offset = data_size - offset; - size_t padding_size = static_cast( - m_verification_block_size - (padding_offset & (m_verification_block_size - 1))); - ASSERT(static_cast(padding_size) < m_verification_block_size); - - // Clear the padding. - std::memset(static_cast(buffer) + padding_offset, 0, padding_size); + if (offset + read_size > data_size) { + // Zero the requested bytes past the end of the data. + const size_t padding_offset = data_size - offset; + std::memset(buffer + padding_offset, 0, size - padding_offset); // Set the new in-bounds size. read_size = static_cast(data_size - offset); diff --git a/src/core/file_sys/fssystem/fssystem_nca_file_system_driver.cpp b/src/core/file_sys/fssystem/fssystem_nca_file_system_driver.cpp index d496d58cec..b9afcfd925 100644 --- a/src/core/file_sys/fssystem/fssystem_nca_file_system_driver.cpp +++ b/src/core/file_sys/fssystem/fssystem_nca_file_system_driver.cpp @@ -4,6 +4,8 @@ // SPDX-FileCopyrightText: Copyright 2023 yuzu Emulator Project // SPDX-License-Identifier: GPL-2.0-or-later +#include + #include "core/file_sys/fssystem/fssystem_aes_ctr_counter_extended_storage.h" #include "core/file_sys/fssystem/fssystem_aes_ctr_storage.h" #include "core/file_sys/fssystem/fssystem_aes_xts_storage.h" @@ -31,6 +33,18 @@ constexpr inline s32 IntegrityHashCacheCount = 8; constexpr inline s32 IntegrityDataCacheCountForMeta = 16; constexpr inline s32 IntegrityHashCacheCountForMeta = 2; +// Offsets and sizes in NCA headers are untrusted s64 values, so ranges are checked without +// letting the sum overflow. +constexpr bool IsRangeWithin(s64 offset, s64 size, s64 end) { + return offset >= 0 && size >= 0 && offset <= end && size <= end - offset; +} + +// Sparse and indirect layers give a section a virtual size taken from header fields. Tables that +// are read into memory whole are also checked against the bytes the NCAs actually store. +s64 GetBodySize(const std::shared_ptr& reader) { + return reader != nullptr ? static_cast(reader->GetSharedBodyStorage()->GetSize()) : 0; +} + class SharedNcaBodyStorage : public IReadOnlyStorage { YUZU_NON_COPYABLE(SharedNcaBodyStorage); YUZU_NON_MOVEABLE(SharedNcaBodyStorage); @@ -160,10 +174,12 @@ Result NcaFileSystemDriver::OpenStorageImpl(VirtualFile* out, NcaFsHeaderReader* if (patch_info.HasAesCtrExTable()) { // Check the encryption type. - ASSERT(out_header_reader->GetEncryptionType() == NcaFsHeader::EncryptionType::None || - out_header_reader->GetEncryptionType() == NcaFsHeader::EncryptionType::AesCtrEx || - out_header_reader->GetEncryptionType() == - NcaFsHeader::EncryptionType::AesCtrExSkipLayerHash); + R_UNLESS(out_header_reader->GetEncryptionType() == NcaFsHeader::EncryptionType::None || + out_header_reader->GetEncryptionType() == + NcaFsHeader::EncryptionType::AesCtrEx || + out_header_reader->GetEncryptionType() == + NcaFsHeader::EncryptionType::AesCtrExSkipLayerHash, + ResultInvalidNcaFsHeaderEncryptionType); // Create the ex meta storage. VirtualFile aes_ctr_ex_storage_meta_storage = patch_meta_aes_ctr_ex_meta_storage; @@ -414,7 +430,7 @@ Result NcaFileSystemDriver::CreateBodySubStorage(VirtualFile* out, s64 offset, s s64 body_size = body_storage->GetSize(); // Check that we're within range. - R_UNLESS(offset + size <= body_size, ResultNcaBaseStorageOutOfRangeB); + R_UNLESS(IsRangeWithin(offset, size, body_size), ResultNcaBaseStorageOutOfRangeB); // Create substorage. auto body_substorage = std::make_shared(std::move(body_storage), size, offset); @@ -487,7 +503,7 @@ Result NcaFileSystemDriver::CreateAesXtsStorage(VirtualFile* out, VirtualFile ba R_UNLESS(aligned_storage != nullptr, ResultAllocationMemoryFailedAllocateShared); // Set the out storage. - *out = std::move(xts_storage); + *out = std::move(aligned_storage); R_SUCCEED(); } @@ -505,7 +521,9 @@ Result NcaFileSystemDriver::CreateSparseStorageMetaStorage(VirtualFile* out, // Get the meta extents. const auto meta_offset = sparse_info.bucket.offset; const auto meta_size = sparse_info.bucket.size; - R_UNLESS(meta_offset + meta_size - offset <= base_size, ResultNcaBaseStorageOutOfRangeB); + R_UNLESS(offset >= 0 && meta_offset >= 0 && meta_size >= 0 && meta_size <= base_size && + meta_offset - offset <= base_size - meta_size, + ResultNcaBaseStorageOutOfRangeB); // Create the encrypted storage. auto enc_storage = @@ -656,13 +674,15 @@ Result NcaFileSystemDriver::CreateSparseStorageMetaStorageWithVerification( // Get the meta extents. const auto meta_offset = sparse_info.bucket.offset; const auto meta_size = sparse_info.bucket.size; - R_UNLESS(meta_offset + meta_size - offset <= base_size, ResultNcaBaseStorageOutOfRangeB); + R_UNLESS(offset >= 0 && meta_offset >= 0 && meta_size >= 0 && meta_size <= base_size && + meta_offset - offset <= base_size - meta_size, + ResultNcaBaseStorageOutOfRangeB); // Get the meta data hash data extents. const s64 meta_data_hash_data_offset = meta_data_hash_data_info.offset; const s64 meta_data_hash_data_size = Common::AlignUp(meta_data_hash_data_info.size, NcaHeader::CtrBlockSize); - R_UNLESS(meta_data_hash_data_offset + meta_data_hash_data_size <= base_size, + R_UNLESS(IsRangeWithin(meta_data_hash_data_offset, meta_data_hash_data_size, base_size), ResultNcaBaseStorageOutOfRangeB); // Check that the meta is before the hash data. @@ -817,7 +837,8 @@ Result NcaFileSystemDriver::CreateAesCtrExStorageMetaStorage( // Validate patch info extents. R_UNLESS(patch_info.indirect_size > 0, ResultInvalidNcaPatchInfoIndirectSize); R_UNLESS(patch_info.aes_ctr_ex_size > 0, ResultInvalidNcaPatchInfoAesCtrExSize); - R_UNLESS(patch_info.indirect_size + patch_info.indirect_offset <= patch_info.aes_ctr_ex_offset, + R_UNLESS(IsRangeWithin(patch_info.indirect_offset, patch_info.indirect_size, + patch_info.aes_ctr_ex_offset), ResultInvalidNcaPatchInfoAesCtrExOffset); // Get the base storage size. @@ -827,7 +848,9 @@ Result NcaFileSystemDriver::CreateAesCtrExStorageMetaStorage( const s64 meta_offset = patch_info.aes_ctr_ex_offset; const s64 meta_size = Common::AlignUp(static_cast(patch_info.aes_ctr_ex_size), NcaHeader::XtsBlockSize); - R_UNLESS(meta_offset + meta_size <= base_size, ResultNcaBaseStorageOutOfRangeB); + R_UNLESS(IsRangeWithin(meta_offset, meta_size, base_size), ResultNcaBaseStorageOutOfRangeB); + + R_UNLESS(meta_size <= GetBodySize(m_reader), ResultNcaBaseStorageOutOfRangeB); // Create the encrypted storage. auto enc_storage = @@ -963,9 +986,11 @@ Result NcaFileSystemDriver::CreateIndirectStorageMetaStorage(VirtualFile* out, s64 base_size = base_storage->GetSize(); // Check that we're within range. - R_UNLESS(patch_info.indirect_offset + patch_info.indirect_size <= base_size, + R_UNLESS(IsRangeWithin(patch_info.indirect_offset, patch_info.indirect_size, base_size), ResultNcaBaseStorageOutOfRangeE); + R_UNLESS(patch_info.indirect_size <= GetBodySize(m_reader), ResultNcaBaseStorageOutOfRangeE); + // Create the meta storage. auto meta_storage = std::make_shared(base_storage, patch_info.indirect_size, patch_info.indirect_offset); @@ -1005,7 +1030,8 @@ Result NcaFileSystemDriver::CreateIndirectStorage( // Get the indirect data size. const s64 indirect_data_size = patch_info.indirect_offset; - ASSERT(Common::IsAligned(indirect_data_size, NcaHeader::XtsBlockSize)); + R_UNLESS(Common::IsAligned(indirect_data_size, NcaHeader::XtsBlockSize), + ResultInvalidNcaIndirectStorageOutOfRange); // Create the indirect data storage. auto indirect_data_storage = @@ -1049,31 +1075,33 @@ Result NcaFileSystemDriver::CreatePatchMetaStorage( ASSERT(out_aes_ctr_ex_meta != nullptr); ASSERT(out_indirect_meta != nullptr); ASSERT(base_storage != nullptr); - ASSERT(Common::IsAligned(patch_info.aes_ctr_ex_size, NcaHeader::XtsBlockSize)); // Validate patch info extents. - R_UNLESS(patch_info.aes_ctr_ex_size >= 0 && patch_info.HasAesCtrExTable(), ResultInvalidNcaPatchInfoAesCtrExSize); + R_UNLESS(patch_info.aes_ctr_ex_size >= 0 && patch_info.HasAesCtrExTable() && + Common::IsAligned(patch_info.aes_ctr_ex_size, NcaHeader::XtsBlockSize), + ResultInvalidNcaPatchInfoAesCtrExSize); R_UNLESS(patch_info.indirect_size > 0 && patch_info.HasIndirectTable(), ResultInvalidNcaPatchInfoIndirectSize); - R_UNLESS(patch_info.indirect_size + patch_info.indirect_offset <= patch_info.aes_ctr_ex_offset, + R_UNLESS(IsRangeWithin(patch_info.indirect_offset, patch_info.indirect_size, + patch_info.aes_ctr_ex_offset), ResultInvalidNcaPatchInfoAesCtrExOffset); - R_UNLESS(patch_info.aes_ctr_ex_offset + patch_info.aes_ctr_ex_size <= - meta_data_hash_data_info.offset, + R_UNLESS(IsRangeWithin(patch_info.aes_ctr_ex_offset, patch_info.aes_ctr_ex_size, + meta_data_hash_data_info.offset), ResultRomNcaInvalidPatchMetaDataHashDataOffset); // Get the base storage size. s64 base_size = base_storage->GetSize(); // Check that extents remain within range. - R_UNLESS(patch_info.indirect_offset + patch_info.indirect_size <= base_size, + R_UNLESS(IsRangeWithin(patch_info.indirect_offset, patch_info.indirect_size, base_size), ResultNcaBaseStorageOutOfRangeE); - R_UNLESS(patch_info.aes_ctr_ex_offset + patch_info.aes_ctr_ex_size <= base_size, + R_UNLESS(IsRangeWithin(patch_info.aes_ctr_ex_offset, patch_info.aes_ctr_ex_size, base_size), ResultNcaBaseStorageOutOfRangeB); // Check that metadata hash data extents remain within range. const s64 meta_data_hash_data_offset = meta_data_hash_data_info.offset; const s64 meta_data_hash_data_size = Common::AlignUp(meta_data_hash_data_info.size, NcaHeader::CtrBlockSize); - R_UNLESS(meta_data_hash_data_offset + meta_data_hash_data_size <= base_size, + R_UNLESS(IsRangeWithin(meta_data_hash_data_offset, meta_data_hash_data_size, base_size), ResultNcaBaseStorageOutOfRangeB); // Create the encrypted storage. @@ -1128,7 +1156,8 @@ Result NcaFileSystemDriver::CreateSha3Storage( using VerificationStorage = HierarchicalSha3Storage; - R_UNLESS(Common::IsPowerOfTwo(hash_data.hash_block_size), + R_UNLESS(Common::IsPowerOfTwo(hash_data.hash_block_size) && + static_cast(hash_data.hash_block_size) >= VerificationStorage::HashSize, ResultInvalidHierarchicalSha256BlockSize); R_UNLESS(hash_data.hash_layer_count == VerificationStorage::LayerCount - 1, ResultInvalidHierarchicalSha256LayerCount); @@ -1136,20 +1165,25 @@ Result NcaFileSystemDriver::CreateSha3Storage( const auto& hash_region = hash_data.hash_layer_region[0]; const auto& data_region = hash_data.hash_layer_region[1]; - constexpr s32 CacheBlockCount = 2; + const s64 base_size = base_storage->GetSize(); + R_UNLESS(IsRangeWithin(hash_region.offset, hash_region.size, base_size), + ResultNcaBaseStorageOutOfRangeC); + R_UNLESS(IsRangeWithin(data_region.offset, data_region.size, base_size), + ResultNcaBaseStorageOutOfRangeC); + const s64 stored_size = GetBodySize(m_reader) + GetBodySize(m_original_reader); + R_UNLESS(hash_region.size <= stored_size, ResultNcaBaseStorageOutOfRangeC); + + R_UNLESS(Common::IsAligned(hash_region.size, VerificationStorage::HashSize) && + hash_region.size <= hash_data.hash_block_size, + ResultInvalidHierarchicalSha256BlockSize); + const auto hash_buffer_size = static_cast(hash_region.size); - const auto cache_buffer_size = CacheBlockCount * hash_data.hash_block_size; - const auto total_buffer_size = hash_buffer_size + cache_buffer_size; auto buffer_hold_storage = std::make_shared( - std::move(base_storage), total_buffer_size); + std::move(base_storage), hash_buffer_size); R_UNLESS(buffer_hold_storage != nullptr, ResultAllocationMemoryFailedAllocateShared); R_UNLESS(buffer_hold_storage->IsValid(), ResultAllocationMemoryFailedInNcaFileSystemDriverI); - s64 base_size = buffer_hold_storage->GetSize(); - R_UNLESS(hash_region.offset + hash_region.size <= base_size, ResultNcaBaseStorageOutOfRangeC); - R_UNLESS(data_region.offset + data_region.size <= base_size, ResultNcaBaseStorageOutOfRangeC); - auto master_hash_storage = std::make_shared>(hash_data.fs_data_master_hash.value); @@ -1181,7 +1215,8 @@ Result NcaFileSystemDriver::CreateSha256Storage( using VerificationStorage = HierarchicalSha256Storage; // Validate the hash data. - R_UNLESS(Common::IsPowerOfTwo(hash_data.hash_block_size), + R_UNLESS(Common::IsPowerOfTwo(hash_data.hash_block_size) && + static_cast(hash_data.hash_block_size) >= VerificationStorage::HashSize, ResultInvalidHierarchicalSha256BlockSize); R_UNLESS(hash_data.hash_layer_count == VerificationStorage::LayerCount - 1, ResultInvalidHierarchicalSha256LayerCount); @@ -1190,25 +1225,27 @@ Result NcaFileSystemDriver::CreateSha256Storage( const auto& hash_region = hash_data.hash_layer_region[0]; const auto& data_region = hash_data.hash_layer_region[1]; - // Determine buffer sizes. - constexpr s32 CacheBlockCount = 2; + // Check that we're within range. + const s64 base_size = base_storage->GetSize(); + R_UNLESS(IsRangeWithin(hash_region.offset, hash_region.size, base_size), + ResultNcaBaseStorageOutOfRangeC); + R_UNLESS(IsRangeWithin(data_region.offset, data_region.size, base_size), + ResultNcaBaseStorageOutOfRangeC); + const s64 stored_size = GetBodySize(m_reader) + GetBodySize(m_original_reader); + R_UNLESS(hash_region.size <= stored_size, ResultNcaBaseStorageOutOfRangeC); + + R_UNLESS(Common::IsAligned(hash_region.size, VerificationStorage::HashSize) && + hash_region.size <= hash_data.hash_block_size, + ResultInvalidHierarchicalSha256BlockSize); + const auto hash_buffer_size = static_cast(hash_region.size); - const auto cache_buffer_size = CacheBlockCount * hash_data.hash_block_size; - const auto total_buffer_size = hash_buffer_size + cache_buffer_size; // Make a buffer holder storage. auto buffer_hold_storage = std::make_shared( - std::move(base_storage), total_buffer_size); + std::move(base_storage), hash_buffer_size); R_UNLESS(buffer_hold_storage != nullptr, ResultAllocationMemoryFailedAllocateShared); R_UNLESS(buffer_hold_storage->IsValid(), ResultAllocationMemoryFailedInNcaFileSystemDriverI); - // Get storage size. - s64 base_size = buffer_hold_storage->GetSize(); - - // Check that we're within range. - R_UNLESS(hash_region.offset + hash_region.size <= base_size, ResultNcaBaseStorageOutOfRangeC); - R_UNLESS(data_region.offset + data_region.size <= base_size, ResultNcaBaseStorageOutOfRangeC); - // Create the master hash storage. auto master_hash_storage = std::make_shared>(hash_data.fs_data_master_hash.value); @@ -1255,9 +1292,16 @@ Result NcaFileSystemDriver::CreateIntegrityVerificationStorageForMeta( ResultInvalidNcaMetaDataHashDataSize); // Read the meta data hash data. + R_UNLESS(meta_data_hash_data_info.offset >= offset, ResultNcaBaseStorageOutOfRangeB); + const s64 meta_data_hash_data_offset = meta_data_hash_data_info.offset - offset; + R_UNLESS(IsRangeWithin(meta_data_hash_data_offset, sizeof(NcaMetaDataHashData), + base_storage->GetSize()), + ResultNcaBaseStorageOutOfRangeB); NcaMetaDataHashData meta_data_hash_data; - base_storage->ReadObject(std::addressof(meta_data_hash_data), - meta_data_hash_data_info.offset - offset); + base_storage->ReadObject(std::addressof(meta_data_hash_data), meta_data_hash_data_offset); + R_UNLESS(meta_data_hash_data.layer_info_offset >= offset && + meta_data_hash_data.layer_info_offset - offset <= meta_data_hash_data_offset, + ResultRomNcaInvalidIntegrityLayerInfoOffset); // Set the out layer info storage, if necessary. if (out_layer_info_storage != nullptr) { @@ -1307,14 +1351,24 @@ Result NcaFileSystemDriver::CreateIntegrityVerificationStorageImpl( R_UNLESS(level_hash_info.max_layers <= IntegrityMaxLayerCount, ResultInvalidNcaHierarchicalIntegrityVerificationLayerCount); + for (u32 i = 0; i < level_hash_info.max_layers - 1; ++i) { + const s32 block_order = level_hash_info.info[i].block_order; + R_UNLESS(block_order >= std::countr_zero(static_cast(IntegrityVerificationStorage::HashSize)) && + block_order < 32, + ResultInvalidNcaHierarchicalIntegrityVerificationLayerCount); + } + // Get the base storage size. s64 base_storage_size = base_storage->GetSize(); + R_UNLESS(layer_info_offset >= 0 && layer_info_offset <= base_storage_size, + ResultNcaBaseStorageOutOfRangeD); // Create storage info. StorageInfo storage_info; for (s32 i = 0; i < static_cast(level_hash_info.max_layers - 2); ++i) { const auto& layer_info = level_hash_info.info[i]; - R_UNLESS(layer_info_offset + layer_info.offset + layer_info.size <= base_storage_size, + R_UNLESS(IsRangeWithin(layer_info.offset, layer_info.size, + base_storage_size - layer_info_offset), ResultNcaBaseStorageOutOfRangeD); storage_info[i + 1] = std::make_shared(base_storage, @@ -1325,10 +1379,10 @@ Result NcaFileSystemDriver::CreateIntegrityVerificationStorageImpl( // Set the last layer info. const auto& layer_info = level_hash_info.info[level_hash_info.max_layers - 2]; const s64 last_layer_info_offset = layer_info_offset > 0 ? 0LL : layer_info.offset.Get(); - R_UNLESS(last_layer_info_offset + layer_info.size <= base_storage_size, + R_UNLESS(IsRangeWithin(last_layer_info_offset, layer_info.size, base_storage_size), ResultNcaBaseStorageOutOfRangeD); if (layer_info_offset > 0) { - R_UNLESS(last_layer_info_offset + layer_info.size <= layer_info_offset, + R_UNLESS(IsRangeWithin(last_layer_info_offset, layer_info.size, layer_info_offset), ResultRomNcaInvalidIntegrityLayerInfoOffset); } @@ -1354,6 +1408,8 @@ Result NcaFileSystemDriver::CreateIntegrityVerificationStorageImpl( case FileSys::HierarchicalIntegrityVerificationStorage::HierarchicalStorageInformation::DataStorage: storage_info.SetDataStorage(std::make_shared(std::move(base_storage), layer_info.size, last_layer_info_offset)); break; + default: + R_THROW(ResultInvalidNcaHierarchicalIntegrityVerificationLayerCount); } // Make the integrity romfs storage. diff --git a/src/core/file_sys/fssystem/fssystem_nca_header.h b/src/core/file_sys/fssystem/fssystem_nca_header.h index b10bb39e66..ebc0fa55fb 100644 --- a/src/core/file_sys/fssystem/fssystem_nca_header.h +++ b/src/core/file_sys/fssystem/fssystem_nca_header.h @@ -306,19 +306,24 @@ struct NcaFsHeader { Result GetHashTargetOffset(s64* out) const { switch (this->hash_type) { case HashType::HierarchicalIntegrityHash: - case HashType::HierarchicalIntegritySha3Hash: - *out = this->hash_data.integrity_meta_info.level_hash_info - .info[this->hash_data.integrity_meta_info.level_hash_info.max_layers - 2] - .offset; + case HashType::HierarchicalIntegritySha3Hash: { + const auto& level_hash_info = this->hash_data.integrity_meta_info.level_hash_info; + R_UNLESS(level_hash_info.max_layers >= 2 && + level_hash_info.max_layers - 2 < level_hash_info.info.size(), + ResultInvalidNcaFsHeader); + *out = level_hash_info.info[level_hash_info.max_layers - 2].offset; R_SUCCEED(); + } case HashType::HierarchicalSha256Hash: - case HashType::HierarchicalSha3256Hash: - *out = - this->hash_data.hierarchical_sha256_data - .hash_layer_region[this->hash_data.hierarchical_sha256_data.hash_layer_count - - 1] - .offset; + case HashType::HierarchicalSha3256Hash: { + const auto& sha256_data = this->hash_data.hierarchical_sha256_data; + R_UNLESS(sha256_data.hash_layer_count >= 1 && + static_cast(sha256_data.hash_layer_count) <= + sha256_data.hash_layer_region.size(), + ResultInvalidNcaFsHeader); + *out = sha256_data.hash_layer_region[sha256_data.hash_layer_count - 1].offset; R_SUCCEED(); + } default: R_THROW(ResultInvalidNcaFsHeader); } diff --git a/src/core/file_sys/fssystem/fssystem_nca_reader.cpp b/src/core/file_sys/fssystem/fssystem_nca_reader.cpp index 08924e2a6c..3f557c20fb 100644 --- a/src/core/file_sys/fssystem/fssystem_nca_reader.cpp +++ b/src/core/file_sys/fssystem/fssystem_nca_reader.cpp @@ -50,6 +50,9 @@ Result NcaReader::Initialize(VirtualFile base_storage, const NcaCryptoConfigurat // We need to be able to generate keys. R_UNLESS(crypto_cfg.generate_key != nullptr, ResultInvalidArgument); + // The header must be there to read. + R_UNLESS(base_storage->GetSize() >= sizeof(NcaHeader), ResultInvalidNcaHeader); + // Generate keys for header. using AesXtsStorageForNcaHeader = AesXtsStorage; @@ -360,7 +363,8 @@ Result NcaReader::ReadHeader(NcaFsHeader* dst, s32 index) const { ASSERT(0 <= index && index < NcaHeader::FsCountMax); const s64 offset = sizeof(NcaHeader) + sizeof(NcaFsHeader) * index; - m_header_storage->ReadObject(dst, offset); + R_UNLESS(m_header_storage->ReadObject(dst, offset) == sizeof(NcaFsHeader), + ResultInvalidNcaFsHeader); R_SUCCEED(); } diff --git a/src/core/file_sys/partition_filesystem.cpp b/src/core/file_sys/partition_filesystem.cpp index 5d88cd8d95..09b3a95cfd 100644 --- a/src/core/file_sys/partition_filesystem.cpp +++ b/src/core/file_sys/partition_filesystem.cpp @@ -8,6 +8,7 @@ #include #include #include +#include #include #include "common/logging.h" @@ -43,29 +44,39 @@ PartitionFilesystem::PartitionFilesystem(VirtualFile file) { is_hfs = pfs_header.magic == Common::MakeMagic('H', 'F', 'S', '0'); - std::size_t entry_size = is_hfs ? sizeof(HFSEntry) : sizeof(PFSEntry); - std::size_t metadata_size = + const u64 entry_size = is_hfs ? sizeof(HFSEntry) : sizeof(PFSEntry); + const u64 metadata_size = sizeof(Header) + (pfs_header.num_entries * entry_size) + pfs_header.strtab_size; + if (metadata_size > file->GetSize()) { + status = Loader::ResultStatus::ErrorIncorrectPFSFileSize; + return; + } // Actually read in now... - std::vector file_data = file->ReadBytes(metadata_size); - const std::size_t total_size = file_data.size(); - file_data.push_back(0); - - if (total_size != metadata_size) { + const std::vector file_data = file->ReadBytes(metadata_size); + if (file_data.size() != metadata_size) { status = Loader::ResultStatus::ErrorIncorrectPFSFileSize; return; } - std::size_t entries_offset = sizeof(Header); - std::size_t strtab_offset = entries_offset + (pfs_header.num_entries * entry_size); + const std::size_t entries_offset = sizeof(Header); + const std::size_t strtab_offset = entries_offset + (pfs_header.num_entries * entry_size); + const std::span strtab{ + reinterpret_cast(file_data.data()) + strtab_offset, pfs_header.strtab_size}; content_offset = strtab_offset + pfs_header.strtab_size; - for (u16 i = 0; i < pfs_header.num_entries; i++) { + for (u32 i = 0; i < pfs_header.num_entries; i++) { FSEntry entry; memcpy(&entry, &file_data[entries_offset + (i * entry_size)], sizeof(FSEntry)); - std::string name( - reinterpret_cast(&file_data[strtab_offset + entry.strtab_offset])); + if (entry.strtab_offset >= strtab.size()) { + offsets.clear(); + sizes.clear(); + pfs_files.clear(); + status = Loader::ResultStatus::ErrorBadPFSHeader; + return; + } + const auto name_bytes = strtab.subspan(entry.strtab_offset); + std::string name(name_bytes.data(), strnlen(name_bytes.data(), name_bytes.size())); offsets.insert_or_assign(name, content_offset + entry.offset); sizes.insert_or_assign(name, entry.size); diff --git a/src/core/file_sys/romfs.cpp b/src/core/file_sys/romfs.cpp index 9cf4928cd9..5227330937 100644 --- a/src/core/file_sys/romfs.cpp +++ b/src/core/file_sys/romfs.cpp @@ -5,9 +5,10 @@ // SPDX-License-Identifier: GPL-2.0-or-later #include +#include -#include "common/assert.h" #include "common/common_types.h" +#include "common/logging.h" #include "common/string_util.h" #include "common/swap.h" #include "core/file_sys/fsmitm_romfsbuild.h" @@ -21,6 +22,9 @@ namespace FileSys { namespace { constexpr u32 ROMFS_ENTRY_EMPTY = 0xFFFFFFFF; +// Each level adds at least two characters to a path, and Horizon paths hold 0x300 of them. The +// limit also bounds the destructor recursion of the extracted tree. +constexpr size_t MaxDirectoryDepth = 0x300 / 2; struct TableLocation { u64_le offset; @@ -63,68 +67,81 @@ struct RomFSTraversalContext { VirtualFile file; std::vector directory_meta; std::vector file_meta; + // A valid RomFS links every entry once, so an entry reached twice means the links loop. + std::vector directory_seen; + std::vector file_seen; }; -template -std::pair GetEntry(const RomFSTraversalContext& ctx, size_t offset) { - const size_t entry_end = offset + sizeof(EntryType); - const std::vector& vec = ctx.*Member; - const size_t size = vec.size(); - const u8* data = vec.data(); - EntryType entry{}; - - if (entry_end > size) { - return {}; +template +std::optional> GetEntry(const std::vector& meta, + std::vector& seen, u32 offset) { + if (offset >= meta.size() || meta.size() - offset < sizeof(EntryType) || seen[offset]) { + return std::nullopt; } - std::memcpy(&entry, data + offset, sizeof(EntryType)); + seen[offset] = true; - const size_t name_length = (std::min)(entry_end + entry.name_length, size) - entry_end; - std::string name(reinterpret_cast(data + entry_end), name_length); - - return {entry, std::move(name)}; -} + EntryType entry{}; + std::memcpy(&entry, meta.data() + offset, sizeof(EntryType)); -std::pair GetDirectoryEntry(const RomFSTraversalContext& ctx, - size_t directory_offset) { - return GetEntry(ctx, directory_offset); -} + const size_t name_offset = offset + sizeof(EntryType); + const size_t name_length = (std::min)(entry.name_length, meta.size() - name_offset); + std::string name(reinterpret_cast(meta.data() + name_offset), name_length); -std::pair GetFileEntry(const RomFSTraversalContext& ctx, - size_t file_offset) { - return GetEntry(ctx, file_offset); + return std::pair{entry, std::move(name)}; } -void ProcessFile(const RomFSTraversalContext& ctx, u32 this_file_offset, - std::shared_ptr& parent) { - while (this_file_offset != ROMFS_ENTRY_EMPTY) { - auto entry = GetFileEntry(ctx, this_file_offset); +bool ProcessFiles(RomFSTraversalContext& ctx, u32 file_offset, VectorVfsDirectory& parent) { + while (file_offset != ROMFS_ENTRY_EMPTY) { + auto entry = GetEntry(ctx.file_meta, ctx.file_seen, file_offset); + if (!entry) { + return false; + } - parent->AddFile(std::make_shared(ctx.file, entry.first.size, - entry.first.offset + ctx.header.data_offset, - std::move(entry.second))); + parent.AddFile(std::make_shared(ctx.file, entry->first.size, + entry->first.offset + ctx.header.data_offset, + std::move(entry->second))); - this_file_offset = entry.first.sibling; + file_offset = entry->first.sibling; } + return true; } -void ProcessDirectory(const RomFSTraversalContext& ctx, u32 this_dir_offset, - std::shared_ptr& parent) { - while (this_dir_offset != ROMFS_ENTRY_EMPTY) { - auto entry = GetDirectoryEntry(ctx, this_dir_offset); - auto current = std::make_shared( - std::vector{}, std::vector{}, entry.second); - - if (entry.first.child_file != ROMFS_ENTRY_EMPTY) { - ProcessFile(ctx, entry.first.child_file, current); +// Walks the directory tree with an explicit stack, since its depth comes from the file. +bool ProcessDirectories(RomFSTraversalContext& ctx, VectorVfsDirectory& root) { + struct Pending { + u32 dir_offset; + VectorVfsDirectory* parent; + size_t depth; + }; + std::vector pending{{0, &root, 0}}; + while (!pending.empty()) { + auto [dir_offset, parent, depth] = pending.back(); + pending.pop_back(); + if (depth > MaxDirectoryDepth) { + return false; } - if (entry.first.child_dir != ROMFS_ENTRY_EMPTY) { - ProcessDirectory(ctx, entry.first.child_dir, current); - } + while (dir_offset != ROMFS_ENTRY_EMPTY) { + auto entry = GetEntry(ctx.directory_meta, ctx.directory_seen, dir_offset); + if (!entry) { + return false; + } + + auto current = std::make_shared( + std::vector{}, std::vector{}, std::move(entry->second)); + parent->AddDirectory(current); - parent->AddDirectory(current); - this_dir_offset = entry.first.sibling; + if (!ProcessFiles(ctx, entry->first.child_file, *current)) { + return false; + } + if (entry->first.child_dir != ROMFS_ENTRY_EMPTY) { + pending.push_back({entry->first.child_dir, current.get(), depth + 1}); + } + + dir_offset = entry->first.sibling; + } } + return true; } } // Anonymous namespace @@ -148,14 +165,24 @@ VirtualDir ExtractRomFS(VirtualFile file) { ctx.directory_meta = file->ReadBytes(ctx.header.directory_meta.size, ctx.header.directory_meta.offset); ctx.file_meta = file->ReadBytes(ctx.header.file_meta.size, ctx.header.file_meta.offset); + if (ctx.directory_meta.size() != ctx.header.directory_meta.size || + ctx.file_meta.size() != ctx.header.file_meta.size) { + LOG_ERROR(Loader, "RomFS directory or file table runs past the end of the file"); + return nullptr; + } + ctx.directory_seen.resize(ctx.directory_meta.size()); + ctx.file_seen.resize(ctx.file_meta.size()); - ProcessDirectory(ctx, 0, root_container); + if (!ProcessDirectories(ctx, *root_container)) { + LOG_ERROR(Loader, "RomFS directory or file tables are malformed"); + return nullptr; + } if (auto root = root_container->GetSubdirectory(""); root) { return root; } - ASSERT(false); + LOG_ERROR(Loader, "RomFS root directory has a name"); return nullptr; } diff --git a/src/core/file_sys/vfs/vfs.cpp b/src/core/file_sys/vfs/vfs.cpp index b85b843e3e..7d10b7fd53 100644 --- a/src/core/file_sys/vfs/vfs.cpp +++ b/src/core/file_sys/vfs/vfs.cpp @@ -176,8 +176,12 @@ std::optional VfsFile::ReadByte(std::size_t offset) const { } std::vector VfsFile::ReadBytes(std::size_t size, std::size_t offset) const { - std::vector out(size); - std::size_t read_size = Read(out.data(), size, offset); + const std::size_t file_size = GetSize(); + if (offset >= file_size || size == 0) { + return {}; + } + std::vector out((std::min)(size, file_size - offset)); + std::size_t read_size = Read(out.data(), out.size(), offset); out.resize(read_size); return out; } diff --git a/src/core/file_sys/vfs/vfs_vector.cpp b/src/core/file_sys/vfs/vfs_vector.cpp index 4d4a90f640..dfb35ecc09 100644 --- a/src/core/file_sys/vfs/vfs_vector.cpp +++ b/src/core/file_sys/vfs/vfs_vector.cpp @@ -40,6 +40,9 @@ bool VectorVfsFile::IsReadable() const { } std::size_t VectorVfsFile::Read(u8* data_, std::size_t length, std::size_t offset) const { + if (offset >= data.size()) { + return 0; + } const auto read = (std::min)(length, data.size() - offset); std::memcpy(data_, data.data() + offset, read); return read; diff --git a/src/fuzz/CMakeLists.txt b/src/fuzz/CMakeLists.txt new file mode 100644 index 0000000000..daa8739c21 --- /dev/null +++ b/src/fuzz/CMakeLists.txt @@ -0,0 +1,8 @@ +# SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +# SPDX-License-Identifier: GPL-3.0-or-later + +foreach(harness fuzz_partition_romfs fuzz_nca fuzz_package) + add_executable(${harness} ${harness}.cpp fuzz_common.h stubs.cpp) + target_link_libraries(${harness} PRIVATE common core video_core) + target_link_options(${harness} PRIVATE -fsanitize=fuzzer) +endforeach() diff --git a/src/fuzz/fuzz_common.h b/src/fuzz/fuzz_common.h new file mode 100644 index 0000000000..07eb676372 --- /dev/null +++ b/src/fuzz/fuzz_common.h @@ -0,0 +1,110 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later + +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "common/common_types.h" +#include "common/fs/path_util.h" +#include "core/file_sys/romfs.h" +#include "core/file_sys/vfs/vfs.h" +#include "core/file_sys/vfs/vfs_vector.h" +#include "core/loader/loader.h" + +extern "C" int LLVMFuzzerInitialize(int* argc, char*** argv); +extern "C" int LLVMFuzzerTestOneInput(const u8* data, std::size_t size); + +namespace Fuzz { + +// Bounds the work one input can cause, so a valid but huge tree does not read as a hang. +struct Budget { + std::size_t files = 256; + std::size_t dirs = 256; + std::size_t bytes = 32u << 20; + int depth = 16; +}; + +// Reads in chunks, so a bogus declared size does not make the harness itself allocate. +inline void ReadAll(const FileSys::VirtualFile& file, Budget& budget) { + if (!file || budget.files == 0) { + return; + } + --budget.files; + std::vector chunk(64 * 1024); + const std::size_t size = file->GetSize(); + std::size_t offset = 0; + while (offset < size && budget.bytes > 0) { + const std::size_t want = std::min({chunk.size(), size - offset, budget.bytes}); + const std::size_t got = file->Read(chunk.data(), want, offset); + if (got == 0) { + break; + } + offset += got; + budget.bytes -= std::min(got, budget.bytes); + } +} + +inline void Walk(const FileSys::VirtualDir& dir, Budget& budget, int depth = 0) { + if (!dir || depth > budget.depth || budget.dirs == 0) { + return; + } + --budget.dirs; + for (const auto& file : dir->GetFiles()) { + ReadAll(file, budget); + } + for (const auto& sub : dir->GetSubdirectories()) { + Walk(sub, budget, depth + 1); + } +} + +inline void ReadRomFS(const FileSys::VirtualFile& romfs, Budget& budget) { + if (!romfs) { + return; + } + Walk(FileSys::ExtractRomFS(romfs), budget); +} + +// Points the key manager at a scratch keys directory holding made-up key area keys for every +// generation, as a user with dumped keys has. Plaintext NCAs then get past the key check without +// any real keys, and the harness never reads or writes the user's own keys. Call it before +// anything touches Core::Crypto::KeyManager. +inline void UseSyntheticKeys() { + const auto dir = std::filesystem::temp_directory_path() / fmt::format("lichii-fuzz-keys-{}", getpid()); + std::filesystem::create_directories(dir); + std::ofstream keys{dir / "prod.keys"}; + for (const char* kind : {"application", "ocean", "system"}) { + for (int generation = 0; generation < 0x20; ++generation) { + keys << fmt::format("key_area_key_{}_{:02x} = {:032x}\n", kind, generation, 0x11); + } + } + Common::FS::SetEdenPath(Common::FS::EdenPath::KeysDir, dir); +} + +// A PFS0 (NSP layout) holding one file. +inline FileSys::VirtualFile WrapInPfs0(const u8* data, std::size_t size, const char* name) { + const std::size_t name_size = std::strlen(name) + 1; + const std::size_t strtab_size = (name_size + 0x1F) & ~std::size_t{0x1F}; + std::vector out(0x10 + 0x18 + strtab_size + size); + const u32 magic = 0x30534650; + const u32 num_entries = 1; + const u32 strtab = static_cast(strtab_size); + const u64 entry_size = size; + std::memcpy(out.data(), &magic, 4); + std::memcpy(out.data() + 4, &num_entries, 4); + std::memcpy(out.data() + 8, &strtab, 4); + std::memcpy(out.data() + 0x10 + 8, &entry_size, 8); + std::memcpy(out.data() + 0x28, name, name_size); + std::memcpy(out.data() + 0x28 + strtab_size, data, size); + return std::make_shared(std::move(out), "fuzz.nsp"); +} + +} // namespace Fuzz diff --git a/src/fuzz/fuzz_nca.cpp b/src/fuzz/fuzz_nca.cpp new file mode 100644 index 0000000000..3a875fb505 --- /dev/null +++ b/src/fuzz/fuzz_nca.cpp @@ -0,0 +1,54 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later + +// One input as an NCA: on its own, as an update NCA whose base is missing (as the patch manager +// opens it), and inside an NSP. Plaintext NCA headers and sections need no keys, so the input +// reaches the FS header, integrity, patch and RomFS parsers directly. + +#include "core/file_sys/content_archive.h" +#include "core/file_sys/submission_package.h" +#include "fuzz/fuzz_common.h" + +namespace { + +void Exercise(FileSys::NCA& nca, Fuzz::Budget& budget) { + if (nca.GetStatus() != Loader::ResultStatus::Success) { + return; + } + (void)nca.GetType(); + (void)nca.GetTitleId(); + (void)nca.GetSDKVersion(); + for (const auto& dir : nca.GetSubdirectories()) { + Fuzz::Walk(dir, budget); + } + Fuzz::ReadRomFS(nca.GetRomFS(), budget); + Fuzz::Walk(nca.GetExeFS(), budget); + Fuzz::Walk(nca.GetLogoPartition(), budget); +} + +} // namespace + +extern "C" int LLVMFuzzerInitialize(int*, char***) { + Fuzz::UseSyntheticKeys(); + return 0; +} + +extern "C" int LLVMFuzzerTestOneInput(const u8* data, std::size_t size) { + auto file = std::make_shared(std::vector(data, data + size), + "00000000000000000000000000000000.nca"); + Fuzz::Budget budget; + { + FileSys::NCA nca{file}; + Exercise(nca, budget); + } + { + FileSys::NCA nca{file, nullptr, true}; + Exercise(nca, budget); + } + FileSys::NSP nsp{Fuzz::WrapInPfs0(data, size, "00000000000000000000000000000000.nca")}; + if (nsp.GetStatus() == Loader::ResultStatus::Success) { + Fuzz::ReadRomFS(nsp.GetRomFS(), budget); + Fuzz::Walk(nsp.GetExeFS(), budget); + } + return 0; +} diff --git a/src/fuzz/fuzz_package.cpp b/src/fuzz/fuzz_package.cpp new file mode 100644 index 0000000000..feec8631f8 --- /dev/null +++ b/src/fuzz/fuzz_package.cpp @@ -0,0 +1,40 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later + +// One input as a whole NSP and as a whole XCI, the containers games ship in. + +#include "core/file_sys/card_image.h" +#include "core/file_sys/submission_package.h" +#include "fuzz/fuzz_common.h" + +extern "C" int LLVMFuzzerInitialize(int*, char***) { + Fuzz::UseSyntheticKeys(); + return 0; +} + +extern "C" int LLVMFuzzerTestOneInput(const u8* data, std::size_t size) { + auto file = std::make_shared(std::vector(data, data + size), + "fuzz.bin"); + Fuzz::Budget budget; + { + FileSys::NSP nsp{file}; + if (nsp.GetStatus() == Loader::ResultStatus::Success) { + (void)nsp.GetProgramTitleIDs(); + Fuzz::ReadRomFS(nsp.GetRomFS(), budget); + Fuzz::Walk(nsp.GetExeFS(), budget); + } + } + { + FileSys::XCI xci{file}; + if (xci.GetStatus() == Loader::ResultStatus::Success) { + for (const auto& partition : xci.GetPartitions()) { + Fuzz::Walk(partition, budget); + } + if (const auto secure = xci.GetSecurePartitionNSP()) { + Fuzz::ReadRomFS(secure->GetRomFS(), budget); + Fuzz::Walk(secure->GetExeFS(), budget); + } + } + } + return 0; +} diff --git a/src/fuzz/fuzz_partition_romfs.cpp b/src/fuzz/fuzz_partition_romfs.cpp new file mode 100644 index 0000000000..91873251df --- /dev/null +++ b/src/fuzz/fuzz_partition_romfs.cpp @@ -0,0 +1,21 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later + +// One input as a raw PFS0/HFS0 partition and as a raw RomFS image. + +#include "core/file_sys/partition_filesystem.h" +#include "fuzz/fuzz_common.h" + +extern "C" int LLVMFuzzerTestOneInput(const u8* data, std::size_t size) { + auto file = std::make_shared(std::vector(data, data + size), + "fuzz.bin"); + Fuzz::Budget budget; + FileSys::PartitionFilesystem pfs{file}; + if (pfs.GetStatus() == Loader::ResultStatus::Success) { + for (const auto& entry : pfs.GetFiles()) { + Fuzz::ReadAll(entry, budget); + } + } + Fuzz::ReadRomFS(file, budget); + return 0; +} diff --git a/src/fuzz/stubs.cpp b/src/fuzz/stubs.cpp new file mode 100644 index 0000000000..10b7db642a --- /dev/null +++ b/src/fuzz/stubs.cpp @@ -0,0 +1,19 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later + +// Symbols the frontends normally provide. + +#define VMA_IMPLEMENTATION +#include "video_core/vulkan_common/vma.h" + +#include "frontend_common/play_time_manager.h" + +namespace PlayTime { + +PlayTimeManager::PlayTimeManager() {} +PlayTimeManager::~PlayTimeManager() {} +u64 PlayTimeManager::GetPlayTime(u64 program_id) const { + return 0; +} + +} // namespace PlayTime diff --git a/src/tests/CMakeLists.txt b/src/tests/CMakeLists.txt index 6254aabb4b..ae4fb618fb 100644 --- a/src/tests/CMakeLists.txt +++ b/src/tests/CMakeLists.txt @@ -47,6 +47,8 @@ add_executable(tests video_core/storage_write_extent.cpp video_core/storage_buffer_control_flow.cpp core/nup_validation.cpp + core/vfs_vector.cpp + core/file_sys_parsers.cpp input_common/calibration_lifetime.cpp video_core/ampere_logged_numeric.cpp video_core/ampere_logged_textures.cpp diff --git a/src/tests/core/file_sys_parsers.cpp b/src/tests/core/file_sys_parsers.cpp new file mode 100644 index 0000000000..c21588002c --- /dev/null +++ b/src/tests/core/file_sys_parsers.cpp @@ -0,0 +1,493 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later + +// Malformed PFS0, RomFS and NCA inputs, built here from scratch, that crashed or hung the parsers +// before they checked their header fields. Each must now fail cleanly, and valid input of the +// same shape must still parse. + +#include +#include +#include +#include +#include +#include + +#include + +#include "core/file_sys/card_image.h" +#include "core/file_sys/errors.h" +#include "core/file_sys/fssystem/fssystem_bucket_tree.h" +#include "core/file_sys/fssystem/fssystem_compression_configuration.h" +#include "core/file_sys/fssystem/fssystem_crypto_configuration.h" +#include "core/file_sys/fssystem/fssystem_indirect_storage.h" +#include "core/file_sys/fssystem/fssystem_integrity_verification_storage.h" +#include "core/file_sys/fssystem/fssystem_nca_file_system_driver.h" +#include "core/file_sys/fssystem/fssystem_nca_header.h" +#include "core/file_sys/partition_filesystem.h" +#include "core/file_sys/romfs.h" +#include "core/file_sys/vfs/vfs_vector.h" +#include "core/loader/loader.h" + +namespace { + +using namespace FileSys; + +template +void Put(std::vector& out, size_t offset, const T& value) { + if (out.size() < offset + sizeof(T)) { + out.resize(offset + sizeof(T)); + } + std::memcpy(out.data() + offset, &value, sizeof(T)); +} + +VirtualFile MakeFile(std::vector bytes) { + return std::make_shared(std::move(bytes), "test.bin"); +} + +// PFS0 with one 0x18-byte entry per name, the names packed into the string table. +std::vector MakePfs0(const std::vector& names) { + std::string strtab; + std::vector name_offsets; + for (const auto& name : names) { + name_offsets.push_back(static_cast(strtab.size())); + strtab += name; + strtab += '\0'; + } + + const u32 num_entries = static_cast(names.size()); + std::vector out; + Put(out, 0, Common::MakeMagic('P', 'F', 'S', '0')); + Put(out, 4, num_entries); + Put(out, 8, static_cast(strtab.size())); + out.resize(0x10 + num_entries * 0x18); + for (u32 i = 0; i < num_entries; ++i) { + Put(out, 0x10 + i * 0x18 + 0x10, name_offsets[i]); + } + out.insert(out.end(), strtab.begin(), strtab.end()); + return out; +} + +TEST_CASE("PFS0 with valid entries parses", "[core][file_sys]") { + PartitionFilesystem pfs{MakeFile(MakePfs0({"main", "main.npdm"}))}; + REQUIRE(pfs.GetStatus() == Loader::ResultStatus::Success); + REQUIRE(pfs.GetFiles().size() == 2); + REQUIRE(pfs.GetFile("main.npdm") != nullptr); +} + +TEST_CASE("PFS0 rejects a name offset past the string table", "[core][file_sys]") { + auto bytes = MakePfs0({"main"}); + Put(bytes, 0x10 + 0x10, u32{0x7FFFFFF0}); + PartitionFilesystem pfs{MakeFile(std::move(bytes))}; + REQUIRE(pfs.GetStatus() == Loader::ResultStatus::ErrorBadPFSHeader); + REQUIRE(pfs.GetFiles().empty()); +} + +TEST_CASE("PFS0 name stops at the end of the string table", "[core][file_sys]") { + auto bytes = MakePfs0({"main"}); + bytes.back() = 'x'; + PartitionFilesystem pfs{MakeFile(std::move(bytes))}; + REQUIRE(pfs.GetStatus() == Loader::ResultStatus::Success); + REQUIRE(pfs.GetFile("mainx") != nullptr); +} + +TEST_CASE("PFS0 rejects a table larger than the file without allocating it", + "[core][file_sys]") { + auto bytes = MakePfs0({"main"}); + Put(bytes, 4, u32{0xFFFFFFFF}); + Put(bytes, 8, u32{0xFFFFFFFF}); + PartitionFilesystem pfs{MakeFile(std::move(bytes))}; + REQUIRE(pfs.GetStatus() == Loader::ResultStatus::ErrorIncorrectPFSFileSize); +} + +TEST_CASE("PFS0 with more than 65535 entries reads every entry", "[core][file_sys]") { + PartitionFilesystem pfs{MakeFile(MakePfs0(std::vector(70000, "a")))}; + REQUIRE(pfs.GetStatus() == Loader::ResultStatus::Success); + REQUIRE(pfs.GetFiles().size() == 70000); +} + +constexpr u32 Empty = 0xFFFFFFFF; +constexpr size_t Sibling = 1; +constexpr size_t ChildDir = 2; + +// RomFS with the given directory and file meta tables and the data after them. +struct RomFSBuilder { + std::vector dirs; + std::vector files; + std::vector data; + + u32 AddDirectory(u32 parent, u32 sibling, u32 child_dir, u32 child_file, + const std::string& name) { + const auto offset = static_cast(dirs.size()); + const u32 fields[] = {parent, sibling, child_dir, child_file, 0, + static_cast(name.size())}; + for (size_t i = 0; i < std::size(fields); ++i) { + Put(dirs, offset + i * 4, fields[i]); + } + dirs.insert(dirs.end(), name.begin(), name.end()); + dirs.resize((dirs.size() + 3) & ~size_t{3}); + return offset; + } + + u32 AddFile(u32 parent, u32 sibling, const std::string& name, const std::string& contents) { + const auto offset = static_cast(files.size()); + Put(files, offset, parent); + Put(files, offset + 4, sibling); + Put(files, offset + 8, static_cast(data.size())); + Put(files, offset + 0x10, static_cast(contents.size())); + Put(files, offset + 0x18, u32{0}); + Put(files, offset + 0x1C, static_cast(name.size())); + files.insert(files.end(), name.begin(), name.end()); + files.resize((files.size() + 3) & ~size_t{3}); + data.insert(data.end(), contents.begin(), contents.end()); + return offset; + } + + void SetDirectory(u32 offset, size_t field, u32 value) { + Put(dirs, offset + field * 4, value); + } + + std::vector Build() const { + const u64 dirs_offset = 0x50; + const u64 files_offset = dirs_offset + dirs.size(); + const u64 data_offset = files_offset + files.size(); + std::vector out; + Put(out, 0x00, u64{0x50}); + Put(out, 0x18, dirs_offset); + Put(out, 0x20, static_cast(dirs.size())); + Put(out, 0x38, files_offset); + Put(out, 0x40, static_cast(files.size())); + Put(out, 0x48, data_offset); + out.insert(out.end(), dirs.begin(), dirs.end()); + out.insert(out.end(), files.begin(), files.end()); + out.insert(out.end(), data.begin(), data.end()); + return out; + } +}; + +// Root, then directory "a" at 0x18 holding file "f" with "hi". +RomFSBuilder MakeSmallRomFS() { + RomFSBuilder romfs; + romfs.AddDirectory(0, Empty, 0x18, Empty, ""); + romfs.AddDirectory(0, Empty, Empty, 0, "a"); + romfs.AddFile(0x18, Empty, "f", "hi"); + return romfs; +} + +TEST_CASE("RomFS with a valid tree extracts", "[core][file_sys]") { + const auto root = ExtractRomFS(MakeFile(MakeSmallRomFS().Build())); + REQUIRE(root != nullptr); + const auto file = root->GetFileRelative("a/f"); + REQUIRE(file != nullptr); + REQUIRE(file->ReadAllBytes() == std::vector{'h', 'i'}); +} + +TEST_CASE("RomFS rejects a directory that is its own sibling", "[core][file_sys]") { + auto romfs = MakeSmallRomFS(); + romfs.SetDirectory(0x18, Sibling, 0x18); + REQUIRE(ExtractRomFS(MakeFile(romfs.Build())) == nullptr); +} + +TEST_CASE("RomFS rejects a directory whose child is its parent", "[core][file_sys]") { + auto romfs = MakeSmallRomFS(); + romfs.SetDirectory(0x18, ChildDir, 0); + REQUIRE(ExtractRomFS(MakeFile(romfs.Build())) == nullptr); +} + +TEST_CASE("RomFS rejects a file that is its own sibling", "[core][file_sys]") { + auto romfs = MakeSmallRomFS(); + Put(romfs.files, 4, u32{0}); + REQUIRE(ExtractRomFS(MakeFile(romfs.Build())) == nullptr); +} + +TEST_CASE("RomFS rejects an entry past the end of its table", "[core][file_sys]") { + auto romfs = MakeSmallRomFS(); + romfs.SetDirectory(0x18, Sibling, 0x1000); + REQUIRE(ExtractRomFS(MakeFile(romfs.Build())) == nullptr); +} + +TEST_CASE("RomFS rejects a table larger than the file without allocating it", + "[core][file_sys]") { + auto bytes = MakeSmallRomFS().Build(); + Put(bytes, 0x20, u64{0x7FFF'FFFF'FFFF'0000}); + REQUIRE(ExtractRomFS(MakeFile(std::move(bytes))) == nullptr); +} + +// A chain of directories the given number of levels below the root. +std::vector MakeDeepRomFS(size_t depth) { + RomFSBuilder romfs; + romfs.AddDirectory(0, Empty, Empty, Empty, ""); + u32 parent = 0; + for (size_t i = 0; i < depth; ++i) { + const u32 child = romfs.AddDirectory(parent, Empty, Empty, Empty, "d"); + romfs.SetDirectory(parent, ChildDir, child); + parent = child; + } + return romfs.Build(); +} + +TEST_CASE("RomFS extracts a tree as deep as a Horizon path allows", "[core][file_sys]") { + auto dir = ExtractRomFS(MakeFile(MakeDeepRomFS(0x300 / 2))); + REQUIRE(dir != nullptr); + size_t depth = 0; + while (!dir->GetSubdirectories().empty()) { + dir = dir->GetSubdirectories().front(); + ++depth; + } + REQUIRE(depth == 0x300 / 2); +} + +TEST_CASE("RomFS rejects a tree deeper than a Horizon path allows", "[core][file_sys]") { + REQUIRE(ExtractRomFS(MakeFile(MakeDeepRomFS(100000))) == nullptr); +} + +// A plaintext NCA, which nx2 opens without a header key, with one RomFS section spanning +// [0xC00, 0x4C00) whose FS header and contents the test fills in. +struct PlaintextNca { + NcaHeader header{}; + NcaFsHeader fs{}; + std::vector section; + + PlaintextNca() { + header.magic = NcaHeader::Magic3; + header.content_type = NcaHeader::ContentType::Program; + header.sdk_addon_version = 0x000C0000; + header.fs_info[0].start_sector = 6; + header.fs_info[0].end_sector = 0x26; + fs.version = 2; + fs.fs_type = NcaFsHeader::FsType::RomFs; + fs.encryption_type = NcaFsHeader::EncryptionType::None; + } + + Result Open() const { + std::vector bytes(std::max(0x4C00, 0xC00 + section.size())); + std::memcpy(bytes.data(), &header, sizeof(header)); + std::memcpy(bytes.data() + sizeof(header), &fs, sizeof(fs)); + std::copy(section.begin(), section.end(), bytes.begin() + 0xC00); + + auto reader = std::make_shared(); + R_TRY(reader->Initialize(MakeFile(std::move(bytes)), GetCryptoConfiguration(), + GetNcaCompressionConfiguration())); + NcaFileSystemDriver driver{reader}; + VirtualFile storage; + NcaFsHeaderReader header_reader; + R_TRY(driver.OpenStorage(&storage, &header_reader, 0)); + R_UNLESS(storage != nullptr, ResultInvalidNcaFsHeader); + R_SUCCEED(); + } +}; + +PlaintextNca MakeSha256Nca() { + PlaintextNca nca; + nca.fs.hash_type = NcaFsHeader::HashType::HierarchicalSha256Hash; + auto& sha = nca.fs.hash_data.hierarchical_sha256_data; + sha.hash_block_size = 0x1000; + sha.hash_layer_count = 2; + sha.hash_layer_region[0].offset = 0; + sha.hash_layer_region[0].size = 0x20; + sha.hash_layer_region[1].offset = 0x1000; + sha.hash_layer_region[1].size = 0x1000; + return nca; +} + +TEST_CASE("NCA with a valid SHA-256 section opens", "[core][file_sys]") { + REQUIRE(MakeSha256Nca().Open() == ResultSuccess); +} + +TEST_CASE("NCA with an AES-XTS section opens a storage", "[core][file_sys]") { + auto nca = MakeSha256Nca(); + nca.fs.encryption_type = NcaFsHeader::EncryptionType::AesXts; + REQUIRE(nca.Open() == ResultSuccess); +} + +TEST_CASE("NCA rejects a file shorter than its header", "[core][file_sys]") { + auto reader = std::make_shared(); + REQUIRE(reader->Initialize(MakeFile(std::vector(0x100)), GetCryptoConfiguration(), + GetNcaCompressionConfiguration()) == ResultInvalidNcaHeader); +} + +TEST_CASE("NCA rejects a SHA-256 block smaller than a hash", "[core][file_sys]") { + auto nca = MakeSha256Nca(); + nca.fs.hash_data.hierarchical_sha256_data.hash_block_size = 8; + REQUIRE(nca.Open() == ResultInvalidHierarchicalSha256BlockSize); +} + +TEST_CASE("NCA with a SHA-256 block larger than the file opens without allocating it", + "[core][file_sys]") { + auto nca = MakeSha256Nca(); + auto& sha = nca.fs.hash_data.hierarchical_sha256_data; + sha.hash_block_size = 0x10000; + REQUIRE(nca.Open() == ResultSuccess); + sha.hash_block_size = 0x40000000; + REQUIRE(nca.Open() == ResultSuccess); +} + +TEST_CASE("NCA rejects a SHA-256 hash layer larger than one block", "[core][file_sys]") { + auto nca = MakeSha256Nca(); + nca.fs.hash_data.hierarchical_sha256_data.hash_layer_region[0].size = 0x2000; + REQUIRE(nca.Open() == ResultInvalidHierarchicalSha256BlockSize); +} + +TEST_CASE("NCA rejects a SHA-256 hash region past the section before allocating it", + "[core][file_sys]") { + auto nca = MakeSha256Nca(); + nca.fs.hash_data.hierarchical_sha256_data.hash_layer_region[0].size = s64{1} << 60; + REQUIRE(nca.Open() == ResultNcaBaseStorageOutOfRangeC); +} + +PlaintextNca MakeIntegrityNca() { + PlaintextNca nca; + nca.fs.hash_type = NcaFsHeader::HashType::HierarchicalIntegrityHash; + auto& levels = nca.fs.hash_data.integrity_meta_info.level_hash_info; + levels.max_layers = 7; + for (size_t i = 0; i < levels.info.size(); ++i) { + levels.info[i].offset = static_cast(i) * 0x200; + levels.info[i].size = 0x200; + levels.info[i].block_order = 9; + } + return nca; +} + +TEST_CASE("NCA with a valid integrity section opens", "[core][file_sys]") { + REQUIRE(MakeIntegrityNca().Open() == ResultSuccess); +} + +TEST_CASE("NCA rejects an integrity block order a shift cannot hold", "[core][file_sys]") { + auto nca = MakeIntegrityNca(); + nca.fs.hash_data.integrity_meta_info.level_hash_info.info[2].block_order = 200; + REQUIRE(nca.Open() == ResultInvalidNcaHierarchicalIntegrityVerificationLayerCount); +} + +TEST_CASE("NCA rejects an integrity tree with fewer than three layers", "[core][file_sys]") { + auto nca = MakeIntegrityNca(); + nca.fs.hash_data.integrity_meta_info.level_hash_info.max_layers = 2; + REQUIRE(nca.Open() == ResultInvalidNcaHierarchicalIntegrityVerificationLayerCount); +} + +TEST_CASE("NCA rejects a layer count outside the hash target table", "[core][file_sys]") { + auto nca = MakeIntegrityNca(); + nca.fs.encryption_type = NcaFsHeader::EncryptionType::AesCtrSkipLayerHash; + nca.fs.hash_data.integrity_meta_info.level_hash_info.max_layers = 0; + REQUIRE(nca.Open() == ResultInvalidNcaFsHeader); +} + +TEST_CASE("NCA rejects a negative indirect table size", "[core][file_sys]") { + auto nca = MakeSha256Nca(); + nca.fs.patch_info.indirect_offset = 0x200; + nca.fs.patch_info.indirect_size = -0x100; + REQUIRE(nca.Open() == ResultNcaBaseStorageOutOfRangeE); +} + +TEST_CASE("NCA rejects an indirect table larger than the file behind a sparse layer", + "[core][file_sys]") { + auto nca = MakeSha256Nca(); + nca.header.fs_info[0].end_sector = 0xFFFFFF06; + nca.fs.sparse_info.generation = 1; + BucketTree::Header sparse_header; + sparse_header.Format(0); + std::memcpy(nca.fs.sparse_info.bucket.header.data(), &sparse_header, sizeof(sparse_header)); + nca.fs.patch_info.indirect_offset = 0; + nca.fs.patch_info.indirect_size = s64{1} << 36; + REQUIRE(nca.Open() == ResultNcaBaseStorageOutOfRangeE); +} + +// A 0x10000-byte section: 0x8000 bytes of patch data, then an indirect table mapping the virtual +// range [0, end) onto them. +PlaintextNca MakeIndirectNca(s64 end) { + auto nca = MakeSha256Nca(); + nca.header.fs_info[0].end_sector = 6 + 0x10000 / 0x200; + nca.section.resize(0x10000); + const BucketTree::NodeHeader node{0, 1, end}; + Put(nca.section, 0x8000, node); + Put(nca.section, 0x8000 + sizeof(node), s64{0}); + Put(nca.section, 0xC000, node); + IndirectStorage::Entry entry{}; + entry.storage_index = 1; + Put(nca.section, 0xC000 + sizeof(node), entry); + BucketTree::Header header; + header.Format(1); + nca.fs.patch_info.indirect_offset = 0x8000; + nca.fs.patch_info.indirect_size = 0x8000; + std::memcpy(nca.fs.patch_info.indirect_header.data(), &header, sizeof(header)); + return nca; +} + +TEST_CASE("NCA with a valid indirect table opens", "[core][file_sys]") { + REQUIRE(MakeIndirectNca(0x8000).Open() == ResultSuccess); +} + +TEST_CASE("NCA rejects a hash region larger than the file behind an indirect layer", + "[core][file_sys]") { + auto nca = MakeIndirectNca(s64{1} << 40); + auto& sha = nca.fs.hash_data.hierarchical_sha256_data; + sha.hash_layer_region[0].size = s64{1} << 36; + sha.hash_layer_region[1].offset = s64{1} << 37; + REQUIRE(nca.Open() == ResultNcaBaseStorageOutOfRangeC); +} + +// A card image whose root HFS0 holds empty partitions with the given names. +std::vector MakeXci(const std::vector& partitions) { + std::vector partition; + Put(partition, 0, Common::MakeMagic('H', 'F', 'S', '0')); + Put(partition, 8, u32{0}); + + std::string strtab; + std::vector root; + Put(root, 0, Common::MakeMagic('H', 'F', 'S', '0')); + Put(root, 4, static_cast(partitions.size())); + root.resize(0x10 + partitions.size() * 0x40); + for (size_t i = 0; i < partitions.size(); ++i) { + Put(root, 0x10 + i * 0x40, static_cast(i * partition.size())); + Put(root, 0x10 + i * 0x40 + 8, static_cast(partition.size())); + Put(root, 0x10 + i * 0x40 + 0x10, static_cast(strtab.size())); + strtab += partitions[i]; + strtab += '\0'; + } + Put(root, 8, static_cast(strtab.size())); + root.insert(root.end(), strtab.begin(), strtab.end()); + for (size_t i = 0; i < partitions.size(); ++i) { + root.insert(root.end(), partition.begin(), partition.end()); + } + + std::vector out(0x200); + Put(out, 0x100, Common::MakeMagic('H', 'E', 'A', 'D')); + Put(out, 0x130, u64{0x200}); + Put(out, 0x138, static_cast(root.size())); + out.insert(out.end(), root.begin(), root.end()); + return out; +} + +TEST_CASE("XCI with a secure partition parses", "[core][file_sys]") { + XCI xci{MakeFile(MakeXci({"update", "normal", "secure"}))}; + REQUIRE(xci.GetStatus() == Loader::ResultStatus::Success); +} + +TEST_CASE("XCI rejects a card without a secure partition", "[core][file_sys]") { + XCI xci{MakeFile(MakeXci({"update", "normal"}))}; + REQUIRE(xci.GetStatus() == Loader::ResultStatus::ErrorXCIMissingPartition); +} + +TEST_CASE("XCI rejects an HFS0 offset past the end of the image", "[core][file_sys]") { + auto bytes = MakeXci({"update", "normal", "secure"}); + Put(bytes, 0x130, u64{0x10'0000}); + XCI xci{MakeFile(std::move(bytes))}; + REQUIRE(xci.GetStatus() == Loader::ResultStatus::ErrorBadXCIHeader); +} + +TEST_CASE("Integrity storage zeroes only the requested bytes past the data", + "[core][file_sys]") { + IntegrityVerificationStorage storage; + storage.Initialize(MakeFile(std::vector(0x20)), MakeFile(std::vector(100, 0x11)), + 0x200, IntegrityVerificationStorage::HashSize, true); + + std::vector buffer(48, 0xAA); + REQUIRE(storage.Read(buffer.data(), 32, 80) == 20); + REQUIRE(buffer[19] == 0x11); + REQUIRE(buffer[20] == 0); + REQUIRE(buffer[31] == 0); + REQUIRE(buffer[32] == 0xAA); + REQUIRE(buffer[47] == 0xAA); + + REQUIRE(storage.Read(buffer.data(), 1, 0x1000) == 0); +} + +} // namespace diff --git a/src/tests/core/vfs_vector.cpp b/src/tests/core/vfs_vector.cpp new file mode 100644 index 0000000000..c51a0360eb --- /dev/null +++ b/src/tests/core/vfs_vector.cpp @@ -0,0 +1,25 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later + +#include +#include + +#include + +#include "core/file_sys/vfs/vfs_vector.h" + +namespace { +TEST_CASE("VectorVfsFile reads stop at the end of the data", "[core][vfs]") { + const FileSys::VectorVfsFile file(std::vector{1, 2, 3, 4}); + std::array out{}; + + REQUIRE(file.Read(out.data(), out.size(), 2) == 2); + REQUIRE(out[0] == 3); + REQUIRE(out[1] == 4); + REQUIRE(file.Read(out.data(), out.size(), 4) == 0); + REQUIRE(file.Read(out.data(), out.size(), 100) == 0); + + const FileSys::VectorVfsFile empty; + REQUIRE(empty.Read(out.data(), out.size(), 0) == 0); +} +} // namespace From 0fb6b9b7d2ac268b928f54113d80280760679e41 Mon Sep 17 00:00:00 2001 From: Zortos Date: Mon, 9 Nov 2026 08:43:19 +0100 Subject: [PATCH 089/105] common: trim all-zero guest pages unconditionally (builds on #431) (#961) * common: trim all-zero guest pages from the working set on Windows (EDEN_NX2_TRIM_ZERO) Windows can't punch holes in the pagefile-backed guest memory section, so pages that only ever hold zeros stay resident. Ocarina of Time boots with ~7 GB of them, leaving a 16 GB PC with 0.2-0.6 GB free. HostMemory::TrimZeroPages takes those pages out of the working set (VirtualUnlock, like HintCold); an nx2:ZeroTrim thread runs it every EDEN_NX2_TRIM_ZERO seconds for the first 2 minutes, then every EDEN_NX2_TRIM_ZERO_LATER (default 60). Off by default. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit f22bbd0d07a605c440e0103ba6ea1dd1530882ce) * common: trim all-zero guest pages unconditionally and only read them through the backing The zero-page trim runs for every title, without EDEN_NX2_TRIM_ZERO. Its thread moves from the nx2:gpu session to DeviceMemory, which lives for the whole app, so a fixed boot window no longer fits: a pass comes every 5 s while it trims 64 MiB or more and backs off to once a minute. Pages are checked through the backing view only. It stays mapped and writable, while a guest view page can be unmapped or protected during a pass; a view page's backing offset comes from the placeholder map under its lock. This also drops __try, which MinGW can't build. A backing page the check pulled into the working set leaves it again, and a run spanning two views, which VirtualUnlock refuses whole, is trimmed page by page. Trimming keeps the contents, so a guest or GPU write landing after the check costs a soft fault. Discarding (DiscardVirtualMemory, MEM_RESET, punching holes) would lose it: neither OS can check and discard a shared section page in one step. Co-authored-by: Ghostwolf13 <353235520+Ghostwolf13@users.noreply.github.com> --------- Co-authored-by: Ghostwolf13 <353235520+Ghostwolf13@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 (cherry picked from commit 2b30c2cb1ceb8f9a6e385b7cea45705dca07b722) --- src/common/host_memory.cpp | 117 +++++++++++++++++++++++++++++++ src/common/host_memory.h | 8 +++ src/core/device_memory.cpp | 54 +++++++++++++- src/core/device_memory.h | 11 +++ src/tests/common/host_memory.cpp | 54 +++++++++++++- 5 files changed, 242 insertions(+), 2 deletions(-) diff --git a/src/common/host_memory.cpp b/src/common/host_memory.cpp index 58e3e3fe26..746a921e33 100644 --- a/src/common/host_memory.cpp +++ b/src/common/host_memory.cpp @@ -10,6 +10,8 @@ #include "common/container/unordered_map.h" #include #include +#include +#include #include "common/dynamic_library.h" #else // ^^^ Windows ^^^ vvv POSIX vvv @@ -276,6 +278,20 @@ class HostMemory::Impl { } } + /// Backing offset the view page at virtual_offset maps, or nullopt when it is not mapped. + std::optional BackingOffset(size_t virtual_offset) { + std::scoped_lock lock{placeholder_mutex}; + const auto it = placeholders.find(virtual_offset); + if (it == placeholders.end()) { + return std::nullopt; + } + const auto host_pointer_it = placeholder_host_pointers.find(it->lower()); + if (host_pointer_it == placeholder_host_pointers.end()) { + return std::nullopt; + } + return host_pointer_it->second + virtual_offset - it->lower(); + } + void EnableDirectMappedAddress() { // TODO UNREACHABLE(); @@ -1102,6 +1118,106 @@ void HostMemory::HintCold(size_t physical_offset, size_t length, } } +namespace { +std::atomic g_zero_filled_bytes{}; + +[[maybe_unused]] bool IsZeroPage(const u8* page) { + const auto* words = reinterpret_cast(page); + u64 bits = 0; + for (size_t i = 0; i < PageAlignment / sizeof(u64); ++i) { + bits |= words[i]; + } + return bits == 0; +} +} // namespace + +u64 HostMemory::ZeroFilledBytes() noexcept { + return g_zero_filled_bytes.load(std::memory_order_relaxed); +} + +std::pair HostMemory::TrimZeroPages() const { +#ifdef _WIN32 + if (backing_base == nullptr) { + return {0, 0}; + } + std::vector entries(size_t{1} << 22); + for (int attempt = 0;; ++attempt) { + if (K32QueryWorkingSet(GetCurrentProcess(), entries.data(), + static_cast(entries.size() * sizeof(ULONG_PTR)))) { + break; + } + if (GetLastError() != ERROR_BAD_LENGTH || attempt == 3) { + return {0, 0}; + } + entries.assign((std::max)(entries.size() * 2, size_t{entries[0]} + (size_t{1} << 18)), 0); + } + const size_t count = (std::min)(size_t{entries[0]}, entries.size() - 1); + const auto backing_begin = reinterpret_cast(backing_base); + const auto view_begin = reinterpret_cast(impl ? impl->virtual_base : nullptr); + const size_t view_size = impl ? impl->virtual_size : 0; + std::vector backing_pages; + std::vector view_pages; + for (size_t i = 1; i <= count; ++i) { + const uintptr_t page = entries[i] & ~uintptr_t{PageAlignment - 1}; + if (page - backing_begin < backing_size) { + backing_pages.push_back(page); + } else if (view_begin != 0 && page - view_begin < view_size) { + view_pages.push_back(page); + } + } + std::sort(backing_pages.begin(), backing_pages.end()); + // Pages are only ever read through the backing, which stays mapped and writable: a view page + // can be unmapped or protected by a guest thread meanwhile. Trimming keeps the contents, so a + // write that lands after the check costs a soft fault and nothing else. + std::vector trim; + size_t zero_bytes = 0; + for (const uintptr_t page : backing_pages) { + if (IsZeroPage(reinterpret_cast(page))) { + trim.push_back(page); + zero_bytes += PageAlignment; + } + } + for (const uintptr_t page : view_pages) { + const auto offset = impl->BackingOffset(page - view_begin); + if (!offset || *offset >= backing_size) { + continue; + } + const uintptr_t backing_page = backing_begin + *offset; + if (IsZeroPage(reinterpret_cast(backing_page))) { + trim.push_back(page); + zero_bytes += PageAlignment; + } + // Reading it put the backing page in the working set: take it out again. + if (!std::binary_search(backing_pages.begin(), backing_pages.end(), backing_page)) { + trim.push_back(backing_page); + } + } + std::sort(trim.begin(), trim.end()); + trim.erase(std::unique(trim.begin(), trim.end()), trim.end()); + // A run can span two views, which VirtualUnlock refuses as a whole: then page by page. + const auto unlock = [](uintptr_t begin, uintptr_t end) { + if (VirtualUnlock(reinterpret_cast(begin), end - begin) || + GetLastError() == ERROR_NOT_LOCKED) { + return; + } + for (uintptr_t page = begin; page < end; page += PageAlignment) { + VirtualUnlock(reinterpret_cast(page), PageAlignment); + } + }; + for (size_t i = 0; i < trim.size();) { + size_t j = i + 1; + while (j < trim.size() && trim[j] == trim[j - 1] + PageAlignment) { + ++j; + } + unlock(trim[i], trim[j - 1] + PageAlignment); + i = j; + } + return {(backing_pages.size() + view_pages.size()) * PageAlignment, zero_bytes}; +#else + return {0, 0}; +#endif +} + void HostMemory::ClearBackingRegion(size_t physical_offset, size_t length, u32 fill_value) { if (length == 0) { return; @@ -1136,6 +1252,7 @@ void HostMemory::ClearBackingRegion(size_t physical_offset, size_t length, u32 f #endif if (!cleared) { std::memset(backing_base + offset, 0, clear_end - offset); + g_zero_filled_bytes.fetch_add(clear_end - offset, std::memory_order_relaxed); } if (whole) { granules[index].fetch_and(static_cast(~GranuleDirty), std::memory_order_release); diff --git a/src/common/host_memory.h b/src/common/host_memory.h index 74a8398e64..647f795de1 100644 --- a/src/common/host_memory.h +++ b/src/common/host_memory.h @@ -9,6 +9,7 @@ #include #include #include +#include #include "common/common_funcs.h" #include "common/common_types.h" @@ -62,6 +63,13 @@ class HostMemory { /// it needs RAM. The contents stay; using the pages brings them back. void HintCold(size_t physical_offset, size_t length, std::optional virtual_offset) const; + /// Windows: resident pages of guest memory (backing and its views) that hold only zeros leave + /// the working set, like HintCold. Returns {resident bytes looked at, bytes trimmed}. + std::pair TrimZeroPages() const; + + /// Bytes ClearBackingRegion has filled with zeros (memset) since start. + static u64 ZeroFilledBytes() noexcept; + /// Pages the kernel hands out (cleared or not). void MarkUsed(size_t physical_offset, size_t length); diff --git a/src/core/device_memory.cpp b/src/core/device_memory.cpp index 1aea56a991..e82da5e674 100644 --- a/src/core/device_memory.cpp +++ b/src/core/device_memory.cpp @@ -1,6 +1,16 @@ // SPDX-FileCopyrightText: Copyright 2020 yuzu Emulator Project // SPDX-License-Identifier: GPL-2.0-or-later +#ifdef _WIN32 +#include +#include +#include +#include + +#include "common/logging.h" +#include "common/thread.h" +#endif + #include "core/device_memory.h" #include "hle/kernel/board/nintendo/nx/k_system_control.h" @@ -14,8 +24,50 @@ constexpr size_t VirtualReserveSize = 1ULL << 39; DeviceMemory::DeviceMemory() : buffer{Kernel::Board::Nintendo::Nx::KSystemControl::Init::GetIntendedMemorySize(), - VirtualReserveSize} {} + VirtualReserveSize} { +#ifdef _WIN32 + zero_trim_thread = std::jthread{[this](std::stop_token stop) { TrimZeroPages(stop); }}; +#endif +} DeviceMemory::~DeviceMemory() = default; +#ifdef _WIN32 +// Windows can't punch holes in the guest memory section, so pages holding only zeros (games clear +// their heaps while booting: Ocarina leaves ~7 GB) stay resident until they leave the working set. +// A pass comes every 5 s while they keep turning up, backing off to once a minute. +void DeviceMemory::TrimZeroPages(std::stop_token stop) { + using namespace std::chrono_literals; + Common::SetCurrentThreadName("ZeroTrim"); + Common::SetCurrentThreadPriority(Common::ThreadPriority::Low); + constexpr size_t Busy = size_t{64} << 20; + std::mutex mutex; + std::condition_variable_any wake; + std::chrono::seconds interval = 5s; + while (true) { + { + std::unique_lock lock{mutex}; + wake.wait_for(lock, stop, interval, [] { return false; }); + } + if (stop.stop_requested()) { + return; + } + const auto start = std::chrono::steady_clock::now(); + const auto [resident, trimmed] = buffer.TrimZeroPages(); + const auto ms = std::chrono::duration_cast( + std::chrono::steady_clock::now() - start) + .count(); + if (trimmed < Busy) { + interval = (std::min)(interval * 2, std::chrono::seconds{60}); + continue; + } + interval = 5s; + LOG_INFO(HW_Memory, + "Trimmed {} MiB of all-zero guest pages out of {} MiB resident in {} ms; the kernel " + "zero-filled {} MiB so far", + trimmed >> 20, resident >> 20, ms, Common::HostMemory::ZeroFilledBytes() >> 20); + } +} +#endif + } // namespace Core diff --git a/src/core/device_memory.h b/src/core/device_memory.h index b5103e23bc..41d39c2753 100644 --- a/src/core/device_memory.h +++ b/src/core/device_memory.h @@ -6,6 +6,10 @@ #pragma once +#ifdef _WIN32 +#include +#endif + #include "common/host_memory.h" #include "common/typed_address.h" @@ -66,6 +70,13 @@ class DeviceMemory { } Common::HostMemory buffer; + +#ifdef _WIN32 +private: + void TrimZeroPages(std::stop_token stop); + + std::jthread zero_trim_thread; +#endif }; } // namespace Core diff --git a/src/tests/common/host_memory.cpp b/src/tests/common/host_memory.cpp index 2f0f79f4a5..4915e941dc 100644 --- a/src/tests/common/host_memory.cpp +++ b/src/tests/common/host_memory.cpp @@ -4,9 +4,15 @@ // SPDX-FileCopyrightText: Copyright 2021 yuzu Emulator Project // SPDX-License-Identifier: GPL-2.0-or-later +#include +#include + #include -#ifndef _WIN32 +#ifdef _WIN32 +#include +#include +#else #include #endif @@ -256,3 +262,49 @@ TEST_CASE("HostMemory: Partial sparse middle unmap and check bindings", "[common REQUIRE(ptr[0x0000] == 19); REQUIRE(ptr[0x3fff] == 12); } + +#ifdef _WIN32 +TEST_CASE("HostMemory: Trim zero pages keeps contents", "[common]") { + constexpr size_t Page = 0x1000; + HostMemory mem(BACKING_SIZE, VIRTUAL_SIZE); + REQUIRE(mem.BackingBasePointer() != nullptr); + mem.Map(0x10000, 0x100000, 2 * Page, PERMS, HEAP); + // Two views next to each other: one run that VirtualUnlock can't take as a whole. + mem.Map(0x12000, 0x200000, Page, PERMS, HEAP); + mem.Map(0x13000, 0x300000, Page, PERMS, HEAP); + u8* const view = mem.VirtualBasePointer() + 0x10000; + u8* const backing = mem.BackingBasePointer(); + std::memset(view, 0, Page); + std::memset(view + Page, 0x5a, Page); + std::memset(view + 2 * Page, 0, 2 * Page); + std::memset(backing + 0x400000, 0, Page); + + const auto resident = [](const void* address) { + PSAPI_WORKING_SET_EX_INFORMATION info{}; + info.VirtualAddress = const_cast(address); + REQUIRE(QueryWorkingSetEx(GetCurrentProcess(), &info, sizeof(info))); + return info.VirtualAttributes.Valid != 0; + }; + REQUIRE(resident(view)); + REQUIRE(!resident(backing + 0x100000 + Page)); + + const auto [looked_at, trimmed] = mem.TrimZeroPages(); + REQUIRE(looked_at >= 5 * Page); + REQUIRE(trimmed >= 4 * Page); + REQUIRE(!resident(view)); + REQUIRE(!resident(view + 2 * Page)); + REQUIRE(!resident(view + 3 * Page)); + REQUIRE(!resident(backing + 0x400000)); + REQUIRE(resident(view + Page)); + REQUIRE(!resident(backing + 0x100000 + Page)); + + const auto all = [](const u8* page, u8 value) { + return std::all_of(page, page + Page, [value](u8 byte) { return byte == value; }); + }; + REQUIRE(all(view, 0)); + REQUIRE(all(view + Page, 0x5a)); + REQUIRE(all(view + 2 * Page, 0)); + REQUIRE(all(view + 3 * Page, 0)); + REQUIRE(all(backing + 0x400000, 0)); +} +#endif From 8c7aad93cc634a5ec0c9c560c8a9cc3947116aef Mon Sep 17 00:00:00 2001 From: the-cashman <55607117+the-cashman@users.noreply.github.com> Date: Sun, 8 Nov 2026 14:53:48 -0500 Subject: [PATCH 090/105] buffer_cache: skip the once-a-frame native bank upload for Ocarina of Time The upload (a Splatoon Raiders fix for bank writes the memory tracker missed) copies every native 64 KiB uniform bank in full each frame. In Ocarina of Time a compare of the banks against shadow copies found no page changed behind the tracker, and skipping the upload is ~1.5 fps faster with the same screenshots. Every other title keeps the upload, and EDEN_NX2_PERF_VERIFY=1 keeps it for Ocarina too. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 913943a751eaab9327c6db96b4982b227f124450) --- src/video_core/buffer_cache/buffer_cache.h | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/video_core/buffer_cache/buffer_cache.h b/src/video_core/buffer_cache/buffer_cache.h index c926a968ca..3e9a5fe523 100644 --- a/src/video_core/buffer_cache/buffer_cache.h +++ b/src/video_core/buffer_cache/buffer_cache.h @@ -2780,7 +2780,12 @@ void BufferCache

::BindHostGraphicsUniformBuffer(size_t stage, u32 index, u32 // Native programs keep tables next to their code in a 64 KiB bank read with indirect offsets // (Splatoon Raiders: the dialogue box corners in c1). Writes to that memory can go unnoticed by // the memory tracker, leaving stale host contents, so upload such a bank again once per frame. - if (has_host_buffer && (*channel_state->uniform_buffer_sizes)[stage][index] == 0x10000 && + // Ocarina of Time is left out (unless EDEN_NX2_PERF_VERIFY=1): a compare of its banks against + // shadow copies found no page changed behind the tracker, and the uploads cost frame rate. + constexpr u64 ocarina_of_time = 0x0400486024DEC000ULL; + const bool bank_recheck = program_id != ocarina_of_time || Nx2Perf::Verify(); + if (bank_recheck && has_host_buffer && + (*channel_state->uniform_buffer_sizes)[stage][index] == 0x10000 && size != 0 && gpu_memory != nullptr && gpu_memory->UsesNativeTiling() && (Nx2Perf::Verify() || !native_indirect_uniforms_this_frame.contains(device_addr)) && !memory_tracker.IsRegionGpuModified(device_addr, size) && From 42dcc249aec556b7201bd6fdf2fb6bb31803c4e0 Mon Sep 17 00:00:00 2001 From: Robert Veloso Date: Sun, 8 Nov 2026 19:17:11 -0300 Subject: [PATCH 091/105] buffer_cache: keep the resolved index buffer while its range, mappings and buffers are The draw macros mark the index buffer dirty on every draw, so every indexed draw translated its address and looked its buffer up again, mostly for the same buffer with another index count. The address and buffer found are kept per range while the GPU mappings (MapSerial) and the buffers (buffer_deletions) are, for sizes up to the one the buffer was found for. EDEN_NX2_PERF_VERIFY resolves anyway and logs a mismatch. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 12262ce3650ac04ec0c4555a0a5a1a00dcbbe92a) (cherry picked from commit 6387f2c09d39559d4b71413f1213bf847542454f) --- src/video_core/buffer_cache/buffer_cache.h | 49 +++++++++++++++++-- .../buffer_cache/buffer_cache_base.h | 14 ++++++ 2 files changed, 60 insertions(+), 3 deletions(-) diff --git a/src/video_core/buffer_cache/buffer_cache.h b/src/video_core/buffer_cache/buffer_cache.h index 3e9a5fe523..43b0fa6912 100644 --- a/src/video_core/buffer_cache/buffer_cache.h +++ b/src/video_core/buffer_cache/buffer_cache.h @@ -3280,18 +3280,61 @@ void BufferCache

::UpdateIndexBuffer() { const GPUVAddr gpu_addr_begin = index_buffer_ref.StartAddress(); const GPUVAddr gpu_addr_end = index_buffer_ref.EndAddress(); - const std::optional device_addr = gpu_memory->GpuToCpuAddress(gpu_addr_begin); - const u32 address_size = static_cast(gpu_addr_end - gpu_addr_begin); const u32 draw_size = (index_buffer_ref.count + index_buffer_ref.first) * u32(index_buffer_ref.FormatSizeInBytes()); + // The draw macros mark the index buffer dirty on every draw, mostly with the same buffer and + // another count: the address and the buffer found are kept while the range, the GPU mappings + // and the buffers are, for any size up to the one the buffer was found for. + IndexBufferMemo& memo = index_buffer_memo; + const u64 map_serial = gpu_memory->MapSerial(); + const u32 address_size = static_cast(gpu_addr_end - gpu_addr_begin); const u32 size = (std::min)(address_size, draw_size); + const bool memo_hit = memo.channel == channel_state && memo.memory == gpu_memory && + memo.begin == gpu_addr_begin && memo.end == gpu_addr_end && + memo.map_serial == map_serial && memo.deletions == buffer_deletions && + size != 0 && size <= memo.found_size; + if (memo_hit && !Nx2Perf::Verify()) { + channel_state->index_buffer = Binding{ + .device_addr = memo.device_addr, + .size = size, + .buffer_id = memo.buffer_id, + }; + return; + } + const std::optional device_addr = gpu_memory->GpuToCpuAddress(gpu_addr_begin); if (size == 0 || !device_addr) { channel_state->index_buffer = NULL_BINDING; return; } + const BufferId buffer_id = FindBuffer(*device_addr, size, false); channel_state->index_buffer = Binding{ .device_addr = *device_addr, .size = size, - .buffer_id = FindBuffer(*device_addr, size, false), + .buffer_id = buffer_id, + }; + if (memo_hit) { + if (memo.device_addr != *device_addr || memo.buffer_id != buffer_id) { + static std::atomic reports{}; + if (reports.fetch_add(1, std::memory_order_relaxed) < 16) { + LOG_ERROR(HW_GPU, + "Nx2Perf index buffer memo mismatch: {:#x}+{:#x} kept {:#x} buffer {}, " + "found {:#x} buffer {}", + gpu_addr_begin, size, memo.device_addr, memo.buffer_id.index, + *device_addr, buffer_id.index); + } + } + return; + } + // FindBuffer may have deleted buffers (joins): the deletion count is read after it. + memo = IndexBufferMemo{ + .channel = channel_state, + .memory = gpu_memory, + .begin = gpu_addr_begin, + .end = gpu_addr_end, + .map_serial = map_serial, + .deletions = buffer_deletions, + .device_addr = *device_addr, + .found_size = size, + .buffer_id = buffer_id, }; } diff --git a/src/video_core/buffer_cache/buffer_cache_base.h b/src/video_core/buffer_cache/buffer_cache_base.h index b63daeda9e..1ef853554b 100644 --- a/src/video_core/buffer_cache/buffer_cache_base.h +++ b/src/video_core/buffer_cache/buffer_cache_base.h @@ -762,6 +762,20 @@ class BufferCache : public VideoCommon::ChannelSetupCaches geometry_syncs{}; /// Counts DeleteBuffer calls: a buffer id older than this count may have been freed or reused. u64 buffer_deletions = 0; + + /// The last index buffer UpdateIndexBuffer resolved, and what it depends on. + struct IndexBufferMemo { + const void* channel{}; + const void* memory{}; + GPUVAddr begin{}; + GPUVAddr end{}; + u64 map_serial{~u64{0}}; + u64 deletions{}; + DAddr device_addr{}; + u32 found_size{}; ///< The size FindBuffer was asked for: the buffer holds at least this + BufferId buffer_id{}; + }; + IndexBufferMemo index_buffer_memo; struct GeometryStats { u64 syncs{}; u64 syncs_skipped{}; From 088f53b97e458668328641dfeac4e82ad8926970 Mon Sep 17 00:00:00 2001 From: Robert Veloso Date: Sun, 8 Nov 2026 19:30:21 -0300 Subject: [PATCH 092/105] buffer_cache: no index buffer memo with accurate GPU fences FindBuffer waits for a buffer's pending writes when GPU fence behaviour is Accurate; a memo hit would skip that wait. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit a95eba8a33eede7ac36ef3b22d362144e55c92ca) (cherry picked from commit eef399b13eb4bddd9be37c03c243b4f31f3994af) --- src/video_core/buffer_cache/buffer_cache.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/video_core/buffer_cache/buffer_cache.h b/src/video_core/buffer_cache/buffer_cache.h index 43b0fa6912..7db04dcb21 100644 --- a/src/video_core/buffer_cache/buffer_cache.h +++ b/src/video_core/buffer_cache/buffer_cache.h @@ -3291,7 +3291,8 @@ void BufferCache

::UpdateIndexBuffer() { const bool memo_hit = memo.channel == channel_state && memo.memory == gpu_memory && memo.begin == gpu_addr_begin && memo.end == gpu_addr_end && memo.map_serial == map_serial && memo.deletions == buffer_deletions && - size != 0 && size <= memo.found_size; + size != 0 && size <= memo.found_size && + !Settings::IsGPUFenceBehaviorAccurate(); // FindBuffer waits for writes if (memo_hit && !Nx2Perf::Verify()) { channel_state->index_buffer = Binding{ .device_addr = memo.device_addr, From 139d936e590a71394bf47f330d95d0dbd402341d Mon Sep 17 00:00:00 2001 From: El-Sashok Date: Mon, 9 Nov 2026 03:04:24 +0100 Subject: [PATCH 093/105] video_core: read texture descriptors without flushing during shader translation The flush could finish the whole GPU queue from the middle of a translation, stalling the GPU thread for seconds. (cherry picked from commit 591ceee8b15cef42391679bd2a2b3b2a96a4acd8) --- src/video_core/shader_environment.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/video_core/shader_environment.cpp b/src/video_core/shader_environment.cpp index c97c6e608e..8442092759 100644 --- a/src/video_core/shader_environment.cpp +++ b/src/video_core/shader_environment.cpp @@ -495,7 +495,7 @@ Tegra::Texture::TICEntry GenericEnvironment::ReadTextureInfo(GPUVAddr tic_addr, return entry; } const GPUVAddr descriptor_addr{tic_addr + index * sizeof(Tegra::Texture::TICEntry)}; - gpu_memory->ReadBlock(descriptor_addr, &entry, sizeof(entry)); + gpu_memory->ReadBlockUnsafe(descriptor_addr, &entry, sizeof(entry)); if (entry.format.Value() == Tegra::Texture::TextureFormat{} && std::getenv("EDEN_NX2_RENDER_TRACE")) { static std::atomic traces{}; From e64b40713f406b6e22596380939269004a879c4a Mon Sep 17 00:00:00 2001 From: Jaramie Morris Date: Sun, 8 Nov 2026 20:34:44 -0600 Subject: [PATCH 094/105] common: keep a faulting stack walk from taking the sampler's process down RtlVirtualUnwind also reads through the frame register, which the walk's stack-pointer check doesn't cover. Sampling CPUCore_* threads (EDEN_NX2_SAMPLE), a thread paused in a fiber switch (KScheduler::EnableScheduling) with Rbp 0 made it read address 0, and the access violation crashed Lichii. The walk now runs under SEH: a fault stops it and keeps the frames found so far. The report counts samples without a stack and walks stopped by a fault per thread. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 7a755fbc2c6a3fb040d9031649d98fde434bf34b) --- src/common/nx2_sampler.cpp | 43 ++++++++++++++++++++++++++++++++------ 1 file changed, 37 insertions(+), 6 deletions(-) diff --git a/src/common/nx2_sampler.cpp b/src/common/nx2_sampler.cpp index 244543709e..31c1ffabe4 100644 --- a/src/common/nx2_sampler.cpp +++ b/src/common/nx2_sampler.cpp @@ -34,6 +34,7 @@ constexpr size_t MaxDepth = 40; struct Sample { unsigned thread; unsigned depth; + bool faulted; ///< the walk stopped at an access violation std::array frames; }; @@ -146,15 +147,15 @@ std::vector FindThreads(const std::vector& patterns) { // could be paused holding the heap lock. A wrong unwind step (code with missing or wrong unwind // data) leaves a garbage stack pointer: reading through it crashed the process, so the walk // stops once the stack pointer leaves the thread's stack. -unsigned Walk(CONTEXT context, std::array& frames, const NT_TIB* tib) { +void Walk(CONTEXT context, std::array& frames, const NT_TIB* tib, + unsigned& depth) { if (tib == nullptr) { - return 0; + return; } const auto on_stack = [tib](DWORD64 rsp) { return rsp >= reinterpret_cast(tib->StackLimit) && rsp + 8 <= reinterpret_cast(tib->StackBase) && (rsp & 7) == 0; }; - unsigned depth = 0; while (depth < MaxDepth && context.Rip != 0 && on_stack(context.Rsp)) { frames[depth++] = context.Rip; DWORD64 image_base{}; @@ -174,6 +175,26 @@ unsigned Walk(CONTEXT context, std::array& frames, const NT_T RtlVirtualUnwind(UNW_FLAG_NHANDLER, image_base, context.Rip, function, &context, &handler_data, &establisher_frame, nullptr); } +} + +// RtlVirtualUnwind also reads through the frame register, which the stack check above doesn't +// cover: a thread paused in a fiber switch with Rbp 0 made it read address 0, and the access +// violation took the process down (EDEN_NX2_SAMPLE=CPUCore_*). Such a walk keeps the frames it +// found. +unsigned GuardedWalk(const CONTEXT& context, std::array& frames, + const NT_TIB* tib, bool& faulted) { + unsigned depth = 0; + faulted = false; +#ifdef _MSC_VER + __try { + Walk(context, frames, tib, depth); + } __except (GetExceptionCode() == EXCEPTION_ACCESS_VIOLATION ? EXCEPTION_EXECUTE_HANDLER + : EXCEPTION_CONTINUE_SEARCH) { + faulted = true; + } +#else + Walk(context, frames, tib, depth); +#endif return depth; } @@ -257,8 +278,15 @@ std::string Report(const std::vector& targets, const std::vector std::map inclusive; std::map self_caller; unsigned total = 0; + unsigned faulted = 0; + unsigned empty = 0; for (const Sample& sample : samples) { - if (sample.thread != thread || sample.depth == 0) { + if (sample.thread != thread) { + continue; + } + faulted += sample.faulted ? 1 : 0; + if (sample.depth == 0) { + ++empty; continue; } ++total; @@ -289,7 +317,9 @@ std::string Report(const std::vector& targets, const std::vector sorted[i].second.c_str()); } }; - text += fmt::format("=== thread \"{}\": {} samples\n", targets[thread].name, total); + text += fmt::format("=== thread \"{}\": {} samples ({} without a stack, {} walks stopped " + "by a fault)\n", + targets[thread].name, total, empty, faulted); print("self (where the thread is)", self, 60); // The hottest instructions as module offsets, for llvm-symbolizer --relative-address // (which line of a function, inlined code included). @@ -371,7 +401,8 @@ std::string SampleThreads(const std::vector& patterns, double secon if (GetThreadContext(handle, &context)) { Sample& sample = samples.emplace_back(); sample.thread = i; - sample.depth = Walk(context, sample.frames, targets[i].tib); + sample.depth = + GuardedWalk(context, sample.frames, targets[i].tib, sample.faulted); } ResumeThread(handle); } From cccac7bb7b55d68676a016d99462360eaf5ca07f Mon Sep 17 00:00:00 2001 From: Julius Bairaktaris Date: Sun, 8 Nov 2026 13:42:49 +0100 Subject: [PATCH 095/105] texture_cache: keep the TIC and TSC lookup maps in flat hash maps TextureCacheChannelInfo::image_views and samplers were node-based std::unordered_maps. VisitImageView and FindSampler look them up whenever a descriptor table entry changes, so each lookup chased a bucket pointer and then a heap node. Every other hot texture cache map is already Common::unordered_map (boost::unordered_flat_map): page tables, framebuffers, sparse views, allocation and join tables. These two now use it too. The API (try_emplace, find, erase by iterator) is the same. The old TODO said the code relies on iterator stability. The only iterator held across other calls is VisitImageView's try_emplace result, held across CreateImageView. CreateImageView can reach RemoveImageViewReferences, which erases from image_views but never inserts. A flat map does not rehash on erase, and the erased entries never include the new one, whose id is still 0 at that point. Inserts happen only in VisitImageView itself. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit a518616c1d0cd08d7b24463667d9f27e7a84b1c6) --- src/video_core/texture_cache/texture_cache_base.h | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/video_core/texture_cache/texture_cache_base.h b/src/video_core/texture_cache/texture_cache_base.h index aa8664d642..a8efa17277 100644 --- a/src/video_core/texture_cache/texture_cache_base.h +++ b/src/video_core/texture_cache/texture_cache_base.h @@ -126,9 +126,11 @@ class TextureCacheChannelInfo : public ChannelInfo { DescriptorTable compute_image_table; DescriptorTable compute_sampler_table; - // TODO: still relies on bad iterators :( - std::unordered_map image_views; - std::unordered_map samplers; + // Flat maps, like the other hot texture cache maps: VisitImageView holds the try_emplace + // iterator across CreateImageView, which only erases from image_views (erasing never + // rehashes, and never removes the new entry: its id is still 0) and never inserts. + ::Common::unordered_map image_views; + ::Common::unordered_map samplers; TextureCacheGPUMap* gpu_page_table = nullptr; TextureCacheGPUMap* sparse_page_table = nullptr; From 0593fd75f4ede643352695087feb78935b5ba208 Mon Sep 17 00:00:00 2001 From: Julius Bairaktaris Date: Sun, 8 Nov 2026 21:56:25 +0100 Subject: [PATCH 096/105] kepler_compute: return a program's image handles by reference NativeImageHandles memoizes per program address but returned the pair of vectors by value, and DispatchNativeCompute asks for every Ocarina of Time dispatch with constant buffer 0 (~4,700 a second in the field): two vector copies, each a heap allocation and free, on the split back. It returns a reference into its memo (an unordered_map: references stay valid as it grows). Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 5edee0ccf895fe95590df2bc552cd5ef81dbae63) --- src/video_core/engines/kepler_compute.cpp | 7 ++++--- src/video_core/engines/kepler_compute.h | 4 +++- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/src/video_core/engines/kepler_compute.cpp b/src/video_core/engines/kepler_compute.cpp index 999b03c614..78a6988cdc 100644 --- a/src/video_core/engines/kepler_compute.cpp +++ b/src/video_core/engines/kepler_compute.cpp @@ -671,7 +671,7 @@ bool KeplerCompute::DispatchNativeCompute(const NativeQmdScheduler::Descriptor& return from_width >= 64 && to_width > from_width && to_width <= from_width * 4 && (cross_a > cross_b ? cross_a - cross_b : cross_b - cross_a) * 32 <= cross_a; }; - const auto [reads, stores] = NativeImageHandles(regs.code_loc.Address()); + const auto& [reads, stores] = NativeImageHandles(regs.code_loc.Address()); std::optional replacement_input; std::optional replacement_output; if (native_upscale_input && network_skip_remaining > 0 && !stores.empty()) { @@ -1214,7 +1214,8 @@ std::atomic g_nx2_trace_draws_after_upscale{}; // The cbuf0 byte offsets of the texture handles a native program reads (TEX, TLD, TLD4, TXD) and // the surface handles it stores to (SUST.P), from its code up to the first unconditional EXIT. -std::pair, std::vector> KeplerCompute::NativeImageHandles(GPUVAddr program) { +const std::pair, std::vector>& KeplerCompute::NativeImageHandles( + GPUVAddr program) { static std::unordered_map, std::vector>> cache; if (const auto it = cache.find(program); it != cache.end()) { return it->second; @@ -1252,7 +1253,7 @@ std::pair, std::vector> KeplerCompute::NativeImageHandles( void KeplerCompute::RememberNativeUpscaleInput() { native_upscale_input.reset(); - const auto [reads, stores] = NativeImageHandles(regs.code_loc.Address()); + const auto& [reads, stores] = NativeImageHandles(regs.code_loc.Address()); // An image-fed matrix pass writes its intermediate results to buffers. Retain its colour // input: the final pass's sampled image is another intermediate that a skipped network never // produced (Ocarina's movie network reads its colour at +0x58). diff --git a/src/video_core/engines/kepler_compute.h b/src/video_core/engines/kepler_compute.h index 68d3292008..f7240af5db 100644 --- a/src/video_core/engines/kepler_compute.h +++ b/src/video_core/engines/kepler_compute.h @@ -312,7 +312,9 @@ class KeplerCompute final : public EngineInterface { void CheckDlssNetwork(); [[nodiscard]] VideoCore::GameDlssReason GameDlssBlocker() const; void UpdateGameDlssStatus(bool complete, VideoCore::GameDlssReason blocker); - std::pair, std::vector> NativeImageHandles(GPUVAddr program); + /// The constant buffer 0 offsets of a program's image handles, read and stored: memoized + /// per program address, returned by reference (OoT asks for each of ~5,000 dispatches/s). + const std::pair, std::vector>& NativeImageHandles(GPUVAddr program); void RememberNativeUpscaleInput(); /// This frame's DLSS evaluation (Fortune's Weave): whether its passes all ran, and whether From d5c9589d0724a7946fdd074a0c8b3cb0a4229008 Mon Sep 17 00:00:00 2001 From: Julius Bairaktaris Date: Sun, 8 Nov 2026 13:34:58 +0100 Subject: [PATCH 097/105] vulkan_device: reuse the device memory usage for 50 ms The buffer and texture caches ask for the device memory usage every frame, and with VK_EXT_memory_budget each answer is a driver and OS query (~0.1-0.5 ms on Windows, PR author). Answers younger than 50 ms are reused. The PR's EDEN_NX2_MEMSTAT_MS tuning knob is the constant 50 ms (its default). Ported from #717 by `stephenjayakar`. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 73f82c80b6b5c527756ea9543db1d2a6e04caab3) --- src/video_core/vulkan_common/vulkan_device.cpp | 17 +++++++++++++++++ src/video_core/vulkan_common/vulkan_device.h | 4 ++++ 2 files changed, 21 insertions(+) diff --git a/src/video_core/vulkan_common/vulkan_device.cpp b/src/video_core/vulkan_common/vulkan_device.cpp index ba0054cce4..bff6668eb1 100644 --- a/src/video_core/vulkan_common/vulkan_device.cpp +++ b/src/video_core/vulkan_common/vulkan_device.cpp @@ -1919,6 +1919,23 @@ void Device::ReleaseCustomBorderColorSamplers(size_t count) const { } u64 Device::GetDeviceMemoryUsage() const { + // The buffer and texture caches ask every frame; with the budget extension each answer is a + // driver and OS query (~0.1-0.5 ms on Windows): answers younger than 50 ms are reused. + constexpr s64 reuse_ns = 50'000'000; + const s64 now = std::chrono::duration_cast( + std::chrono::steady_clock::now().time_since_epoch()) + .count(); + const s64 last = memory_usage_time.load(std::memory_order_acquire); + if (last != 0 && now - last < reuse_ns) { + return memory_usage_value.load(std::memory_order_relaxed); + } + const u64 usage = QueryDeviceMemoryUsage(); + memory_usage_value.store(usage, std::memory_order_relaxed); + memory_usage_time.store(now, std::memory_order_release); + return usage; +} + +u64 Device::QueryDeviceMemoryUsage() const { VkPhysicalDeviceMemoryBudgetPropertiesEXT budget{}; budget.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MEMORY_BUDGET_PROPERTIES_EXT; physical.GetMemoryProperties(&budget); diff --git a/src/video_core/vulkan_common/vulkan_device.h b/src/video_core/vulkan_common/vulkan_device.h index d74c24d848..c4b8decc21 100644 --- a/src/video_core/vulkan_common/vulkan_device.h +++ b/src/video_core/vulkan_common/vulkan_device.h @@ -1073,7 +1073,9 @@ FN_MAX_LIMIT_LIST return extensions.memory_budget; } + /// Device memory in use, reused for 50 ms (QueryDeviceMemoryUsage asks the driver). u64 GetDeviceMemoryUsage() const; + u64 QueryDeviceMemoryUsage() const; /// The driver's current memory budget for this process (VK_EXT_memory_budget), in /// GetDeviceMemoryUsage() terms: the usage at which the main device-local heap reaches its @@ -1379,6 +1381,8 @@ FN_MAX_LIMIT_LIST bool dynamic_state3_alpha_to_one{}; bool supports_conditional_barriers{}; ///< Allows barriers in conditional control flow. mutable std::atomic custom_border_color_samplers_used{}; + mutable std::atomic memory_usage_value{}; ///< Last GetDeviceMemoryUsage() answer + mutable std::atomic memory_usage_time{}; ///< When it was taken (steady clock ns) u64 device_access_memory{}; ///< Total size of device local memory in bytes. u32 sets_per_pool{}; ///< Sets per Description Pool NvidiaArchitecture nvidia_arch{NvidiaArchitecture::Arch_AmpereOrNewer}; From 58144d258043591a0b534400dee762fb2e6aec41 Mon Sep 17 00:00:00 2001 From: Julius Bairaktaris Date: Sun, 8 Nov 2026 16:05:21 +0100 Subject: [PATCH 098/105] fence_manager: don't flush again after queueing a fence SignalFence queued the fence and then, when it had downloads to flush, called FlushCommands. On Vulkan the fence's Queue already submits (or defers to the split's back with Scheduler::DeferFlush) everything up to its tick, so the second call submitted again whenever draws or native work had been counted since the last FlushCommands, at fences with downloads or queries (thousands a second in Switch 2 titles). OpenGL's fence only inserts a sync object: its QueueFence now flushes the commands before it itself. Ported from Citron `3a3297553` (by `hopesuffers`), with the OpenGL flush moved into its backend. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 5b6fd59ff05b6700ba6fd41d1294c8117071fbba) --- src/video_core/fence_manager.h | 3 --- src/video_core/renderer_opengl/gl_fence_manager.cpp | 2 ++ 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/src/video_core/fence_manager.h b/src/video_core/fence_manager.h index d7523db85f..48914d5e83 100644 --- a/src/video_core/fence_manager.h +++ b/src/video_core/fence_manager.h @@ -129,9 +129,6 @@ class FenceManager { func(); } fences.push(std::move(new_fence)); - if (should_flush) { - rasterizer.FlushCommands(); - } if constexpr (can_async_check) { guard.unlock(); cv.notify_all(); diff --git a/src/video_core/renderer_opengl/gl_fence_manager.cpp b/src/video_core/renderer_opengl/gl_fence_manager.cpp index 5326172af8..3cce24c9e6 100644 --- a/src/video_core/renderer_opengl/gl_fence_manager.cpp +++ b/src/video_core/renderer_opengl/gl_fence_manager.cpp @@ -49,6 +49,8 @@ Fence FenceManagerOpenGL::CreateFence(bool is_stubbed) { void FenceManagerOpenGL::QueueFence(Fence& fence) { fence->Queue(); + // Vulkan's fences submit when queued; here the commands before the fence are flushed. + rasterizer.FlushCommands(); } bool FenceManagerOpenGL::IsFenceSignaled(Fence& fence) const { From 6ec24332c1244800459d92f4ca67195f9e5ba1a2 Mon Sep 17 00:00:00 2001 From: Copilot App <223556219+Copilot@users.noreply.github.com> Date: Mon, 9 Nov 2026 09:37:27 -0300 Subject: [PATCH 099/105] video_core: align delta draws and memo image preparation (port of nx2 1c25fbdfb4 / #921) Keep the #805 draw profiling scope and move the b2304fa55a null-image guard into PreparesImage, shared by full resolves, memo hits and sampled-image checks. Add the upstream sampled-view tests plus a valid-view/null-image regression. Validation: git diff --cached --check passed. Build and runtime verification deferred to the caller; no CMake/Ninja build started. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- src/tests/CMakeLists.txt | 1 + src/tests/video_core/sampled_views.cpp | 142 ++++++++++++++++++ .../renderer_vulkan/vk_graphics_pipeline.cpp | 31 +++- .../renderer_vulkan/vk_graphics_pipeline.h | 4 + .../renderer_vulkan/vk_rasterizer.cpp | 45 +++++- .../renderer_vulkan/vk_rasterizer.h | 8 +- src/video_core/texture_cache/sampled_views.h | 42 ++++++ src/video_core/texture_cache/texture_cache.h | 22 +-- .../texture_cache/texture_cache_base.h | 16 ++ 9 files changed, 289 insertions(+), 22 deletions(-) create mode 100644 src/tests/video_core/sampled_views.cpp create mode 100644 src/video_core/texture_cache/sampled_views.h diff --git a/src/tests/CMakeLists.txt b/src/tests/CMakeLists.txt index ae4fb618fb..9367345054 100644 --- a/src/tests/CMakeLists.txt +++ b/src/tests/CMakeLists.txt @@ -54,6 +54,7 @@ add_executable(tests video_core/ampere_logged_textures.cpp video_core/native_texture_header_index.cpp video_core/texture_alias_ticks.cpp + video_core/sampled_views.cpp video_core/texture_handles.cpp video_core/ampere_selevet_modes.cpp video_core/ampere_selevet_spills.cpp diff --git a/src/tests/video_core/sampled_views.cpp b/src/tests/video_core/sampled_views.cpp new file mode 100644 index 0000000000..cee2fb138a --- /dev/null +++ b/src/tests/video_core/sampled_views.cpp @@ -0,0 +1,142 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later + +#include + +#include + +#include "common/slot_vector.h" +#include "video_core/texture_cache/image_base.h" +#include "video_core/texture_cache/image_view_base.h" +#include "video_core/texture_cache/image_view_info.h" +#include "video_core/texture_cache/sampled_views.h" + +namespace { + +using VideoCommon::ImageBase; +using VideoCommon::ImageFlagBits; +using VideoCommon::ImageId; +using VideoCommon::ImageInfo; +using VideoCommon::ImageType; +using VideoCommon::ImageViewBase; +using VideoCommon::ImageViewId; +using VideoCommon::ImageViewInfo; +using VideoCommon::ImageViewType; +using VideoCore::Surface::PixelFormat; + +ImageInfo Info() { + ImageInfo info; + info.format = PixelFormat::A8B8G8R8_UNORM; + info.type = ImageType::e2D; + info.size = {64, 64, 1}; + info.resources.levels = 1; + info.resources.layers = 1; + return info; +} + +struct View { + ImageViewId id; +}; + +/// Images and views laid out as TextureCache makes them: the null image and view first. +struct Cache { + Cache() { + REQUIRE(images.insert(VideoCommon::NullImageParams{}) == VideoCommon::NULL_IMAGE_ID); + REQUIRE(views.insert(VideoCommon::NullImageViewParams{}) == + VideoCommon::NULL_IMAGE_VIEW_ID); + } + + ImageId AddImage() { + const ImageId id = images.insert(Info(), 0x10000, 0x10000); + images[id].flags &= ~ImageFlagBits::CpuModified; // uploaded + return id; + } + + ImageViewId AddView(ImageId image) { + const ImageViewInfo info{ImageViewType::e2D, PixelFormat::A8B8G8R8_UNORM}; + return views.insert(info, images[image].info, image, 0x10000); + } + + bool Change(std::vector sampled) const { + return VideoCommon::SampledViewsChange(sampled, views, images); + } + + Common::SlotVector images; + Common::SlotVector views; +}; + +} // namespace + +TEST_CASE("The null view is never prepared", "[video_core][texture_cache]") { + Cache cache; + // Its image has no valid format and stays CpuModified: preparing it uploads it (#904). + REQUIRE(True(cache.images[VideoCommon::NULL_IMAGE_ID].flags & ImageFlagBits::CpuModified)); + CHECK_FALSE(VideoCommon::PreparesImage(cache.views, VideoCommon::NULL_IMAGE_VIEW_ID)); + CHECK_FALSE(cache.Change({{VideoCommon::NULL_IMAGE_VIEW_ID}})); + + // A draw sampling an uploaded texture beside a null one changes nothing. + const ImageViewId view = cache.AddView(cache.AddImage()); + CHECK_FALSE(cache.Change({{VideoCommon::NULL_IMAGE_VIEW_ID}, {view}})); +} + +TEST_CASE("Buffer views and empty ids are never prepared", "[video_core][texture_cache]") { + Cache cache; + ImageInfo info; + info.type = ImageType::Buffer; + info.format = PixelFormat::R32_UINT; + info.size = {256, 1, 1}; + const ImageViewInfo view_info{ImageViewType::Buffer, PixelFormat::R32_UINT}; + const ImageViewId buffer = cache.views.insert(info, view_info, 0x20000); + // A buffer view points at the null image. + REQUIRE(cache.views[buffer].image_id == VideoCommon::NULL_IMAGE_ID); + CHECK_FALSE(VideoCommon::PreparesImage(cache.views, buffer)); + CHECK_FALSE(VideoCommon::PreparesImage(cache.views, ImageViewId{})); + CHECK_FALSE(cache.Change({{buffer}, {ImageViewId{}}})); +} + +TEST_CASE("Empty descriptor views with their own ids never prepare the null image", + "[video_core][texture_cache]") { + Cache cache; + const ImageViewId empty = cache.views.insert(VideoCommon::NullImageViewParams{}); + REQUIRE(empty != VideoCommon::NULL_IMAGE_VIEW_ID); + REQUIRE_FALSE(cache.views[empty].IsBuffer()); + REQUIRE(cache.views[empty].image_id == VideoCommon::NULL_IMAGE_ID); + CHECK_FALSE(VideoCommon::PreparesImage(cache.views, empty)); + CHECK_FALSE(cache.Change({{empty}, {cache.AddView(cache.AddImage())}})); +} + +TEST_CASE("A sampled image with an upload pending changes", "[video_core][texture_cache]") { + Cache cache; + const ImageId image = cache.AddImage(); + const ImageViewId view = cache.AddView(image); + CHECK(VideoCommon::PreparesImage(cache.views, view)); + CHECK_FALSE(cache.Change({{view}})); + cache.images[image].flags |= ImageFlagBits::CpuModified; + CHECK(cache.Change({{view}})); +} + +TEST_CASE("A sampled image changes while a newer alias has not been copied in", + "[video_core][texture_cache]") { + Cache cache; + const ImageId sampled = cache.AddImage(); + const ImageId target = cache.AddImage(); + const ImageViewId view = cache.AddView(sampled); + REQUIRE(AddImageAlias(cache.images[sampled], cache.images[target], sampled, target)); + VideoCommon::SetModificationTick(cache.images, cache.images[sampled], 4); + VideoCommon::SetModificationTick(cache.images, cache.images[target], 3); + CHECK_FALSE(cache.Change({{view}})); + + // A draw renders into the alias (TextureCache::MarkModification): the next sample copies it. + VideoCommon::SetModificationTick(cache.images, cache.images[target], 5); + CHECK(cache.Change({{view}})); + + // As SynchronizeAliases leaves it after the copy. + VideoCommon::SetModificationTick(cache.images, cache.images[sampled], 5); + cache.images[sampled].newest_alias_tick = 5; + CHECK_FALSE(cache.Change({{view}})); + + // A newer alias deleted before it was copied leaves nothing to copy. + VideoCommon::SetModificationTick(cache.images, cache.images[target], 6); + cache.images[sampled].aliased_images.clear(); + CHECK_FALSE(cache.Change({{view}})); +} diff --git a/src/video_core/renderer_vulkan/vk_graphics_pipeline.cpp b/src/video_core/renderer_vulkan/vk_graphics_pipeline.cpp index 916c94c862..5b75a62278 100644 --- a/src/video_core/renderer_vulkan/vk_graphics_pipeline.cpp +++ b/src/video_core/renderer_vulkan/vk_graphics_pipeline.cpp @@ -11,6 +11,7 @@ #include #include #include +#include #include #include @@ -612,6 +613,8 @@ bool GraphicsPipeline::ConfigureImpl(bool is_indexed) { boost::container::small_vector handle_reads; boost::container::small_vector memo_views; boost::container::small_vector memo_samplers; + std::vector memo_prepared; + std::vector full_prepared; bool memo_hit{}; if (use_memo) { const auto collect_addresses{[&](size_t stage) LAMBDA_FORCEINLINE { @@ -641,9 +644,16 @@ bool GraphicsPipeline::ConfigureImpl(bool is_indexed) { } views.resize(num_image_elements); samplers.resize(num_textures); + if (verify_memo) { + // The full resolve prepares the images: the memo only lists those it would prepare. + texture_cache.LogPrepares(&memo_prepared, true); + } memo_hit = texture_cache.ResolveMemoizedTextures( texture_memo, std::span(handle_addresses.data(), handle_addresses.size()), std::span(views.data(), views.size()), std::span(samplers.data(), samplers.size())); + if (verify_memo) { + texture_cache.LogPrepares(nullptr, false); + } if (memo_hit && verify_memo) { memo_views.assign(views.begin(), views.end()); memo_samplers.assign(samplers.begin(), samplers.end()); @@ -774,8 +784,14 @@ bool GraphicsPipeline::ConfigureImpl(bool is_indexed) { ASSERT(samplers.size() == num_textures); if (resolve_in_full) { Profile::Scope scope{Profile::Zone::FillImageViews}; + if (memo_hit && verify_memo) { + texture_cache.LogPrepares(&full_prepared, false); + } texture_cache.FillImageViews(std::span(views.data(), views.size()), false, Spec::has_images); + if (memo_hit && verify_memo) { + texture_cache.LogPrepares(nullptr, false); + } } if (use_memo) { if (!memo_hit) { @@ -792,17 +808,22 @@ bool GraphicsPipeline::ConfigureImpl(bool is_indexed) { Count(g_memo_hits); } if (memo_hit && verify_memo) { + // Bindings alone miss what the paths do to the images (#904 prepared the null one). + for (auto* const prepared : {&memo_prepared, &full_prepared}) { + std::ranges::sort(*prepared); + prepared->erase(std::unique(prepared->begin(), prepared->end()), prepared->end()); + } const bool same = std::ranges::equal(views, memo_views, [](const auto& lhs, const auto& rhs) { return lhs.index == rhs.index && lhs.id == rhs.id; }) && - std::ranges::equal(samplers, memo_samplers); + std::ranges::equal(samplers, memo_samplers) && memo_prepared == full_prepared; if (!same) { Count(g_memo_mismatches); LOG_ERROR(Render_Vulkan, - "Texture memo: pipeline {:016x} resolved other views or samplers than " - "its memo", + "Texture memo: pipeline {:016x} resolved other views, samplers or " + "prepared images than its memo", key.Hash()); } } @@ -1018,6 +1039,10 @@ bool GraphicsPipeline::ConfigureImpl(bool is_indexed) { return ConfigureDraw(rescaling, render_area); } +bool GraphicsPipeline::DeltaImagesChange() const noexcept { + return texture_cache.SampledViewsChange(delta_views); +} + bool GraphicsPipeline::ConfigureDelta(bool is_indexed) { namespace Profile = VideoCommon::Nx2Profile; // ConfigureImpl without reading handles and resolving image views, samplers and texture diff --git a/src/video_core/renderer_vulkan/vk_graphics_pipeline.h b/src/video_core/renderer_vulkan/vk_graphics_pipeline.h index 33d489cfc0..631e21a06f 100644 --- a/src/video_core/renderer_vulkan/vk_graphics_pipeline.h +++ b/src/video_core/renderer_vulkan/vk_graphics_pipeline.h @@ -319,6 +319,10 @@ class GraphicsPipeline { return delta_mismatch; } + /// Whether an image the kept views sample has an upload pending or a newer alias to copy in: + /// ConfigureDelta prepares none, Configure does. + [[nodiscard]] bool DeltaImagesChange() const noexcept; + template static auto MakeConfigureSpecFunc() { return [](GraphicsPipeline* pl, bool is_indexed) { return pl->ConfigureImpl(is_indexed); }; diff --git a/src/video_core/renderer_vulkan/vk_rasterizer.cpp b/src/video_core/renderer_vulkan/vk_rasterizer.cpp index f2756c829f..97841c2a38 100644 --- a/src/video_core/renderer_vulkan/vk_rasterizer.cpp +++ b/src/video_core/renderer_vulkan/vk_rasterizer.cpp @@ -512,11 +512,36 @@ GraphicsPipeline* RasterizerVulkan::TakeReusableDraw(bool is_indexed, bool is_in } } } + taken_draw = draw; return draw.pipeline; } +bool RasterizerVulkan::DeltaStillHolds(u64 serial, u64 map_serial) { + const auto full = [&](FullDraw reason) { + ++fast_draw_stats.full[static_cast(reason)]; + return false; + }; + // TakeReusableDraw ran before the caches were locked, while other threads could still write, + // unmap or delete what the delta draw keeps. + if (taken_draw.serial != serial || taken_draw.map_serial != map_serial) { + return full(FullDraw::StateChanged); + } + if (taken_draw.tick != scheduler.CurrentTick()) { + return full(FullDraw::Submitted); + } + if (taken_draw.render_pass != scheduler.RenderPassSerial() || !scheduler.IsRenderPassActive()) { + return full(FullDraw::RenderPass); + } + // A delta draw prepares none of the images it samples: one with an upload pending or a newer + // alias to copy in (a render target aliasing it, written by the draw before) needs Configure. + if (taken_draw.pipeline->DeltaImagesChange()) { + return full(FullDraw::SampledImages); + } + return true; +} + void RasterizerVulkan::NoteResolvedDraw(GraphicsPipeline* pipeline, bool is_indexed, - bool is_indirect) { + bool is_indirect, u64 serial, u64 map_serial) { const auto& draw_state = maxwell3d->draw_manager.draw_state; const auto topology = draw_state.topology; const auto ineligible = [&]() -> std::optional { @@ -543,8 +568,8 @@ void RasterizerVulkan::NoteResolvedDraw(GraphicsPipeline* pipeline, bool is_inde reusable_draw = ReusableDraw{ .pipeline = pipeline, .maxwell3d = maxwell3d, - .serial = VideoCommon::Nx2FastDraw::Serial(), - .map_serial = gpu_memory->MapSerial(), + .serial = serial, + .map_serial = map_serial, .tick = scheduler.CurrentTick(), .render_pass = scheduler.RenderPassSerial(), .topology = topology, @@ -575,12 +600,12 @@ void RasterizerVulkan::ReportFastDraws() { "Fast draw per second: {:.0f} delta | full: {:.0f} after an ineligible draw, {:.0f} " "different kind, {:.0f} state changed, {:.0f} after a submit, {:.0f} render pass or " "pipeline changed, {:.0f} pipeline takes no delta draws, {:.0f} handle buffer bound, " - "{:.0f} handle data written", + "{:.0f} handle data written, {:.0f} sampled image to prepare", static_cast(stats.delta) / seconds, per_second(FullDraw::NoCandidate), per_second(FullDraw::Kind), per_second(FullDraw::StateChanged), per_second(FullDraw::Submitted), per_second(FullDraw::RenderPass), per_second(FullDraw::DeltaPipeline), per_second(FullDraw::HandleBind), - per_second(FullDraw::HandleData)); + per_second(FullDraw::HandleData), per_second(FullDraw::SampledImages)); static constexpr std::array(Ineligible::Count)> ineligible_names{"indirect", "mesh", "not built", "writes images", "transform feedback", "inline index", "quads", "repacked vertices", @@ -995,6 +1020,10 @@ void RasterizerVulkan::PrepareDraw(bool is_indexed, Func&& draw_func, bool is_in mesh_emulation_vertices_per_group = pipeline->MeshEmulationVerticesPerGroup(); } std::scoped_lock lock{buffer_cache.mutex, texture_cache.mutex}; + // What the next draw's TakeReusableDraw compares with, read once the caches are locked: a + // break from here on, this draw's own resolve included, ends the delta chain. + const u64 draw_serial = FastDraw::Serial(); + const u64 draw_map_serial = gpu_memory->MapSerial(); // update engine as channel may be different. pipeline->SetEngine(maxwell3d, gpu_memory); const u32 repack_dirty = buffer_cache_runtime.SetVertexRepackLayout( @@ -1010,7 +1039,9 @@ void RasterizerVulkan::PrepareDraw(bool is_indexed, Func&& draw_func, bool is_in } } setup_scope.reset(); - const DrawPath path = delta_pipeline && !verify_pipeline ? DrawPath::Delta : DrawPath::Full; + const bool delta_holds = + delta_pipeline != nullptr && DeltaStillHolds(draw_serial, draw_map_serial); + const DrawPath path = delta_holds && !verify_pipeline ? DrawPath::Delta : DrawPath::Full; if (path == DrawPath::Delta) { if (Profile::Scope scope{Profile::Zone::Configure}; !pipeline->ConfigureDelta(is_indexed)) { report_failure("configure failed"); @@ -1118,7 +1149,7 @@ void RasterizerVulkan::PrepareDraw(bool is_indexed, Func&& draw_func, bool is_in } #endif if (fast_draw_on) { - NoteResolvedDraw(pipeline, is_indexed, is_indirect); + NoteResolvedDraw(pipeline, is_indexed, is_indirect, draw_serial, draw_map_serial); const size_t index = static_cast(path); fast_draw_stats.prepare_ns[index] += Profile::Now() - fast_draw_start; ++fast_draw_stats.prepare_draws[index]; diff --git a/src/video_core/renderer_vulkan/vk_rasterizer.h b/src/video_core/renderer_vulkan/vk_rasterizer.h index be27f129ac..572d042d45 100644 --- a/src/video_core/renderer_vulkan/vk_rasterizer.h +++ b/src/video_core/renderer_vulkan/vk_rasterizer.h @@ -400,6 +400,7 @@ class RasterizerVulkan final : public VideoCore::RasterizerInterface, DeltaPipeline, HandleBind, HandleData, + SampledImages, Count, }; /// Why a recorded draw can't be followed by a delta draw. @@ -440,9 +441,14 @@ class RasterizerVulkan final : public VideoCore::RasterizerInterface, /// The pipeline the draw can take the delta path with, or null for a full resolve. Clears /// the remembered draw either way: only a recorded draw remembers one again. GraphicsPipeline* TakeReusableDraw(bool is_indexed, bool is_indirect); - void NoteResolvedDraw(GraphicsPipeline* pipeline, bool is_indexed, bool is_indirect); + /// Whether the draw TakeReusableDraw took still holds once the caches are locked (serial and + /// map_serial as read then), and the images it samples need no preparing. + bool DeltaStillHolds(u64 serial, u64 map_serial); + void NoteResolvedDraw(GraphicsPipeline* pipeline, bool is_indexed, bool is_indirect, u64 serial, + u64 map_serial); void ReportFastDraws(); ReusableDraw reusable_draw; + ReusableDraw taken_draw; ///< The draw TakeReusableDraw last returned the pipeline of /// Device memory of the constant buffers TakeReusableDraw checks for texture handle writes. std::array, Maxwell::MaxShaderStage> diff --git a/src/video_core/texture_cache/sampled_views.h b/src/video_core/texture_cache/sampled_views.h new file mode 100644 index 0000000000..12b48a9835 --- /dev/null +++ b/src/video_core/texture_cache/sampled_views.h @@ -0,0 +1,42 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later + +#pragma once + +#include + +#include "common/common_funcs.h" +#include "video_core/texture_cache/image_base.h" +#include "video_core/texture_cache/image_view_base.h" +#include "video_core/texture_cache/types.h" + +namespace VideoCommon { + +/// Whether TextureCache::PrepareImageView prepares the view's image. Neither a view of the null +/// image (including empty descriptors with their own view ids), whose format is invalid and which +/// stays CpuModified, nor a buffer view is prepared. +template +[[nodiscard]] bool PreparesImage(const ImageViews& image_views, ImageViewId id) noexcept { + return id && id != NULL_IMAGE_VIEW_ID && !image_views[id].IsBuffer() && + image_views[id].image_id != NULL_IMAGE_ID; +} + +/// Whether TextureCache::PrepareImage would change a sampled image: upload it from guest memory +/// or copy a newer alias into it (SynchronizeAliases). +[[nodiscard]] inline bool SamplingChangesImage(const ImageBase& image) noexcept { + return True(image.flags & ImageFlagBits::CpuModified) || + (!image.aliased_images.empty() && image.newest_alias_tick > image.modification_tick); +} + +/// Whether sampling any of the views now needs a resolve that prepares their images (fast draw +/// delta draws keep the previous resolve's views and prepare none). +template +[[nodiscard]] bool SampledViewsChange(const Views& views, const ImageViews& image_views, + const Images& images) noexcept { + return std::ranges::any_of(views, [&](const auto& view) { + return PreparesImage(image_views, view.id) && + SamplingChangesImage(images[image_views[view.id].image_id]); + }); +} + +} // namespace VideoCommon diff --git a/src/video_core/texture_cache/texture_cache.h b/src/video_core/texture_cache/texture_cache.h index 77a90a348a..fad5fbca7b 100644 --- a/src/video_core/texture_cache/texture_cache.h +++ b/src/video_core/texture_cache/texture_cache.h @@ -692,10 +692,7 @@ bool TextureCache

::ResolveMemoizedTextures(SampledTextureMemo& memo, const SampledTextureMemo::Entry& entry = memo.entries[i]; views[i] = ImageViewInOut{.index = entry.view_index, .blacklist = false, .id = entry.view}; samplers[i] = entry.sampler; - // As in VisitImageView: the null image has nothing to upload and no valid format. - if (entry.view != NULL_IMAGE_VIEW_ID) { - PrepareImageView(entry.view, false, false); - } + PrepareImageView(entry.view, false, false); } if (has_deleted_images) { // Preparing an image deleted others: the full path resolves the views again. @@ -2544,6 +2541,8 @@ bool TextureCache

::ImageCanRescale(ImageBase& image) { template void TextureCache

::InvalidateScale(Image& image) { + // Its views are deleted: a fast draw delta draw keeping one must not follow. + VideoCommon::Nx2FastDraw::Break(VideoCommon::Nx2FastDraw::Reason::Invalidation); if (image.scale_tick <= frame_tick) { image.scale_tick = frame_tick + 1; } @@ -4090,6 +4089,12 @@ void TextureCache

::SynchronizeAliases(ImageId image_id) { template void TextureCache

::PrepareImage(ImageId image_id, bool is_modification, bool invalidate) { + if (prepare_log != nullptr) { + prepare_log->push_back(image_id); + if (prepare_log_dry) { + return; + } + } Image& image = slot_images[image_id]; if (invalidate) { image.flags &= ~(ImageFlagBits::CpuModified | ImageFlagBits::GpuModified); @@ -4145,15 +4150,10 @@ void TextureCache

::PrepareImage(ImageId image_id, bool is_modification, bool template void TextureCache

::PrepareImageView(ImageViewId image_view_id, bool is_modification, bool invalidate) { - if (!image_view_id) { - return; - } - const ImageViewBase& image_view = slot_image_views[image_view_id]; - // Views of empty descriptors point at the null image: there is nothing to upload or track. - if (image_view.IsBuffer() || image_view.image_id == NULL_IMAGE_ID) { + if (!PreparesImage(slot_image_views, image_view_id)) { return; } - PrepareImage(image_view.image_id, is_modification, invalidate); + PrepareImage(slot_image_views[image_view_id].image_id, is_modification, invalidate); } template diff --git a/src/video_core/texture_cache/texture_cache_base.h b/src/video_core/texture_cache/texture_cache_base.h index a8efa17277..0b2b34d405 100644 --- a/src/video_core/texture_cache/texture_cache_base.h +++ b/src/video_core/texture_cache/texture_cache_base.h @@ -45,6 +45,7 @@ #include "video_core/texture_cache/image_info.h" #include "video_core/texture_cache/image_view_base.h" #include "video_core/texture_cache/render_targets.h" +#include "video_core/texture_cache/sampled_views.h" #include "video_core/texture_cache/stale_pages.h" #include "video_core/texture_cache/types.h" #include "video_core/textures/texture.h" @@ -300,6 +301,19 @@ class TextureCache : public VideoCommon::ChannelSetupCaches views); + /// Whether sampling the views now needs PrepareImageView: an image with an upload pending or + /// a newer alias to copy in (fast draw delta draws keep their views and prepare none). + [[nodiscard]] bool SampledViewsChange(std::span views) const noexcept { + return VideoCommon::SampledViewsChange(views, slot_image_views, slot_images); + } + + /// EDEN_NX2_TEXTURE_MEMO_VERIFY: PrepareImage adds the images it is asked to prepare to `log` + /// (nullptr stops it) and, with `dry`, does nothing else. + void LogPrepares(std::vector* log, bool dry) noexcept { + prepare_log = log; + prepare_log_dry = dry; + } + /// Get the sampler from the graphics/compute descriptor table in the specified index Sampler* GetSampler(u32 index, bool compute); @@ -635,6 +649,8 @@ class TextureCache : public VideoCommon::ChannelSetupCaches ImageCpuAddress(GPUVAddr gpu_addr, size_t size) const; bool has_deleted_images = false; + std::vector* prepare_log{}; ///< LogPrepares + bool prepare_log_dry{}; /// True while a clear binds its render targets (UpdateRenderTargets(true)), so JoinImages can /// skip the upload of a new target the clear overwrites whole. bool clearing_render_targets = false; From a21232a5b8405546b16f0de497c9d702e789e519 Mon Sep 17 00:00:00 2001 From: Copilot App <223556219+Copilot@users.noreply.github.com> Date: Mon, 9 Nov 2026 09:41:20 -0300 Subject: [PATCH 100/105] video_core: keep uniforms and replay delta image descriptors (partial port of nx2 a4f6287327 / #949) Port 6b38851501 (uniform keeps), ee98215f14 (descriptor replay), and 3e86407514 (notified CPU-write invalidation). The ClearBuffer and buffer-copy test support from b2375cff7f already exists on #805. Retain #805 scheduler/acquire/drain behavior, async readback bookkeeping, in-place GetSpan uniform comparisons and fine profiling. Keep all existing buffer-cache tests; add keep range/lifetime and write-overflow coverage. Skip empty image ranges during descriptor replay. Document the unnotified CPU-write assumption and separate verify passes. Validation: git diff 58144d2580 --check and static port invariants passed (excluded split/acquire code unchanged, all 45 existing buffer-port tests retained, test registration and conflict/hint checks). Compilation and executable tests deferred to the caller as requested; no CMake/Ninja build started. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- NX2_GPU_THREAD_SPLIT.md | 43 ++++ src/tests/video_core/buffer_cache_ports.cpp | 235 +++++++++++++++++- src/tests/video_core/uniform_stream_reuse.cpp | 39 +++ src/video_core/buffer_cache/buffer_cache.h | 91 +++++++ .../buffer_cache/buffer_cache_base.h | 44 ++++ .../buffer_cache/uniform_stream_reuse.h | 45 +++- src/video_core/nx2_fast_draw.h | 8 +- .../renderer_vulkan/pipeline_helper.h | 5 + .../renderer_vulkan/vk_graphics_pipeline.cpp | 116 ++++++++- .../renderer_vulkan/vk_graphics_pipeline.h | 10 + .../renderer_vulkan/vk_rasterizer.cpp | 2 + .../renderer_vulkan/vk_rasterizer.h | 2 + .../renderer_vulkan/vk_update_descriptor.h | 8 + src/video_core/texture_cache/texture_cache.h | 1 + .../texture_cache/texture_cache_base.h | 6 + 15 files changed, 642 insertions(+), 13 deletions(-) diff --git a/NX2_GPU_THREAD_SPLIT.md b/NX2_GPU_THREAD_SPLIT.md index 75fd0e4105..c05af5867b 100644 --- a/NX2_GPU_THREAD_SPLIT.md +++ b/NX2_GPU_THREAD_SPLIT.md @@ -61,3 +61,46 @@ with stage 2, 50,000-66,500 with stage 3. The front waits ~4,000-5,000 times a s acquires, MME DMA reads); the back still spends ~200 ms a second outside its zones (ring dispatch, method sink). Clean A/B (NX2_NOTES row 145): 1.46x throughput, frames 17% shorter; on by default since. The back is ~92% busy, ~53% of it draw setup: the next step is there, not the ring. + +## Fast draw ports on the #805 performance branch (2026-11-09) + +The port of nx2 `1c25fbdfb4` (#921) makes full resolves and texture memo hits use +the same image-preparation predicate. Empty descriptor views backed by the null +image are excluded even when their view id is not the null view id. Delta draws +fall back when a sampled image needs an upload or alias synchronization, or when +the draw's serial, mappings, command buffer or render pass changed before the +caches were locked. Rescaling ends the chain too. + +Only the uniform-keep and image-descriptor replay portions of nx2 `a4f6287327` +(#949) are ported: `6b38851501`, `ee98215f14`, and the CPU-write follow-up +`3e86407514`. The test-runtime support in `b2375cff7f` already exists on #805. +Its semaphore, waiter, acquire/drain and scheduler-ownership changes are not +included; #805 retains its own implementations. + +Small streamed uniforms can reuse the previous allocation without rereading +guest memory when the stage, slot, device range and command-buffer tick still +match, and no overlapping inline data, notified CPU write, GPU write or buffer +deletion invalidated it. More than eight notified writes between draws disable +keeps for the next draw. Unsynchronized CPU writes unknown to the caches remain +an assumption to check on the game. Per-stage image descriptor entries and +rescaling words are replayed until the views are recaptured or the scale epoch +changes; buffer descriptors still bind on every draw. + +Validate a fixed scene with these diagnostics separately (set each to `1`): + +- `EDEN_NX2_TEXTURE_MEMO_VERIFY`: compare views, samplers and prepared image sets. +- `EDEN_NX2_FAST_DRAW_VERIFY`: full-resolve comparison; this deliberately bypasses + delta draws, so it does not exercise uniform keeps or descriptor replay. +- `EDEN_NX2_UBO_KEEP_VERIFY`: reread would-be kept uniforms and report changed bytes. +- `EDEN_NX2_DELTA_DESCRIPTORS_VERIFY`: regenerate and compare image descriptors + and rescaling words. +- `EDEN_NX2_GPU_PROFILE`: report keep/replay counts and draw fallback reasons; + value `2` also retains #805's fine uniform-binding counters. + +After building the test target, run `tests.exe "[buffer_ports]"`, +`tests.exe "[video_core][texture_cache]"`, and +`tests.exe "Uniform stream*"`. These ports were reviewed without building on +the porting machine; compilation, runtime verification and visual A/B testing +are still required. #949's reported Bananza gain (+8–9%, back thread about +56.0 to 52.2 ms/frame on a Ryzen 9 5900X) includes its acquire changes and does +not predict the gain of this partial port in Ocarina of Time. diff --git a/src/tests/video_core/buffer_cache_ports.cpp b/src/tests/video_core/buffer_cache_ports.cpp index b4aa66d376..993ea497f0 100644 --- a/src/tests/video_core/buffer_cache_ports.cpp +++ b/src/tests/video_core/buffer_cache_ports.cpp @@ -239,6 +239,37 @@ struct Policy { using Cache = VideoCommon::BufferCache; +/// Streams small uniform buffers as the Vulkan runtime does (StreamUniformReusable). +struct KeepRuntime : Runtime { + struct StreamUniformRef { + size_t allocation; + }; + std::span BindMappedUniformBuffer(size_t, u32, u32 size) { + allocations.emplace_back(size); + last = bound = StreamUniformRef{allocations.size() - 1}; + return allocations.back(); + } + StreamUniformRef LastStreamUniformRef() const noexcept { + return last; + } + void RebindStreamUniform(StreamUniformRef ref, u32 size) { + REQUIRE(allocations.at(ref.allocation).size() == size); + bound = ref; + } + u64 CurrentTick() const noexcept { + return tick; + } + std::vector> allocations; + StreamUniformRef last{}, bound{}; + u64 tick{1}; +}; + +struct KeepPolicy : Policy { + using Runtime = ::KeepRuntime; +}; + +using KeepCache = VideoCommon::BufferCache; + } // namespace namespace VideoCommon { @@ -249,7 +280,8 @@ struct BufferCacheTestAccess { cache.gpu_memory = &memory; return cache.StorageBufferBinding(descriptor, 0, written, native, table); } - static BufferId Add(Cache& cache, DAddr address, u32 size, bool mapped = false) { + template + static BufferId Add(C& cache, DAddr address, u32 size, bool mapped = false) { const auto id = cache.slot_buffers.insert(cache.runtime, address, size, mapped); cache.Register(id); cache.memory_tracker.ForEachUploadRange(address, size, [](auto, auto) {}); @@ -332,6 +364,26 @@ struct BufferCacheTestAccess { static const auto& ReadChecks(Cache& cache) { return cache.read_check; } + static void KeepDraw(KeepCache& cache, BufferCacheChannelInfo& channel, + const Nx2FastDraw::Delta* delta, BufferId id, DAddr address, + u32 size) { + cache.channel_state = &channel; + cache.maxwell3d = &channel.maxwell3d; + cache.gpu_memory = &channel.gpu_memory; + cache.BeginGraphicsUniforms(delta); + channel.uniform_buffers[0][0] = {.device_addr = address, .size = size, .buffer_id = id}; + cache.BindHostGraphicsUniformBuffer(0, 0, 0, true); + } + struct KeepCounts { + u64 kept, streamed, verified, mismatches; + }; + static KeepCounts Counts(const KeepCache& cache) { + const auto& keep = cache.uniform_keep; + return {keep.kept, keep.streamed_binds, keep.verified, keep.mismatches}; + } + static void GpuWrite(KeepCache& cache, BufferId id, DAddr address, u32 size) { + cache.MarkWrittenBuffer(id, address, size); + } }; } // namespace VideoCommon @@ -340,7 +392,8 @@ namespace { using Access = VideoCommon::BufferCacheTestAccess; -struct Fixture { +template +struct BasicFixture { ScopedSystemTestContext context; Core::System system; Core::DeviceMemory physical; @@ -348,13 +401,13 @@ struct Fixture { Tegra::GPU gpu{system, false, false}; Null::RasterizerNull rasterizer{gpu}; std::optional memory; - Runtime runtime; - std::unique_ptr cache; + R runtime; + std::unique_ptr cache; std::unique_ptr process; Core::Asid asid{}; u8* bytes{}; - Fixture() { + BasicFixture() { system.Initialize(); process = std::make_unique(system.Kernel()); auto& table = process->GetPageTable().GetImpl(); @@ -371,9 +424,9 @@ struct Fixture { device.BindInterface(&rasterizer); device.Map(0x10000, 0x10000, 0x40000, asid); ResetMemory(); - cache = std::make_unique(device, runtime); + cache = std::make_unique(device, runtime); } - ~Fixture() { + ~BasicFixture() { cache.reset(); memory.reset(); device.Unmap(0x10000, 0x40000); @@ -448,6 +501,81 @@ class ReadRasterizer final : public VideoCore::RasterizerInterface { Cache& cache; }; +using Fixture = BasicFixture; + +void SetEnv(const char* name, const char* value) { +#ifdef _WIN32 + _putenv_s(name, value ? value : ""); +#else + if (value) { + setenv(name, value, 1); + } else { + unsetenv(name); + } +#endif +} + +struct EnvGate { + explicit EnvGate(const char* name_) : name{name_} { + if (name != nullptr) { + SetEnv(name, "1"); + } + } + ~EnvGate() { + if (name != nullptr) { + SetEnv(name, nullptr); + } + } + const char* name; +}; + +/// A buffer cache built with the switch `gate` set (none if null), drawing with one 0x100-byte +/// uniform buffer at stage 0 slot 0, device 0x10800 (GPU 0x100800). +struct KeepFixture : EnvGate, BasicFixture { + static constexpr DAddr Ubo = 0x10800; + static constexpr GPUVAddr UboGpu = 0x100800; + static constexpr u32 UboSize = 0x100; + + explicit KeepFixture(const char* gate) : EnvGate{gate} { + state.memory_manager = &*memory; + state.payload.emplace(system, *memory, state); + channel.emplace(state); + sizes[0][0] = UboSize; + channel->uniform_buffer_sizes = &sizes; + id = Access::Add(*cache, 0x10000, 0x10000); + } + u8* UboBytes() { + return bytes + (Ubo - 0x10000); + } + /// Draws and checks that the bound uniform data is what guest memory holds. + void Draw(const VideoCommon::Nx2FastDraw::Delta* delta) { + Access::KeepDraw(*cache, *channel, delta, id, Ubo, UboSize); + const auto& data = runtime.allocations.at(runtime.bound.allocation); + REQUIRE(data.size() == UboSize); + REQUIRE(std::memcmp(data.data(), UboBytes(), UboSize) == 0); + } + Access::KeepCounts Counts() const { + return Access::Counts(*cache); + } + + Tegra::Control::ChannelState state{0}; + std::optional channel; + VideoCommon::UniformBufferSizes sizes{}; + VideoCommon::BufferId id; +}; + +/// Inline constant data over device [begin, end), through GPU address begin + UboGpu - Ubo. +VideoCommon::Nx2FastDraw::Delta Written(DAddr begin, DAddr end) { + VideoCommon::Nx2FastDraw::Delta delta{}; + delta.write_begin[0] = begin + (KeepFixture::UboGpu - KeepFixture::Ubo); + delta.write_end[0] = end + (KeepFixture::UboGpu - KeepFixture::Ubo); + delta.num_writes = 1; + delta.device_begin[0] = begin; + delta.device_end[0] = end; + delta.num_device_writes = 1; + return delta; +} + } // namespace TEST_CASE("Storage binding cache tracks map changes and descriptor dependencies", @@ -1970,3 +2098,96 @@ TEST_CASE("Read download check leaves out early downloaded bytes an upload repla REQUIRE(Access::ReadChecks(*f.cache).mismatched == 0); REQUIRE(Access::ReadChecks(*f.cache).uncovered == 0); } + +TEST_CASE("Uniform keep binds unwritten uniform buffers of delta draws without reading them", + "[buffer_ports]") { + KeepFixture f{nullptr}; + using Delta = VideoCommon::Nx2FastDraw::Delta; + const Delta unchanged{}; + std::memset(f.UboBytes(), 1, KeepFixture::UboSize); + f.Draw(nullptr); + f.Draw(&unchanged); + REQUIRE(f.Counts().kept == 1); + REQUIRE(f.runtime.allocations.size() == 1); + + // Inline constant data over the range, then a rebind of the slot. + f.UboBytes()[0x10] = 2; + const Delta written = Written(KeepFixture::Ubo + 0x10, KeepFixture::Ubo + 0x14); + f.Draw(&written); + REQUIRE(f.Counts().kept == 1); + REQUIRE(f.runtime.allocations.size() == 2); + Delta rebound{}; + rebound.binds[0] = 1; + f.Draw(&rebound); + REQUIRE(f.Counts().kept == 1); + f.Draw(&unchanged); + REQUIRE(f.Counts().kept == 2); + + // Data written next to the range, and GPU writes elsewhere, leave it kept. + const Delta beside = Written(KeepFixture::Ubo + 0x100, KeepFixture::Ubo + 0x140); + f.Draw(&beside); + REQUIRE(f.Counts().kept == 3); + Access::GpuWrite(*f.cache, f.id, 0x18000, 0x100); + f.Draw(&unchanged); + REQUIRE(f.Counts().kept == 4); + + // A GPU write over the range: the original path binds it from the buffer from now on. + Access::GpuWrite(*f.cache, f.id, KeepFixture::Ubo + 0x80, 4); + const size_t allocations = f.runtime.allocations.size(); + Access::KeepDraw(*f.cache, *f.channel, &unchanged, f.id, KeepFixture::Ubo, KeepFixture::UboSize); + Access::KeepDraw(*f.cache, *f.channel, &unchanged, f.id, KeepFixture::Ubo, KeepFixture::UboSize); + REQUIRE(f.Counts().kept == 4); + REQUIRE(f.runtime.allocations.size() == allocations); +} + +TEST_CASE("Uniform keep ends at CPU writes the caches are told about", "[buffer_ports]") { + KeepFixture f{nullptr}; + const VideoCommon::Nx2FastDraw::Delta unchanged{}; + f.Draw(nullptr); + f.Draw(&unchanged); + REQUIRE(f.Counts().kept == 1); + f.UboBytes()[0x20] = 7; + f.cache->WriteMemory(KeepFixture::Ubo + 0x20, 1); + f.Draw(&unchanged); + REQUIRE(f.Counts().kept == 1); + f.cache->WriteMemory(0x18000, 0x10); + f.Draw(&unchanged); + REQUIRE(f.Counts().kept == 2); + + // Losing the write list is conservative even when the recorded ranges are elsewhere. + for (u32 write = 0; write < 9; ++write) { + f.cache->WriteMemory(0x18000 + write * 0x10, 0x10); + } + f.Draw(&unchanged); + REQUIRE(f.Counts().kept == 2); + f.Draw(&unchanged); + REQUIRE(f.Counts().kept == 3); +} + +TEST_CASE("Uniform keep never keeps across command buffers", "[buffer_ports]") { + KeepFixture f{nullptr}; + const VideoCommon::Nx2FastDraw::Delta unchanged{}; + f.Draw(nullptr); + ++f.runtime.tick; + f.Draw(&unchanged); + REQUIRE(f.Counts().kept == 0); + REQUIRE(f.runtime.allocations.size() == 2); + f.Draw(nullptr); + f.Draw(&unchanged); + REQUIRE(f.Counts().kept == 1); +} + +TEST_CASE("Uniform keep verify counts guest writes the delta did not report", + "[buffer_ports]") { + KeepFixture f{"EDEN_NX2_UBO_KEEP_VERIFY"}; + const VideoCommon::Nx2FastDraw::Delta unchanged{}; + f.Draw(nullptr); + f.Draw(&unchanged); + REQUIRE(f.Counts().verified == 1); + REQUIRE(f.Counts().mismatches == 0); + f.UboBytes()[3] = 9; + f.Draw(&unchanged); + REQUIRE(f.Counts().verified == 2); + REQUIRE(f.Counts().mismatches == 1); + REQUIRE(f.Counts().kept == 0); +} diff --git a/src/tests/video_core/uniform_stream_reuse.cpp b/src/tests/video_core/uniform_stream_reuse.cpp index a0265b123b..567b3bb7c6 100644 --- a/src/tests/video_core/uniform_stream_reuse.cpp +++ b/src/tests/video_core/uniform_stream_reuse.cpp @@ -128,3 +128,42 @@ TEST_CASE("Uniform stream reuse preserves data through mutation and session rese Check(memory.gathered > 1000 && memory.in_place > 1000, "the reads did not take both the gathered and the in-place path"); } + +TEST_CASE("Uniform stream keeps require the same range and live allocation", "[video_core]") { + using VideoCommon::BindSmallUniformReusing; + using VideoCommon::RebindKeptUniform; + Memory memory; + Runtime runtime; + VideoCommon::ResetUniformStreamReuse(); + CHECK_FALSE(RebindKeptUniform(runtime, 0, 0, 0, 64)); + REQUIRE(BindSmallUniformReusing(runtime, memory, 0, 0, 0, 0, 64)); + const u32 gathered = memory.gathered; + const u32 in_place = memory.in_place; + REQUIRE(RebindKeptUniform(runtime, 0, 0, 0, 64)); + CheckBound(runtime, memory, 0, 64); + CHECK(memory.gathered == gathered); + CHECK(memory.in_place == in_place); + CHECK_FALSE(RebindKeptUniform(runtime, 0, 0, 64, 64)); + CHECK_FALSE(RebindKeptUniform(runtime, 0, 0, 0, 32)); + CHECK_FALSE(RebindKeptUniform(runtime, 0, 1, 0, 64)); + CHECK_FALSE(RebindKeptUniform(runtime, 1, 0, 0, 64)); + CHECK_FALSE(RebindKeptUniform(runtime, 5, 0, 0, 64)); + CHECK_FALSE(RebindKeptUniform(runtime, 0, 18, 0, 64)); + CHECK_FALSE(RebindKeptUniform(runtime, 0, 0, 0, 0)); + CHECK_FALSE(RebindKeptUniform(runtime, 0, 0, 0, 4097)); + + // Equal bytes at a different address reuse the allocation, but update the keep's range. + REQUIRE(BindSmallUniformReusing(runtime, memory, 0, 0, 0, 64, 64)); + CHECK(runtime.allocations.size() == 1); + CHECK_FALSE(RebindKeptUniform(runtime, 0, 0, 0, 64)); + REQUIRE(RebindKeptUniform(runtime, 0, 0, 64, 64)); + CHECK(VideoCommon::KeptUniformHolds(memory, 0, 0, 64, 64)); + memory.bytes[67] = 9; + CHECK_FALSE(VideoCommon::KeptUniformHolds(memory, 0, 0, 64, 64)); + + runtime.NextTick(); + CHECK_FALSE(RebindKeptUniform(runtime, 0, 0, 64, 64)); + REQUIRE(BindSmallUniformReusing(runtime, memory, 0, 0, 0, 64, 64)); + VideoCommon::ResetUniformStreamReuse(); + CHECK_FALSE(RebindKeptUniform(runtime, 0, 0, 64, 64)); +} diff --git a/src/video_core/buffer_cache/buffer_cache.h b/src/video_core/buffer_cache/buffer_cache.h index 7db04dcb21..895b82318e 100644 --- a/src/video_core/buffer_cache/buffer_cache.h +++ b/src/video_core/buffer_cache/buffer_cache.h @@ -127,6 +127,16 @@ void BufferCache

::TickFrame() { } geometry_stats = {}; } + if (frame_tick % 600 == 0) { + UniformKeep& keep = uniform_keep; + if (Nx2Profile::Enabled() || ubo_keep_verify) { + LOG_INFO(HW_GPU, + "Uniform keeps in 600 frames: {} kept, {} streamed, {} verified, {} verify " + "mismatches", + keep.kept, keep.streamed_binds, keep.verified, keep.mismatches); + } + keep.kept = keep.streamed_binds = keep.verified = keep.mismatches = 0; + } if (frame_tick % 600 == 0 && !uploaded_buffer_ranges.empty()) { for (const auto& [address, size] : uploaded_buffer_ranges) { device_memory.HintCold(address, size); @@ -208,6 +218,7 @@ void BufferCache

::UnmapGPUMemory(size_t as_id, GPUVAddr gpu_addr, size_t size template void BufferCache

::WriteMemory(DAddr device_addr, u64 size, bool keep_page_gpu_data) { Nx2NoteListHostWrite(device_addr, size); + NoteUniformWrite(device_addr, size); // TEST (EDEN_NX2_READ_CHECK): a newer value where an early download landed, as ClearDownload // forgets one over GPU-modified bytes: without early downloads the write had taken the bytes // out of the GPU-modified ones, and the read's synchronous flush leaves them. Landed bytes @@ -283,6 +294,7 @@ void BufferCache

::CachedWriteMemory(DAddr device_addr, u64 size) { } Nx2NoteListHostWrite(device_addr, size); + NoteUniformWrite(device_addr, size); tmp_buffer.resize_destructive(size); device_memory.ReadBlockUnsafe(device_addr, tmp_buffer.data(), size); @@ -1749,6 +1761,7 @@ bool BufferCache

::DMACopy(GPUVAddr src_address, GPUVAddr dest_address, u64 am } if (has_new_downloads) { memory_tracker.MarkRegionAsGpuModified(*cpu_dest_address, amount); + NoteUniformWrite(*cpu_dest_address, amount); const bool should_sync = Settings::IsGPUFenceBehaviorBalanced() || Settings::IsGPUFenceBehaviorAccurate(); if (should_sync) { runtime.Finish(); @@ -1998,6 +2011,48 @@ void BufferCache

::SetUniformBuffersState(const std::array& m channel_state->uniform_buffer_sizes = sizes; } +template +void BufferCache

::BeginGraphicsUniforms(const Nx2FastDraw::Delta* delta) { + UniformKeep& keep = uniform_keep; + const bool unwritten = !keep.memory_writes_lost && keep.draw_deletions == buffer_deletions; + keep.draw_deletions = buffer_deletions; + keep.candidates = {}; + keep.writes.clear(); + const std::array streamed = std::exchange(keep.streamed, {}); + const bool usable = delta != nullptr && unwritten; + if (usable) { + keep.writes.insert(keep.writes.end(), keep.memory_writes.begin(), + keep.memory_writes.end()); + } + keep.memory_writes.clear(); + keep.memory_writes_lost = false; + if (!usable) { + return; + } + // The device memory inline constant data landed on (writes to unmapped addresses land + // nowhere). + for (u32 write = 0; write < delta->num_device_writes; ++write) { + keep.writes.emplace_back(delta->device_begin[write], delta->device_end[write]); + } + for (size_t stage = 0; stage < NUM_STAGES; ++stage) { + keep.candidates[stage] = streamed[stage] & ~delta->binds[stage]; + } +} + +template +bool BufferCache

::UniformRangeKept(size_t stage, u32 index, DAddr device_addr, + u32 size) const noexcept { + const UniformKeep& keep = uniform_keep; + if (((keep.candidates[stage] >> index) & 1) == 0) { + return false; + } + const auto overlaps = [&](const std::pair& write) { + return write.first < device_addr + size && device_addr < write.second; + }; + return !keep.memory_writes_lost && std::ranges::none_of(keep.writes, overlaps) && + std::ranges::none_of(keep.memory_writes, overlaps); +} + template void BufferCache

::SetComputeUniformBufferState(u32 mask, const ComputeUniformBufferSizes* sizes) { @@ -2764,6 +2819,32 @@ void BufferCache

::BindHostGraphicsUniformBuffer(size_t stage, u32 index, u32 const u32 size = (std::min)(binding.size, (*channel_state->uniform_buffer_sizes)[stage][index]); Buffer& buffer = slot_buffers[binding.buffer_id]; TouchBuffer(buffer, binding.buffer_id); + bool verify_kept = false; + if constexpr (!IS_OPENGL && VideoCommon::StreamUniformReusable) { + if (UniformRangeKept(stage, index, device_addr, size)) { + if (ubo_keep_verify) { + ++uniform_keep.verified; + verify_kept = true; + if (!VideoCommon::KeptUniformHolds(device_memory, stage, index, + device_addr, size)) { + if (uniform_keep.mismatches++ < 16) { + LOG_ERROR(HW_GPU, + "Uniform keep verify: stage {} slot {} at {:#x}+{:#x} changed " + "since the previous draw", + stage, index, device_addr, size); + } + } + } else if (VideoCommon::RebindKeptUniform(runtime, stage, index, device_addr, size)) { + channel_state->fast_bound_uniform_buffers[stage] |= 1u << binding_index; + channel_state->uniform_buffer_binding_sizes[stage][binding_index] = size; + uniform_keep.streamed[stage] |= 1u << index; + ++uniform_keep.kept; + Nx2Profile::AddFine(Nx2Profile::FineCounter::UniformRebinds, 1); + Nx2Profile::AddFine(Nx2Profile::FineCounter::UniformBytes, size); + return; + } + } + } const bool has_host_buffer = binding.buffer_id != NULL_BUFFER_ID; const u32 offset = has_host_buffer ? buffer.Offset(device_addr) : 0; const bool needs_alignment_stream = [&]() { @@ -2851,6 +2932,10 @@ void BufferCache

::BindHostGraphicsUniformBuffer(size_t stage, u32 index, u32 Nx2Profile::AddFine(Nx2Profile::FineCounter::UniformRebinds, rebound); Nx2Profile::AddFine(Nx2Profile::FineCounter::UniformStreamed, streamed ? 1 - rebound : 0); + if (streamed) { + uniform_keep.streamed[stage] |= 1u << index; + ++uniform_keep.streamed_binds; + } } } if (!streamed) { @@ -2862,6 +2947,11 @@ void BufferCache

::BindHostGraphicsUniformBuffer(size_t stage, u32 index, u32 } Nx2Profile::AddFine(Nx2Profile::FineCounter::UniformCached, 1); // Classic cached path + if (verify_kept && uniform_keep.mismatches++ < 16) { + LOG_ERROR(HW_GPU, + "Uniform keep verify: stage {} slot {} at {:#x}+{:#x} is no longer streamed", + stage, index, device_addr, size); + } if (SynchronizeBuffer(buffer, device_addr, size)) { ++channel_state->uniform_cache_hits[0]; } @@ -3544,6 +3634,7 @@ void BufferCache

::MarkWrittenBuffer(BufferId buffer_id, DAddr device_addr, u3 OutdateReadDownloads(device_addr, size); NoteReadBackWrite(device_addr, size); memory_tracker.MarkRegionAsGpuModified(device_addr, size); + NoteUniformWrite(device_addr, size); MarkVertexBuffersStale(device_addr, size); gpu_modified_ranges.Add(device_addr, size); QueueFenceDownload(device_addr, size); diff --git a/src/video_core/buffer_cache/buffer_cache_base.h b/src/video_core/buffer_cache/buffer_cache_base.h index 1ef853554b..4c89fdc650 100644 --- a/src/video_core/buffer_cache/buffer_cache_base.h +++ b/src/video_core/buffer_cache/buffer_cache_base.h @@ -11,6 +11,7 @@ #include #include #include +#include #include #include #include @@ -48,6 +49,7 @@ #include "video_core/memory_manager.h" #include "video_core/nx2.h" #include "video_core/nx2_download_trace.h" +#include "video_core/nx2_fast_draw.h" #include "video_core/nx2_read_trace.h" #include "video_core/rasterizer_download_area.h" #include "video_core/surface.h" @@ -520,6 +522,11 @@ class BufferCache : public VideoCommon::ChannelSetupCacheshas_deleted_buffers); } + /// Called before each graphics draw binds its buffers (uniform keeps). `delta`: the constant + /// buffer binds and data since the previous draw when it is a fast draw delta draw, else + /// nullptr. + void BeginGraphicsUniforms(const Nx2FastDraw::Delta* delta); + Common::SpinRecursiveMutex mutex; Runtime& runtime; bool any_buffer_uploaded = false; @@ -782,6 +789,43 @@ class BufferCache : public VideoCommon::ChannelSetupCaches candidates{}; ///< Streamed by the previous draw, not rebound + std::array streamed{}; ///< Streamed by this draw + /// Device ranges written since the previous draw began: inline constant data, then the + /// CPU writes and GPU-modified marks. + boost::container::static_vector, Nx2FastDraw::Delta::MaxWrites + 8> + writes; + boost::container::static_vector, 8> memory_writes; + bool memory_writes_lost{}; ///< More writes than memory_writes holds + u64 draw_deletions{}; ///< buffer_deletions when the previous draw began + u64 kept{}; + u64 streamed_binds{}; + u64 verified{}; + u64 mismatches{}; + } uniform_keep; + boost::container::small_vector upload_copies; MemoryTracker memory_tracker; diff --git a/src/video_core/buffer_cache/uniform_stream_reuse.h b/src/video_core/buffer_cache/uniform_stream_reuse.h index 43a85244f2..defc07b55e 100644 --- a/src/video_core/buffer_cache/uniform_stream_reuse.h +++ b/src/video_core/buffer_cache/uniform_stream_reuse.h @@ -61,7 +61,8 @@ class UniformStreamReuse { struct Slot { Ref ref{}; u64 tick{}; - u32 size{}; // 0: nothing streamed yet + u64 device_addr{}; ///< Where `bytes` were last read + u32 size{}; // 0: nothing streamed yet std::array bytes{}; }; @@ -120,6 +121,7 @@ template guest = reuse.scratch.data(); } const bool same_bytes = slot.size == size && std::memcmp(slot.bytes.data(), guest, size) == 0; + slot.device_addr = device_addr; if (same_bytes && slot.tick == runtime.CurrentTick()) { runtime.RebindStreamUniform(slot.ref, size); ++reuse.rebinds; @@ -139,4 +141,45 @@ template return true; } +/// Uniform keeps: binds the slot's last allocation again without reading guest memory, +/// when that bind read the same range in this command buffer. The caller guarantees that nothing +/// ordered before this draw wrote the range since that bind (the previous draw's). Returns false +/// (nothing bound) otherwise. +template +[[nodiscard]] bool RebindKeptUniform(Runtime& runtime, size_t stage, u32 index, u64 device_addr, + u32 size) { + using Reuse = UniformStreamReuse; + if (size == 0 || size > Reuse::MaxBytes || stage >= Reuse::NumStages || + index >= Reuse::NumSlots) { + return false; + } + const typename Reuse::Slot& slot = + UniformStreamReuseState().slots[stage][index]; + if (slot.size != size || slot.device_addr != device_addr || + slot.tick != runtime.CurrentTick()) { + return false; + } + runtime.RebindStreamUniform(slot.ref, size); + return true; +} + +/// EDEN_NX2_UBO_KEEP_VERIFY: whether guest memory still holds the bytes the slot's last +/// bind read from the range RebindKeptUniform would reuse. +template +[[nodiscard]] bool KeptUniformHolds(DeviceMemory& device_memory, size_t stage, u32 index, + u64 device_addr, u32 size) { + using Reuse = UniformStreamReuse; + if (size == 0 || size > Reuse::MaxBytes || stage >= Reuse::NumStages || + index >= Reuse::NumSlots) { + return false; + } + Reuse& reuse = UniformStreamReuseState(); + const typename Reuse::Slot& slot = reuse.slots[stage][index]; + if (slot.size != size || slot.device_addr != device_addr) { + return false; + } + device_memory.ReadBlockUnsafe(device_addr, reuse.scratch.data(), size); + return std::memcmp(slot.bytes.data(), reuse.scratch.data(), size) == 0; +} + } // namespace VideoCommon diff --git a/src/video_core/nx2_fast_draw.h b/src/video_core/nx2_fast_draw.h index 13cbb253f0..9e90c92930 100644 --- a/src/video_core/nx2_fast_draw.h +++ b/src/video_core/nx2_fast_draw.h @@ -17,9 +17,11 @@ class MemoryManager; /// A draw that follows a draw of the same pipeline with only light state changes in between takes /// the delta path. It keeps the previous draw's pipeline, image views, samplers and texture buffer -/// bindings, and resolves and binds every buffer (index, vertex, uniform, storage, texture) again -/// as a full resolve does, so guest memory written without a notification is streamed as in a -/// full resolve. +/// bindings. Buffers are resolved and bound again, except small streamed uniforms whose ranges +/// were not rebound or written: those can keep the previous stream allocation in the same command +/// buffer. This assumes no unsynchronized guest CPU writes unknown to the caches between draws; +/// EDEN_NX2_UBO_KEEP_VERIFY checks that assumption. Kept image descriptors and rescaling words are +/// replayed while the views remain valid; EDEN_NX2_DELTA_DESCRIPTORS_VERIFY checks the replay. /// Light changes are draw parameters, index buffer and vertex stream registers, constant buffer /// binds and inline constant buffer data (collected in g_delta: the rasterizer resolves in full /// when they reach the pipeline's texture handles at their GPU addresses or through another diff --git a/src/video_core/renderer_vulkan/pipeline_helper.h b/src/video_core/renderer_vulkan/pipeline_helper.h index bd513b4e53..4fd0232300 100644 --- a/src/video_core/renderer_vulkan/pipeline_helper.h +++ b/src/video_core/renderer_vulkan/pipeline_helper.h @@ -373,6 +373,11 @@ class RescalingPushConstant { return words; } + /// The words of an earlier draw's pushes (delta draws); push no more. + void Assign(const std::array& data) noexcept { + words = data; + } + private: std::array words{}; u32* texture_ptr{words.data()}; diff --git a/src/video_core/renderer_vulkan/vk_graphics_pipeline.cpp b/src/video_core/renderer_vulkan/vk_graphics_pipeline.cpp index 5b75a62278..278e8ed695 100644 --- a/src/video_core/renderer_vulkan/vk_graphics_pipeline.cpp +++ b/src/video_core/renderer_vulkan/vk_graphics_pipeline.cpp @@ -299,6 +299,42 @@ bool TextureMemoVerify() { return verify; } +// EDEN_NX2_DELTA_DESCRIPTORS_VERIFY=1: delta draws resolve their image descriptors as before and +// compare them with the kept ones. +const bool delta_descriptors_verify = [] { + const char* const value = std::getenv("EDEN_NX2_DELTA_DESCRIPTORS_VERIFY"); + return value != nullptr && value[0] == '1'; +}(); + +// Delta draws on the drawing thread, logged every 10 s with EDEN_NX2_GPU_PROFILE or the verify +// env. +struct DeltaDescriptorStats { + u64 replayed{}; + u64 pushed{}; + u64 verified{}; + u64 mismatches{}; + std::chrono::steady_clock::time_point next_report{}; +} delta_descriptor_stats; + +void ReportDeltaDescriptors() { + DeltaDescriptorStats& stats = delta_descriptor_stats; + if (((stats.replayed + stats.pushed) & 1023) != 0 || + (!VideoCommon::Nx2Profile::Enabled() && !delta_descriptors_verify)) { + return; + } + const auto now = std::chrono::steady_clock::now(); + if (now < stats.next_report) { + return; + } + if (stats.next_report != std::chrono::steady_clock::time_point{}) { + LOG_INFO(Render_Vulkan, + "Delta descriptors in 10 s: {} replayed, {} pushed, {} verified, {} verify " + "mismatches", + stats.replayed, stats.pushed, stats.verified, stats.mismatches); + } + stats = {.next_report = now + std::chrono::seconds{10}}; +} + // Texture memo counts, logged every 10 s with EDEN_NX2_GPU_PROFILE or the verify env. std::atomic g_memo_hits{}; std::atomic g_memo_misses{}; @@ -854,7 +890,9 @@ bool GraphicsPipeline::ConfigureImpl(bool is_indexed) { } } } - if (std::exchange(capture_delta, false) && delta_blocker == DeltaBlocker::None) { + const bool captured_delta = + std::exchange(capture_delta, false) && delta_blocker == DeltaBlocker::None; + if (captured_delta) { const auto same_view = [](const VideoCommon::ImageViewInOut& lhs, const VideoCommon::ImageViewInOut& rhs) { return lhs.index == rhs.index && lhs.id == rhs.id; @@ -868,6 +906,10 @@ bool GraphicsPipeline::ConfigureImpl(bool is_indexed) { delta_samplers.assign(samplers.begin(), samplers.end()); configured_stages = Spec::enabled_stages; delta_views_valid = true; + kept_images_valid = false; + } + if (captured_delta) { + kept_image_entries.clear(); } if constexpr (Spec::has_texture_buffers || Spec::has_image_buffers) { @@ -974,8 +1016,12 @@ bool GraphicsPipeline::ConfigureImpl(bool is_indexed) { } { Profile::Scope scope{Profile::Zone::PushDescriptors}; + const size_t first_entry = guest_descriptor_queue.UpdateCount(); PushImageDescriptors(texture_cache, guest_descriptor_queue, stage_infos[stage], rescaling, samplers_it, views_it); + if (captured_delta) { + KeepImageDescriptors(stage, first_entry); + } } const auto& info{stage_infos[stage]}; if (info.uses_render_area) { @@ -1002,6 +1048,11 @@ bool GraphicsPipeline::ConfigureImpl(bool is_indexed) { if constexpr (Spec::enabled_stages[4]) { prepare_stage(4); } + if (captured_delta) { + kept_rescaling = rescaling.Data(); + kept_scale_epoch = texture_cache.ScaleEpoch(); + kept_images_valid = true; + } if (mesh_emulation) { if (!mesh_output_buffer) { return false; @@ -1079,6 +1130,22 @@ bool GraphicsPipeline::ConfigureDelta(bool is_indexed) { RenderAreaPushConstant render_area; const VideoCommon::SamplerId* samplers_it{delta_samplers.data()}; const VideoCommon::ImageViewInOut* views_it{delta_views.data()}; + // Push the entries kept for delta_views instead of resolving every view and sampler again; + // with EDEN_NX2_DELTA_DESCRIPTORS_VERIFY resolve them and compare. + const bool kept_usable = kept_images_valid && kept_scale_epoch == texture_cache.ScaleEpoch(); + const bool replay_images = kept_usable && !delta_descriptors_verify; + struct Kept { + std::vector entries; + std::array, NUM_STAGES> ranges; + std::array rescaling; + }; + std::optional verify_kept; + if (!replay_images) { + if (kept_usable) { + verify_kept.emplace(std::move(kept_image_entries), kept_image_ranges, kept_rescaling); + } + kept_image_entries.clear(); + } for (size_t stage = 0; stage < NUM_STAGES; ++stage) { if (!configured_stages[stage]) { continue; @@ -1087,10 +1154,18 @@ bool GraphicsPipeline::ConfigureDelta(bool is_indexed) { Profile::Scope scope{Profile::Zone::BindStageBuffers}; buffer_cache.BindHostStageBuffers(stage); } - { + if (replay_images) { + Profile::Scope scope{Profile::Zone::PushDescriptors}; + const auto [first, count] = kept_image_ranges[stage]; + if (count != 0) { + guest_descriptor_queue.AddEntries({kept_image_entries.data() + first, count}); + } + } else { Profile::Scope scope{Profile::Zone::PushDescriptors}; + const size_t first_entry = guest_descriptor_queue.UpdateCount(); PushImageDescriptors(texture_cache, guest_descriptor_queue, stage_infos[stage], rescaling, samplers_it, views_it); + KeepImageDescriptors(stage, first_entry); } if (stage_infos[stage].uses_render_area) { render_area.uses_render_area = true; @@ -1100,6 +1175,34 @@ bool GraphicsPipeline::ConfigureDelta(bool is_indexed) { static_cast(clip_set ? clip_height : u32{regs.rt[0].height})}; } } + if (replay_images) { + rescaling.Assign(kept_rescaling); + ++delta_descriptor_stats.replayed; + } else { + kept_rescaling = rescaling.Data(); + kept_scale_epoch = texture_cache.ScaleEpoch(); + kept_images_valid = true; + ++delta_descriptor_stats.pushed; + if (verify_kept) { + ++delta_descriptor_stats.verified; + const auto same_image = [](const DescriptorUpdateEntry& lhs, + const DescriptorUpdateEntry& rhs) { + return lhs.image.sampler == rhs.image.sampler && + lhs.image.imageView == rhs.image.imageView && + lhs.image.imageLayout == rhs.image.imageLayout; + }; + if ((verify_kept->ranges != kept_image_ranges || + verify_kept->rescaling != kept_rescaling || + !std::ranges::equal(verify_kept->entries, kept_image_entries, same_image)) && + delta_descriptor_stats.mismatches++ < 16) { + LOG_ERROR(Render_Vulkan, + "Delta descriptors verify: pipeline {:016x} resolved other image " + "descriptors or rescaling words than the previous draw", + key.Hash()); + } + } + } + ReportDeltaDescriptors(); if (buffer_cache.any_buffer_uploaded) { buffer_cache.runtime.PostCopyBarrier(); buffer_cache.any_buffer_uploaded = false; @@ -1121,6 +1224,15 @@ bool GraphicsPipeline::ConfigureDelta(bool is_indexed) { return ConfigureDraw(rescaling, render_area); } +void GraphicsPipeline::KeepImageDescriptors(size_t stage, size_t first_entry) { + const DescriptorUpdateEntry* const entries = guest_descriptor_queue.UpdateData(); + const size_t end_entry = guest_descriptor_queue.UpdateCount(); + kept_image_ranges[stage] = {static_cast(kept_image_entries.size()), + static_cast(end_entry - first_entry)}; + kept_image_entries.insert(kept_image_entries.end(), entries + first_entry, + entries + end_entry); +} + bool GraphicsPipeline::ConfigureDraw(const RescalingPushConstant& rescaling, const RenderAreaPushConstant& render_area) { if (guest_descriptor_queue.UpdateCount() != num_descriptor_entries) { diff --git a/src/video_core/renderer_vulkan/vk_graphics_pipeline.h b/src/video_core/renderer_vulkan/vk_graphics_pipeline.h index 631e21a06f..09607fdeb8 100644 --- a/src/video_core/renderer_vulkan/vk_graphics_pipeline.h +++ b/src/video_core/renderer_vulkan/vk_graphics_pipeline.h @@ -337,6 +337,8 @@ class GraphicsPipeline { template bool ConfigureImpl(bool is_indexed); + void KeepImageDescriptors(size_t stage, size_t first_entry); + bool ConfigureDraw(const RescalingPushConstant& rescaling, const RenderAreaPushConstant& render_are); @@ -389,6 +391,14 @@ class GraphicsPipeline { std::array handle_constant_buffers{}; std::vector delta_views; std::vector delta_samplers; + /// Delta draws: the image descriptor entries (first entry and count per + /// stage) and rescaling words pushed for delta_views, which delta draws push again while no + /// image changed its scale (TextureCache::ScaleEpoch). + std::vector kept_image_entries; + std::array, NUM_STAGES> kept_image_ranges{}; + std::array kept_rescaling{}; + u64 kept_scale_epoch{}; + bool kept_images_valid{}; vk::DescriptorSetLayout descriptor_set_layout; DescriptorAllocator descriptor_allocator; diff --git a/src/video_core/renderer_vulkan/vk_rasterizer.cpp b/src/video_core/renderer_vulkan/vk_rasterizer.cpp index 97841c2a38..43f8606b7f 100644 --- a/src/video_core/renderer_vulkan/vk_rasterizer.cpp +++ b/src/video_core/renderer_vulkan/vk_rasterizer.cpp @@ -457,6 +457,7 @@ GraphicsPipeline* RasterizerVulkan::TakeReusableDraw(bool is_indexed, bool is_in namespace FastDraw = VideoCommon::Nx2FastDraw; const ReusableDraw draw = std::exchange(reusable_draw, ReusableDraw{}); const FastDraw::Delta delta = std::exchange(FastDraw::g_delta, FastDraw::Delta{}); + taken_delta = delta; const auto full = [&](FullDraw reason) -> GraphicsPipeline* { ++fast_draw_stats.full[static_cast(reason)]; return nullptr; @@ -1042,6 +1043,7 @@ void RasterizerVulkan::PrepareDraw(bool is_indexed, Func&& draw_func, bool is_in const bool delta_holds = delta_pipeline != nullptr && DeltaStillHolds(draw_serial, draw_map_serial); const DrawPath path = delta_holds && !verify_pipeline ? DrawPath::Delta : DrawPath::Full; + buffer_cache.BeginGraphicsUniforms(path == DrawPath::Delta ? &taken_delta : nullptr); if (path == DrawPath::Delta) { if (Profile::Scope scope{Profile::Zone::Configure}; !pipeline->ConfigureDelta(is_indexed)) { report_failure("configure failed"); diff --git a/src/video_core/renderer_vulkan/vk_rasterizer.h b/src/video_core/renderer_vulkan/vk_rasterizer.h index 572d042d45..3c35d4a0fc 100644 --- a/src/video_core/renderer_vulkan/vk_rasterizer.h +++ b/src/video_core/renderer_vulkan/vk_rasterizer.h @@ -453,6 +453,8 @@ class RasterizerVulkan final : public VideoCore::RasterizerInterface, std::array, Maxwell::MaxShaderStage> handle_spans; + /// The constant buffer binds and data since the previous draw. + VideoCommon::Nx2FastDraw::Delta taken_delta; FastDrawStats fast_draw_stats; /// Nx2FastDraw::Enabled and VerifyEnabled as the drawing thread last read them. bool fast_draw_on{}; diff --git a/src/video_core/renderer_vulkan/vk_update_descriptor.h b/src/video_core/renderer_vulkan/vk_update_descriptor.h index 6b22c5c3ff..3e1cf9df3e 100644 --- a/src/video_core/renderer_vulkan/vk_update_descriptor.h +++ b/src/video_core/renderer_vulkan/vk_update_descriptor.h @@ -6,7 +6,9 @@ #pragma once +#include #include +#include #include #include #include "video_core/vulkan_common/vulkan_wrapper.h" @@ -105,6 +107,12 @@ class UpdateDescriptorQueue final { *(payload_cursor++) = texel_buffer; } + /// Entries an earlier draw added (delta draws). + void AddEntries(std::span entries) { + std::memcpy(payload_cursor, entries.data(), entries.size_bytes()); + payload_cursor += entries.size(); + } + void AddTexelBuffer(VkBufferView texel_buffer, VkDeviceAddress base_address, VkDeviceSize offset, VkDeviceSize size, VkFormat format) { if (!use_descriptor_buffer) { diff --git a/src/video_core/texture_cache/texture_cache.h b/src/video_core/texture_cache/texture_cache.h index fad5fbca7b..328022c528 100644 --- a/src/video_core/texture_cache/texture_cache.h +++ b/src/video_core/texture_cache/texture_cache.h @@ -2543,6 +2543,7 @@ template void TextureCache

::InvalidateScale(Image& image) { // Its views are deleted: a fast draw delta draw keeping one must not follow. VideoCommon::Nx2FastDraw::Break(VideoCommon::Nx2FastDraw::Reason::Invalidation); + ++scale_epoch; if (image.scale_tick <= frame_tick) { image.scale_tick = frame_tick + 1; } diff --git a/src/video_core/texture_cache/texture_cache_base.h b/src/video_core/texture_cache/texture_cache_base.h index 0b2b34d405..c52c4cb040 100644 --- a/src/video_core/texture_cache/texture_cache_base.h +++ b/src/video_core/texture_cache/texture_cache_base.h @@ -184,6 +184,7 @@ class TextureCache : public VideoCommon::ChannelSetupCaches astc_scan_buffer; u64 images_deleted{}; + u64 scale_epoch{}; std::map> created_shapes; bool debug_stats{}; ///< Collect what only the debug statistics lines use @@ -413,6 +414,11 @@ class TextureCache : public VideoCommon::ChannelSetupCaches Date: Mon, 9 Nov 2026 02:29:13 +0100 Subject: [PATCH 101/105] video_core: keep vertex_id_base writes from breaking fast draws NVN writes the first vertex to vertex_id_base before each draw. No part of a draw's resolve reads it, yet every change counted as a state change, so draws after one took the full path. In Ocarina of Time that was the most common fast draw break (21k to 36k per second during play). (cherry picked from commit d6fd421e49ae096ef19b53c7600632cccae5992a) --- src/tests/video_core/fast_draw.cpp | 8 ++++++++ src/video_core/engines/maxwell_3d.cpp | 2 ++ 2 files changed, 10 insertions(+) diff --git a/src/tests/video_core/fast_draw.cpp b/src/tests/video_core/fast_draw.cpp index bf5a0a6206..3ca6f1a72a 100644 --- a/src/tests/video_core/fast_draw.cpp +++ b/src/tests/video_core/fast_draw.cpp @@ -345,6 +345,14 @@ TEST_CASE("Fast draw keeps base vertex and instance as draw parameters", "[fast_ CHECK(f.Reusable()); } +TEST_CASE("Fast draw keeps the vertex id base as a draw parameter", "[fast_draw]") { + FastDrawFixture f; + f.DrawArrays(0, 3); + f.Write(MAXWELL3D_REG_INDEX(vertex_id_base), f.Reg(MAXWELL3D_REG_INDEX(vertex_id_base)) + 3); + f.DrawArrays(3, 3); + CHECK(f.Reusable()); +} + TEST_CASE("Fast draw keeps vertex stream addresses and sizes light", "[fast_draw]") { DeltaFixture f; const auto stream = [](u32 index, u32 word) { diff --git a/src/video_core/engines/maxwell_3d.cpp b/src/video_core/engines/maxwell_3d.cpp index 3f2839b712..f644f57923 100644 --- a/src/video_core/engines/maxwell_3d.cpp +++ b/src/video_core/engines/maxwell_3d.cpp @@ -357,6 +357,8 @@ bool Maxwell3D::IsFastDrawParameter(u32 method) { case MAXWELL3D_REG_INDEX(const_buffer.offset): case MAXWELL3D_REG_INDEX(global_base_vertex_index): case MAXWELL3D_REG_INDEX(global_base_instance_index): + // Written with the first vertex before each draw; nothing a draw resolves reads it. + case MAXWELL3D_REG_INDEX(vertex_id_base): case MAXWELL3D_REG_INDEX(index_buffer.start_addr_high): case MAXWELL3D_REG_INDEX(index_buffer.start_addr_low): case MAXWELL3D_REG_INDEX(index_buffer.limit_addr_high): From 8a452bcdb3b8de4b1923ee447e5614f70d0c0211 Mon Sep 17 00:00:00 2001 From: mstampfli Date: Mon, 9 Nov 2026 02:50:39 +0100 Subject: [PATCH 102/105] video_core: keep counter resets and render enable writes from breaking fast draws A counter reset only touches the query cache, and the render enable condition decides whether the next draws run, not what they bind. Both ended draw reuse as executable methods. With vertex_id_base no longer breaking it, they were the most common fast draw breaks in Ocarina of Time (11k to 23k and 6k to 12k per second during play). A render pass either one ends is still caught by the reusable draw's render pass check. (cherry picked from commit b39d94b4da70903436ac8235c43f2850b87ab2ed) --- src/tests/video_core/fast_draw.cpp | 13 +++++++++++++ src/video_core/engines/maxwell_3d.cpp | 18 ++++++++++++++++-- src/video_core/engines/maxwell_3d.h | 3 +++ 3 files changed, 32 insertions(+), 2 deletions(-) diff --git a/src/tests/video_core/fast_draw.cpp b/src/tests/video_core/fast_draw.cpp index 3ca6f1a72a..19d8de85c3 100644 --- a/src/tests/video_core/fast_draw.cpp +++ b/src/tests/video_core/fast_draw.cpp @@ -345,6 +345,19 @@ TEST_CASE("Fast draw keeps base vertex and instance as draw parameters", "[fast_ CHECK(f.Reusable()); } +TEST_CASE("Fast draw keeps counter resets and the render enable condition light", "[fast_draw]") { + FastDrawFixture f; + f.DrawArrays(0, 3); + constexpr u32 clear_report = MAXWELL3D_REG_INDEX(clear_report_value); + f.Write(clear_report, f.Reg(clear_report) ^ 1); + f.DrawArrays(3, 3); + CHECK(f.Reusable()); + f.Write(MAXWELL3D_REG_INDEX(render_enable.mode), 1); // True: the next draws run + f.DrawArrays(6, 3); + CHECK(f.rasterizer.draws.size() == 3); + CHECK(f.Reusable()); +} + TEST_CASE("Fast draw keeps the vertex id base as a draw parameter", "[fast_draw]") { FastDrawFixture f; f.DrawArrays(0, 3); diff --git a/src/video_core/engines/maxwell_3d.cpp b/src/video_core/engines/maxwell_3d.cpp index f644f57923..5266644cc3 100644 --- a/src/video_core/engines/maxwell_3d.cpp +++ b/src/video_core/engines/maxwell_3d.cpp @@ -367,7 +367,20 @@ bool Maxwell3D::IsFastDrawParameter(u32 method) { return true; default: return DrawManager::IsInstanceLoopMethod(method) || IsFastDrawDeltaMethod(method) || - IsFastDrawVertexBuffer(method); + IsFastDrawVertexBuffer(method) || IsFastDrawNeutralMethod(method); + } +} + +bool Maxwell3D::IsFastDrawNeutralMethod(u32 method) { + switch (method) { + // A counter reset only touches the query cache. The render enable condition decides whether + // the next draws run (ShouldExecute, or host conditional rendering), not what they bind; a + // render pass either one ends is caught by the reusable draw's render pass check. + case MAXWELL3D_REG_INDEX(clear_report_value): + case MAXWELL3D_REG_INDEX(render_enable.mode): + return true; + default: + return false; } } @@ -403,7 +416,8 @@ bool Maxwell3D::IsFastDrawDeltaMethod(u32 method) { } void Maxwell3D::ProcessMethodCall(u32 method, u32 argument, u32 nonshadow_argument, bool is_last_call) { - if (!DrawManager::IsInstanceLoopMethod(method) && !IsFastDrawDeltaMethod(method)) { + if (!DrawManager::IsInstanceLoopMethod(method) && !IsFastDrawDeltaMethod(method) && + !IsFastDrawNeutralMethod(method)) { VideoCommon::Nx2FastDraw::Break(VideoCommon::Nx2FastDraw::Reason::Method, method); } if (GetNativeMacroEngine().native_turing && method == MAXWELL3D_REG_INDEX(clear_surface)) { diff --git a/src/video_core/engines/maxwell_3d.h b/src/video_core/engines/maxwell_3d.h index 1e3dcd3eef..bfc4d70bc4 100644 --- a/src/video_core/engines/maxwell_3d.h +++ b/src/video_core/engines/maxwell_3d.h @@ -3280,6 +3280,9 @@ class Maxwell3D final : public EngineInterface { static bool IsFastDrawDeltaMethod(u32 method); /// A vertex stream's address, size or limit (not its stride or enable). static bool IsFastDrawVertexBuffer(u32 method); + /// Executable methods whose work leaves what a draw resolves to alone (counter resets, the + /// render enable condition): fast draw records nothing for them. + static bool IsFastDrawNeutralMethod(u32 method); void NoteFastDrawConstantData(GPUVAddr begin, GPUVAddr end); /// Handles a write to the CB_DATA[i] register. From d252c5b97dddc96d73a1059efeaad65e16eb5c08 Mon Sep 17 00:00:00 2001 From: mstampfli Date: Mon, 9 Nov 2026 03:13:42 +0100 Subject: [PATCH 103/105] video_core: keep per-draw reports from breaking fast draws Ocarina of Time writes a report semaphore address and releases a report around most draws, and points the macro engine's memory reads somewhere new. Each ended draw reuse, and together they were behind most of the remaining full resolves (17k to 29k draws per second during play). The report's operands and the macro memory addresses are now draw parameters. The release itself is recorded as constant data at the address it writes: a delta draw whose texture handles live there, or a write into the texture or sampler headers, still takes the full path. (cherry picked from commit 7e843d3245d290977c058fd35b996d5dd47f186b) --- src/tests/video_core/fast_draw.cpp | 29 +++++++++++++++++++++++++ src/video_core/engines/maxwell_3d.cpp | 31 ++++++++++++++++++++++++++- src/video_core/engines/maxwell_3d.h | 2 ++ 3 files changed, 61 insertions(+), 1 deletion(-) diff --git a/src/tests/video_core/fast_draw.cpp b/src/tests/video_core/fast_draw.cpp index 19d8de85c3..e4fbccbf61 100644 --- a/src/tests/video_core/fast_draw.cpp +++ b/src/tests/video_core/fast_draw.cpp @@ -358,6 +358,35 @@ TEST_CASE("Fast draw keeps counter resets and the render enable condition light" CHECK(f.Reusable()); } +TEST_CASE("Fast draw records a report's memory write instead of breaking on it", "[fast_draw]") { + DeltaFixture f; + f.Write(MAXWELL3D_REG_INDEX(tex_header.address_high), 0); + f.Write(MAXWELL3D_REG_INDEX(tex_header.address_low), 0x40000); + f.Write(MAXWELL3D_REG_INDEX(tex_header.limit), 7); + const auto report = [&](u32 address) { + f.Write(MAXWELL3D_REG_INDEX(report_semaphore.address_high), 0); + f.Write(MAXWELL3D_REG_INDEX(report_semaphore.address_low), address); + f.Write(MAXWELL3D_REG_INDEX(report_semaphore.payload), + f.Reg(MAXWELL3D_REG_INDEX(report_semaphore.payload)) + 1); + f.Write(MAXWELL3D_REG_INDEX(report_semaphore.query), 0); // long release + }; + f.DrawArrays(0, 3); + const u64 serial = FastDraw::Serial(); + FastDraw::g_delta = {}; + f.Write(0x155, f.Reg(0x155) + 0x40); // SET_MME_MEM_ADDRESS_B + report(0x30000); + CHECK(FastDraw::Serial() == serial); + REQUIRE(FastDraw::g_delta.num_writes == 1); + CHECK(FastDraw::g_delta.write_begin[0] == 0x30000); + CHECK(FastDraw::g_delta.write_end[0] == 0x30010); + f.DrawArrays(3, 3); + CHECK(f.Reusable()); + + // Into the texture headers a delta draw keeps: a full resolve. + report(0x40020); + CHECK(FastDraw::Serial() != serial); +} + TEST_CASE("Fast draw keeps the vertex id base as a draw parameter", "[fast_draw]") { FastDrawFixture f; f.DrawArrays(0, 3); diff --git a/src/video_core/engines/maxwell_3d.cpp b/src/video_core/engines/maxwell_3d.cpp index 5266644cc3..241ba939c3 100644 --- a/src/video_core/engines/maxwell_3d.cpp +++ b/src/video_core/engines/maxwell_3d.cpp @@ -367,7 +367,8 @@ bool Maxwell3D::IsFastDrawParameter(u32 method) { return true; default: return DrawManager::IsInstanceLoopMethod(method) || IsFastDrawDeltaMethod(method) || - IsFastDrawVertexBuffer(method) || IsFastDrawNeutralMethod(method); + IsFastDrawVertexBuffer(method) || IsFastDrawNeutralMethod(method) || + IsFastDrawReportOperand(method); } } @@ -378,6 +379,28 @@ bool Maxwell3D::IsFastDrawNeutralMethod(u32 method) { // render pass either one ends is caught by the reusable draw's render pass check. case MAXWELL3D_REG_INDEX(clear_report_value): case MAXWELL3D_REG_INDEX(render_enable.mode): + // A report or semaphore release writes guest memory, which ProcessQueryGet notes as constant + // data: a delta draw that reads texture handles there takes the full path. + case MAXWELL3D_REG_INDEX(report_semaphore.query): + return true; + default: + return false; + } +} + +bool Maxwell3D::IsFastDrawReportOperand(u32 method) { + // SET_MME_MEM_ADDRESS_A/B and the data RAM address: only the macro engine's memory reads use + // them, and what a macro does with the words reaches the engine as writes of its own. + constexpr u32 MmeMemAddressA = 0x154; + constexpr u32 MmeDataRamAddress = 0x156; + if (method >= MmeMemAddressA && method <= MmeDataRamAddress) { + return true; + } + switch (method) { + // Only the report_semaphore.query method reads them. + case MAXWELL3D_REG_INDEX(report_semaphore.address_high): + case MAXWELL3D_REG_INDEX(report_semaphore.address_low): + case MAXWELL3D_REG_INDEX(report_semaphore.payload): return true; default: return false; @@ -900,6 +923,12 @@ void Maxwell3D::ProcessQueryGet() { static_cast(regs.report_semaphore.query.report.Value()); const u32 payload = regs.report_semaphore.payload; const u32 subreport = regs.report_semaphore.query.sub_report; + if (VideoCommon::Nx2FastDraw::Enabled()) { + // The release lands in guest memory: a short one writes the payload, a long one the + // 16-byte report. + const u64 bytes = regs.report_semaphore.query.short_query != 0 ? 4 : 16; + NoteFastDrawConstantData(sequence_address, sequence_address + bytes); + } switch (regs.report_semaphore.query.operation) { case Regs::ReportSemaphore::Operation::Release: if (regs.report_semaphore.query.short_query != 0) { diff --git a/src/video_core/engines/maxwell_3d.h b/src/video_core/engines/maxwell_3d.h index bfc4d70bc4..21081cd585 100644 --- a/src/video_core/engines/maxwell_3d.h +++ b/src/video_core/engines/maxwell_3d.h @@ -3283,6 +3283,8 @@ class Maxwell3D final : public EngineInterface { /// Executable methods whose work leaves what a draw resolves to alone (counter resets, the /// render enable condition): fast draw records nothing for them. static bool IsFastDrawNeutralMethod(u32 method); + /// Registers only a report or a macro memory read takes as operands. + static bool IsFastDrawReportOperand(u32 method); void NoteFastDrawConstantData(GPUVAddr begin, GPUVAddr end); /// Handles a write to the CB_DATA[i] register. From 9bf7c43b26910d54575dc608bd6bafef83ce5fe4 Mon Sep 17 00:00:00 2001 From: Copilot App <223556219+Copilot@users.noreply.github.com> Date: Mon, 9 Nov 2026 13:13:22 -0300 Subject: [PATCH 104/105] video_core: keep synced payloads out of deferred report tracking The #1028 pending-report list retires callbacks in fence order. #874's synced payload callback runs immediately, possibly ahead of an older report at the same address. Do not register that callback as deferred or let it retire the older report, whose rewritten bytes must remain GPU-modified until its own landing. Preserve the existing is_synced/fenced_landed contract. Add regression coverage alongside the imported read-back tests and document the interaction. Validation: git diff --cached --check passed. C++ tests are not built (BUILD_TESTING=OFF); the requested integration build is still running and will be rerun for this commit. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- NX2_GPU_THREAD_SPLIT.md | 3 ++- src/tests/video_core/buffer_cache_ports.cpp | 17 +++++++++++++++++ src/video_core/query_cache/query_cache.h | 4 +++- 3 files changed, 22 insertions(+), 2 deletions(-) diff --git a/NX2_GPU_THREAD_SPLIT.md b/NX2_GPU_THREAD_SPLIT.md index d0d967b895..b87f6a8c0f 100644 --- a/NX2_GPU_THREAD_SPLIT.md +++ b/NX2_GPU_THREAD_SPLIT.md @@ -113,7 +113,8 @@ not predict the gain of this partial port in Ocarina of Time. profiling or diagnostics. Its correctness fixes are retained: discard/DMA destinations first land overlapping read downloads, and older deferred query results cannot mark newer GPU writes clean. Synced #874 payload callbacks still - do not advance `fenced_landed`; `FencedPayloadLanding` remains authoritative. + do not advance `fenced_landed` or retire an older deferred report's tracking; + `FencedPayloadLanding` remains authoritative. - #979/#1014 retain every registered semaphore region and the direct-release shadow. Mirrored releases are emitted only when `ViewApplies()` is false (unsplit, back macros, or delayed/high-accuracy releases). The full #805 front diff --git a/src/tests/video_core/buffer_cache_ports.cpp b/src/tests/video_core/buffer_cache_ports.cpp index 3b3cc1a36d..3fb2dea4b1 100644 --- a/src/tests/video_core/buffer_cache_ports.cpp +++ b/src/tests/video_core/buffer_cache_ports.cpp @@ -1469,6 +1469,23 @@ TEST_CASE("An older query result leaves overlapping GPU writes to be read again" REQUIRE(f.bytes[0x10043] == 3); } +TEST_CASE("A synced payload does not end an older deferred report's tracking", + "[buffer_ports]") { + Fixture f; + const Nx2TitleScope nx2_title; + const auto id = Access::Add(*f.cache, 0x20000, 0x10000); + ReportOnReadBackPage(f, id); + WriteByDispatch(f, id, 0x20000, 0x100, 3); + // #874's synced payload runs now, ahead of the older report. CounterReport does not + // register or finish tracking for this callback, just as for an immediate payload. + WriteReportResult(f, 0x11111111, false); + REQUIRE_FALSE(ReadWord(f, 0x20040).preemtive); + REQUIRE(f.bytes[0x10040] == 3); + WriteReportResult(f, 0x22222222, true); + REQUIRE_FALSE(ReadWord(f, 0x20040).preemtive); + REQUIRE(f.bytes[0x10040] == 3); +} + TEST_CASE("A query result newer than the GPU writes under it stays in guest memory", "[buffer_ports]") { Fixture f; diff --git a/src/video_core/query_cache/query_cache.h b/src/video_core/query_cache/query_cache.h index d60977970e..8889a80396 100644 --- a/src/video_core/query_cache/query_cache.h +++ b/src/video_core/query_cache/query_cache.h @@ -300,8 +300,10 @@ void QueryCacheBase::CounterReport(GPUVAddr addr, QueryType counter_type const u64 report_size = has_timestamp ? 16 : 4; const bool immediate = !is_fence && !Settings::IsGPULevelHigh() && counter_type == QueryType::Payload; + // Synced payload callbacks can run ahead of older fenced reports at the same address. + // They must not retire one of those reports from the buffer cache's ordered tracking. const bool report_noted = - !immediate && impl->rasterizer.NoteReportPending(cpu_addr, report_size); + !immediate && !is_synced && impl->rasterizer.NoteReportPending(cpu_addr, report_size); std::function operation([this, is_synced, streamer, query_base = query, query_location, pointer, pointer_timestamp, cpu_addr, addr, asid = gpu_memory->GetID(), counter_type, split_entry, mirrored, From 94621e1c52261d127a9bc880f6320eaee14fd831 Mon Sep 17 00:00:00 2001 From: Copilot App <223556219+Copilot@users.noreply.github.com> Date: Mon, 9 Nov 2026 13:28:26 -0300 Subject: [PATCH 105/105] tests: extend the existing synced-writer regression The #1056 test name already existed in the full #805 suite. Fold the new unrelated-work scenario into that test instead of registering it twice. Keep the restart assertion. Correct the preceding merge's validation note: its duplicate-name assertion failed before commit; this commit removes that duplication and the repeated check now passes. git diff --cached --check also passes. BUILD_TESTING remains OFF. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- src/tests/video_core/maxwell_fast_method.cpp | 150 +++++-------------- 1 file changed, 37 insertions(+), 113 deletions(-) diff --git a/src/tests/video_core/maxwell_fast_method.cpp b/src/tests/video_core/maxwell_fast_method.cpp index ec249a6927..5b15094580 100644 --- a/src/tests/video_core/maxwell_fast_method.cpp +++ b/src/tests/video_core/maxwell_fast_method.cpp @@ -950,6 +950,43 @@ TEST_CASE("Split command list fetches wait only for the writers before the last REQUIRE_FALSE(split.WaitForSyncedWriters()); release[2] = true; split.Drain(VideoCommon::DrainSite::Other); + + // Work between the earlier writer and the sync is not a writer, even if a later + // writer was sent after the sync. + std::atomic other{}; + std::atomic later{}; + std::atomic other_done{}; + bool other_waited{true}; + std::thread other_waiter; + SCOPE_EXIT { + other = true; + later = true; + if (other_waiter.joinable()) { + other_waiter.join(); + } + }; + split.Defer([] {}); + split.NoteWriter(); + split.Drain(VideoCommon::DrainSite::Other); + blocking(other); + split.NoteSync(); + blocking(later); + split.NoteWriter(); + split.Publish(); + other_waiter = std::thread([&] { + other_waited = split.WaitForSyncedWriters(); + other_done = true; + }); + const auto other_deadline = std::chrono::steady_clock::now() + std::chrono::seconds{5}; + while (!other_done.load() && std::chrono::steady_clock::now() < other_deadline) { + std::this_thread::sleep_for(std::chrono::milliseconds{1}); + } + REQUIRE(other_done.load()); + other_waiter.join(); + REQUIRE_FALSE(other_waited); + other = true; + later = true; + split.Drain(VideoCommon::DrainSite::Other); } TEST_CASE("Split tells whether compute or copy work it was sent is still queued", @@ -1157,119 +1194,6 @@ TEST_CASE("Split waits for a semaphore write's position, not for the work sent a REQUIRE_FALSE(split.WaitForPosition(release_position, VideoCommon::DrainSite::Acquire)); } -TEST_CASE("Split command list fetches wait only for the writers before the last host sync", - "[video_core][gpu_split_dependencies]") { - using Access = VideoCommon::GpuSplitTestAccess; - Fixture f; - f.system.Initialize(); - VideoCommon::GpuSplit split; - std::array, 3> release{}; - std::atomic done{}; - bool waited{}; - std::thread waiter; - Access::Start(split, f.system); - SCOPE_EXIT { - split.Stop(); - }; - SCOPE_EXIT { - for (auto& flag : release) { - flag = true; - } - if (waiter.joinable()) { - waiter.join(); - } - }; - const auto blocking = [&split](std::atomic& flag) { - split.Defer([&flag] { - while (!flag.load()) { - std::this_thread::yield(); - } - }); - }; - const auto finishes = [&done] { - const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds{5}; - while (!done.load() && std::chrono::steady_clock::now() < deadline) { - std::this_thread::sleep_for(std::chrono::milliseconds{1}); - } - return done.load(); - }; - - split.NoteSync(); - blocking(release[0]); - split.NoteWriter(); - split.Publish(); - REQUIRE_FALSE(split.WaitForSyncedWriters()); - - split.NoteSync(); - blocking(release[1]); - split.NoteWriter(); - split.Publish(); - waiter = std::thread([&] { - waited = split.WaitForSyncedWriters(); - done = true; - }); - const auto wait_deadline = std::chrono::steady_clock::now() + std::chrono::seconds{5}; - while (!Access::FrontWaits(split) && !done.load() && - std::chrono::steady_clock::now() < wait_deadline) { - std::this_thread::yield(); - } - std::this_thread::sleep_for(std::chrono::milliseconds{20}); - REQUIRE_FALSE(done.load()); - release[0] = true; - REQUIRE(finishes()); - waiter.join(); - REQUIRE(waited); - release[1] = true; - split.Drain(VideoCommon::DrainSite::Other); - - split.Defer([] {}); - split.NoteWriter(); - split.Drain(VideoCommon::DrainSite::Other); - blocking(release[2]); - split.NoteSync(); - split.Publish(); - REQUIRE_FALSE(split.WaitForSyncedWriters()); - release[2] = true; - split.Drain(VideoCommon::DrainSite::Other); - - // Work between the earlier writer and the sync is not a writer, even if a later - // writer was sent after the sync. - std::atomic other{}; - std::atomic later{}; - std::atomic other_done{}; - bool other_waited{true}; - std::thread other_waiter; - SCOPE_EXIT { - other = true; - later = true; - if (other_waiter.joinable()) { - other_waiter.join(); - } - }; - split.Defer([] {}); - split.NoteWriter(); - split.Drain(VideoCommon::DrainSite::Other); - blocking(other); - split.NoteSync(); - blocking(later); - split.NoteWriter(); - split.Publish(); - other_waiter = std::thread([&] { - other_waited = split.WaitForSyncedWriters(); - other_done = true; - }); - const auto other_deadline = std::chrono::steady_clock::now() + std::chrono::seconds{5}; - while (!other_done.load() && std::chrono::steady_clock::now() < other_deadline) { - std::this_thread::sleep_for(std::chrono::milliseconds{1}); - } - REQUIRE(other_done.load()); - other_waiter.join(); - REQUIRE_FALSE(other_waited); - other = true; - later = true; - split.Drain(VideoCommon::DrainSite::Other); -} - TEST_CASE("Split restart resets writer positions without replaying old releases", "[video_core][gpu_split_dependencies]") { using Access = VideoCommon::GpuSplitTestAccess;