Build F: newest Lichii (with Ocarina DLSS) + build D (shader cache change + upstream PR 345)

src/core/memory.cpp

@@ -6,6 +6,8 @@
 // SPDX-License-Identifier: GPL-2.0-or-later
 
 #include <atomic>
+#include <chrono>
+#include <unordered_map>
 #include <algorithm>
 #include <cstdint>
 #include <cstring>
@@ -494,10 +496,58 @@ struct Memory::Impl {
         }
     }
 
+    // EDEN_NX2_MME_DMA_STATS: how often pages are protected again after an upload and
+    // unprotected by a CPU write, and how many distinct pages that is.
+    struct ProtectStats {
+        std::mutex mutex;
+        std::array<u64, 2> calls{};
+        std::array<u64, 2> pages{};
+        std::unordered_map<u64, u32> protected_pages;
+        std::chrono::steady_clock::time_point last_log{std::chrono::steady_clock::now()};
+
+        static ProtectStats* Get() {
+            static const bool enabled = std::getenv("EDEN_NX2_MME_DMA_STATS") != nullptr;
+            static ProtectStats stats;
+            return enabled ? &stats : nullptr;
+        }
+
+        void Note(u64 vaddr, u64 size, bool cached) {
+            std::scoped_lock lk{mutex};
+            const u64 first = vaddr >> YUZU_PAGEBITS;
+            const u64 last = (vaddr + size - 1) >> YUZU_PAGEBITS;
+            ++calls[cached];
+            pages[cached] += last - first + 1;
+            if (cached) {
+                for (u64 page = first; page <= last; ++page) {
+                    ++protected_pages[page];
+                }
+            }
+            const auto now = std::chrono::steady_clock::now();
+            if (now - last_log < std::chrono::seconds{5}) {
+                return;
+            }
+            last_log = now;
+            size_t hot = 0;
+            for (const auto& [page, count] : protected_pages) {
+                hot += count >= 100 ? 1 : 0;
+            }
+            LOG_INFO(HW_Memory,
+                     "Page protection (5 s): protect {} calls {} pages ({} distinct, {} protected "
+                     "100+ times), unprotect {} calls {} pages",
+                     calls[1], pages[1], protected_pages.size(), hot, calls[0], pages[0]);
+            calls = {};
+            pages = {};
+            protected_pages.clear();
+        }
+    };
+
     void RasterizerMarkRegionCached(u64 vaddr, u64 size, bool cached) {
         if (vaddr == 0 || !AddressSpaceContains(*current_page_table, vaddr, size)) {
             return;
         }
+        if (auto* stats = ProtectStats::Get()) {
+            stats->Note(vaddr, size, cached);
+        }
 
         if (current_page_table->fastmem_arena) {
             Common::MemoryPermission perm{};

src/tests/video_core/buffer_cache_ports.cpp

@@ -229,9 +229,6 @@ struct BufferCacheTestAccess {
         channel.uniform_buffers[0][0] = {.device_addr = address, .size = size, .buffer_id = id};
         cache.BindHostGraphicsUniformBuffer(0, 0, 0, true);
     }
-    static void NextFrame(Cache& cache) {
-        cache.native_indirect_uniforms_this_frame.clear();
-    }
     static u32 CpuMarks(Cache& cache) {
         return cache.memory_tracker.cpu_marks;
     }
@@ -544,7 +541,7 @@ TEST_CASE("Command page readers remain ordered during batch completion", "[buffe
     REQUIRE(f.runtime.downloads == 1);
 }
 
-TEST_CASE("Native uniform banks upload directly without overwriting GPU-owned bytes",
+TEST_CASE("Native uniform banks mark only changed pages and keep GPU-owned bytes",
           "[buffer_ports]") {
     Fixture f;
     f.memory->EnableNativeTiling();
@@ -559,20 +556,26 @@ TEST_CASE("Native uniform banks upload directly without overwriting GPU-owned by
     std::memset(f.bytes + 0x10000, 0x5a, 0x10000);
     Access::Uniform(*f.cache, channel, id, 0x20000, 0x10000);
     REQUIRE(f.runtime.upload_bytes == 0x10000);
-    REQUIRE(Access::CpuMarks(*f.cache) == 0);
+    REQUIRE(Access::CpuMarks(*f.cache) == 16);
     REQUIRE(Access::Get(*f.cache, id).bytes[0xffff] == 0x5a);
+    const u32 uploads = f.runtime.uploads;
+    Access::Uniform(*f.cache, channel, id, 0x20000, 0x10000);
+    REQUIRE(f.runtime.uploads == uploads);
+    REQUIRE(Access::CpuMarks(*f.cache) == 16);
+    // A write the tracker missed is found on the next bind, and only its page is marked.
+    std::memset(f.bytes + 0x13000, 0x77, 0x1000);
     Access::Uniform(*f.cache, channel, id, 0x20000, 0x10000);
-    REQUIRE(f.runtime.uploads == 1);
-    Access::NextFrame(*f.cache);
+    REQUIRE(Access::CpuMarks(*f.cache) == 17);
+    REQUIRE(f.runtime.upload_bytes == 0x11000);
+    REQUIRE(Access::Get(*f.cache, id).bytes[0x3000] == 0x77);
     Access::Dirty(*f.cache, 0x21000, 0x1000);
     std::memset(f.bytes + 0x10000, 0xa5, 0x10000);
     Access::Uniform(*f.cache, channel, id, 0x20000, 0x10000);
-    REQUIRE(f.runtime.uploads == 1);
+    REQUIRE(f.runtime.upload_bytes == 0x11000);
     REQUIRE(Access::Get(*f.cache, id).bytes[0x1000] == 0x5a);
 }
 
-TEST_CASE("Native uniform uploads stop at holes and keep unaligned bank boundaries",
-          "[buffer_ports]") {
+TEST_CASE("Native uniform bank checks stop at holes", "[buffer_ports]") {
     Fixture f;
     f.memory->EnableNativeTiling();
     Tegra::Control::ChannelState state{0};
@@ -586,20 +589,18 @@ TEST_CASE("Native uniform uploads stop at holes and keep unaligned bank boundari
     std::memset(f.bytes + 0x10000, 0x6b, 0x20000);
     f.device.Unmap(0x22000, 0x1000);
     Access::Uniform(*f.cache, channel, id, 0x20123, 0x10000);
-    REQUIRE(f.runtime.upload_bytes == 0x1edd);
+    REQUIRE(Access::CpuMarks(*f.cache) == 2);
+    REQUIRE(f.runtime.upload_bytes == 0x2000);
     const auto& data = Access::Get(*f.cache, id).bytes;
-    REQUIRE(data[0x122] == 0);
     REQUIRE(data[0x123] == 0x6b);
     REQUIRE(data[0x1fff] == 0x6b);
     REQUIRE(data[0x2000] == 0);
     REQUIRE(data[0x3000] == 0);
-    REQUIRE(Access::CpuMarks(*f.cache) == 0);
     f.device.Map(0x22000, 0x22000, 0x1000, f.asid);
-    Access::NextFrame(*f.cache);
     Access::Uniform(*f.cache, channel, id, 0x20123, 0x10000);
-    REQUIRE(f.runtime.upload_bytes == 0x11edd);
+    REQUIRE(Access::CpuMarks(*f.cache) == 17);
+    REQUIRE(data[0x2000] == 0x6b);
     REQUIRE(data[0x10122] == 0x6b);
-    REQUIRE(data[0x10123] == 0);
 }
 
 TEST_CASE("Downloads keep CPU writes collected and not yet applied", "[buffer_ports]") {

src/tests/video_core/maxwell_fast_method.cpp

@@ -3,6 +3,7 @@
 
 #include <algorithm>
 #include <array>
+#include <atomic>
 #include <chrono>
 #include <cstring>
 #include <functional>
@@ -12,6 +13,7 @@
 #include <vector>
 
 #include <catch2/catch_test_macros.hpp>
+#include <catch2/generators/catch_generators.hpp>
 
 #include "common/page_table.h"
 #include "common/scope_exit.h"
@@ -837,3 +839,85 @@ TEST_CASE("Maxwell3D's split front uploads macro code sent as one multi-method r
     REQUIRE(macros.native_write_offset == pointer * 3 + code.size());
     CHECK(std::equal(code.begin(), code.end(), macros.native_code.begin() + pointer * 3));
 }
+
+TEST_CASE("Split front orders queued draws before reads at the next wait for idle",
+          "[video_core][gpu_split_dependencies]") {
+    using Access = VideoCommon::GpuSplitTestAccess;
+    constexpr u32 WaitForIdle = MAXWELL3D_REG_INDEX(wait_for_idle);
+    constexpr GPUVAddr address = 0x100000;
+    // Instance id in draw.begin bits 26-27: a plain draw, or the first draw of an instanced
+    // loop that the back defers until the wait for idle flushes it.
+    const u32 begin = GENERATE(0u, 1u << 26);
+    CAPTURE(begin);
+    Fixture f;
+    VideoCommon::GpuSplit split;
+    Access::Initialize(split, f.system);
+    auto& engine = f.fast;
+    const auto ready = [&] { return split.ReadReadyPosition(&f.memory, address, address + 4); };
+
+    // A wait for idle without draws orders nothing.
+    engine.FrontCallMethod(f.system, split, WaitForIdle, 0, true);
+    REQUIRE(ready() == 0);
+    // The back has executed nothing, so nothing has told the front this draw writes memory.
+    engine.FrontCallMethod(f.system, split, DrawBegin, begin, true);
+    engine.FrontCallMethod(f.system, split, DrawEnd, 0, true);
+    // Hardware orders macro reads after draws only across a wait for idle.
+    REQUIRE(ready() == 0);
+    engine.FrontCallMethod(f.system, split, WaitForIdle, 0, true);
+    REQUIRE(ready() == Access::End(split));
+    REQUIRE(split.WriterPosition() == Access::End(split));
+    // Recorded once: a second wait for idle does not move it.
+    const u64 after_draw = Access::End(split);
+    engine.FrontCallMethod(f.system, split, WaitForIdle, 0, true);
+    REQUIRE(ready() == after_draw);
+}
+
+TEST_CASE("Split host syncs order queued draws as waits for idle do",
+          "[video_core][gpu_split_dependencies]") {
+    using Access = VideoCommon::GpuSplitTestAccess;
+    Fixture f;
+    VideoCommon::GpuSplit split;
+    Access::Initialize(split, f.system);
+    split.NoteSync();
+    REQUIRE(split.WriterPosition() == 0);
+    f.fast.FrontCallMethod(f.system, split, DrawBegin, 0, true);
+    f.fast.FrontCallMethod(f.system, split, DrawEnd, 0, true);
+    REQUIRE(split.WriterPosition() == 0);
+    split.NoteSync();
+    REQUIRE(split.WriterPosition() == Access::End(split));
+}
+
+TEST_CASE("Split front read after a draw and a wait for idle waits for a delayed back",
+          "[video_core][gpu_split_dependencies]") {
+    using Access = VideoCommon::GpuSplitTestAccess;
+    constexpr GPUVAddr address = 0x100000;
+    Fixture f;
+    f.system.Initialize();
+    VideoCommon::GpuSplit split;
+    Access::Start(split, f.system);
+    SCOPE_EXIT { split.Stop(); };
+    auto& engine = f.fast;
+    // The front drains once as it starts: start it before the back stalls.
+    engine.FrontCallMethod(f.system, split, f.state_methods.front(), 0x1234, true);
+    std::atomic<bool> release{false};
+    std::atomic<bool> drawn{false};
+    // The back stalls before the first draw it executes, as it can behind real work.
+    split.Defer([&release] {
+        while (!release.load(std::memory_order_acquire)) {
+            std::this_thread::yield();
+        }
+    });
+    engine.FrontCallMethod(f.system, split, DrawBegin, 0, true);
+    engine.FrontCallMethod(f.system, split, DrawEnd, 0, true);
+    // Stands for the draw's shader stores.
+    split.Defer([&drawn] { drawn.store(true, std::memory_order_release); });
+    engine.FrontCallMethod(f.system, split, MAXWELL3D_REG_INDEX(wait_for_idle), 0, true);
+    const u64 target = split.ReadReadyPosition(&f.memory, address, address + 4);
+    REQUIRE(target > split.Consumed());
+    std::jthread releaser([&release] {
+        std::this_thread::sleep_for(std::chrono::milliseconds{20});
+        release.store(true, std::memory_order_release);
+    });
+    split.WaitForPosition(target, VideoCommon::DrainSite::MacroDmaRead);
+    REQUIRE(drawn.load(std::memory_order_acquire));
+}

src/tests/video_core/uniform_stream_reuse.cpp

@@ -23,6 +23,10 @@ struct Memory {
         Check(address + size <= bytes.size(), "read outside guest memory");
         std::memcpy(destination, bytes.data() + address, size);
     }
+    template <typename T>
+    T* GetPointer(u64 address) {
+        return address < bytes.size() ? reinterpret_cast<T*>(bytes.data() + address) : nullptr;
+    }
 };
 
 struct Runtime {

src/video_core/buffer_cache/buffer_cache.h

@@ -112,7 +112,6 @@ void BufferCache<P>::TickFrame() {
         return;
     }
     runtime.TickFrame(slot_buffers);
-    native_indirect_uniforms_this_frame.clear();
     if (frame_tick % 600 == 0 && !uploaded_buffer_ranges.empty()) {
         for (const auto& [address, size] : uploaded_buffer_ranges) {
             device_memory.HintCold(address, size);
@@ -1397,26 +1396,62 @@ void BufferCache<P>::BindHostGraphicsUniformBuffer(size_t stage, u32 index, u32
     }();
     // Native programs keep tables next to their code in a 64 KiB bank read with indirect offsets
     // (Splatoon Raiders: the dialogue box corners in c1). Writes to that memory can go unnoticed by
-    // the memory tracker, leaving stale host contents, so upload such a bank again once per frame.
+    // the memory tracker, leaving stale host contents, so check such a bank on every bind: games
+    // rewrite per-draw data between draws, and checking once per frame left it stale for a frame
+    // now and then (white flashes in Ocarina of Time, Eglantine 031e07436c).
     if (has_host_buffer && (*channel_state->uniform_buffer_sizes)[stage][index] == 0x10000 &&
         size != 0 && gpu_memory != nullptr && gpu_memory->UsesNativeTiling() &&
-        (Nx2Perf::Verify() || !native_indirect_uniforms_this_frame.contains(device_addr)) &&
-        !memory_tracker.IsRegionGpuModified(device_addr, size) &&
-        native_indirect_uniforms_this_frame.insert(device_addr).second) {
+        !memory_tracker.IsRegionGpuModified(device_addr, size)) {
         // The bank size is only an upper limit: stop at the end of the mapped memory.
         u64 mapped_size = 0;
         while (mapped_size < size &&
                device_memory.template GetPointer<u8>(device_addr + mapped_size) != nullptr) {
             mapped_size += Core::DEVICE_PAGESIZE - ((device_addr + mapped_size) & Core::DEVICE_PAGEMASK);
         }
-        const u64 upload_size = (std::min<u64>)(mapped_size, size);
-        if (upload_size != 0) {
-            std::array<BufferCopy, 1> copies{BufferCopy{
-                .src_offset = 0,
-                .dst_offset = buffer.Offset(device_addr),
-                .size = upload_size,
-            }};
-            UploadMemory(buffer, upload_size, upload_size, copies);
+        const u64 check_size = (std::min<u64>)(mapped_size, size);
+        // Banks slide through larger buffers, a new address per draw: compare each page with
+        // its last contents and mark only the pages that changed.
+        if (native_indirect_page_snapshots.size() >= 16384) {
+            native_indirect_page_snapshots.clear();
+        }
+        const DAddr first_page = Common::AlignDown(device_addr, Core::DEVICE_PAGESIZE);
+        const DAddr end = device_addr + check_size;
+        for (DAddr page = first_page; page < end; page += Core::DEVICE_PAGESIZE) {
+            const u8* const guest = device_memory.template GetPointer<u8>(page);
+            if (guest == nullptr) {
+                continue;
+            }
+            auto [it, inserted] = native_indirect_page_snapshots.try_emplace(page);
+            const bool same =
+                !inserted && std::memcmp(it->second.data(), guest, Core::DEVICE_PAGESIZE) == 0;
+            // EDEN_NX2_MME_DMA_STATS: how many checked pages had changed.
+            if (static const bool stats = std::getenv("EDEN_NX2_MME_DMA_STATS") != nullptr;
+                stats) {
+                static std::array<u64, 3> counts{};
+                static std::array<u64, 4> changed_state{};
+                ++counts[same ? 0 : (inserted ? 1 : 2)];
+                if (!same && !inserted) {
+                    // Did the tracker already know (CPU-modified), or was the page clean, and
+                    // was it GPU-modified?
+                    const bool known =
+                        memory_tracker.IsRegionCpuModified(page, Core::DEVICE_PAGESIZE);
+                    const bool gpu =
+                        memory_tracker.IsRegionGpuModified(page, Core::DEVICE_PAGESIZE);
+                    ++changed_state[(known ? 1 : 0) | (gpu ? 2 : 0)];
+                }
+                if (((counts[0] + counts[1] + counts[2]) & 0x3ffff) == 0) {
+                    LOG_INFO(HW_GPU,
+                             "Native bank pages checked: same {}, new {}, changed {} (tracker "
+                             "clean {}, CPU-modified {}, GPU {}, both {})",
+                             counts[0], counts[1], counts[2], changed_state[0],
+                             changed_state[1], changed_state[2], changed_state[3]);
+                }
+            }
+            if (same) {
+                continue;
+            }
+            std::memcpy(it->second.data(), guest, Core::DEVICE_PAGESIZE);
+            memory_tracker.MarkRegionAsCpuModified(page, Core::DEVICE_PAGESIZE);
         }
     }
     const bool use_fast_buffer = needs_alignment_stream

src/video_core/buffer_cache/buffer_cache_base.h

@@ -463,8 +463,10 @@ class BufferCache : public VideoCommon::ChannelSetupCaches<BufferCacheChannelInf
     Common::SpinRecursiveMutex mutex;
     Runtime& runtime;
     bool any_buffer_uploaded = false;
-    /// Indirectly read banks of native programs already uploaded this frame.
-    std::unordered_set<DAddr> native_indirect_uniforms_this_frame;
+    /// The guest words of each page of native programs' indirectly read banks when last checked:
+    /// unchanged pages keep their tracking (marking them CPU-modified costs two mprotect calls
+    /// per page).
+    std::unordered_map<DAddr, std::array<u8, Core::DEVICE_PAGESIZE>> native_indirect_page_snapshots;
 
 private:
     template <typename Mask, typename Func>
@@ -513,6 +515,7 @@ class BufferCache : public VideoCommon::ChannelSetupCaches<BufferCacheChannelInf
 
     void BindHostGraphicsStorageBuffers(size_t stage);
 
+
     void BindHostGraphicsTextureBuffers(size_t stage);
 
     void BindHostTransformFeedbackBuffers();