feat(vk): frame generation Auto: a frame for each of the display's refreshes

gpu/vk/vk_api.h

@@ -177,12 +177,14 @@ __attribute__((visibility("default"))) const MkwVkRenderer *mkwvk_init(const Mkw
 #ifndef _WIN32
 /* (linux) what start() is given in place of a window handle (host/posix/display_sdl.c): the window makes the
    surface (SDL_Vulkan_CreateSurface: X11 or Wayland) and says its size in pixels */
-#define MKWVK_WINDOW_VERSION 1
+#define MKWVK_WINDOW_VERSION 2
 typedef struct MkwVkWindow {
     uint32_t version;   /* MKWVK_WINDOW_VERSION */
     int (*create_surface)(void *window, void *instance, void *surface_out);   /* (VkInstance, VkSurfaceKHR *): 1 if made */
     void (*drawable_size)(void *window, int *width, int *height);
     void *window;
+    /* (version 2) the refresh rate of the display the window is on, in mHz; 0 when not known (frame generation's Auto) */
+    int (*refresh_mhz)(void *window);
 } MkwVkWindow;
 #endif
 

gpu/vk/vk_framegen.cpp

@@ -16,6 +16,8 @@
 #include <mutex>
 #include <thread>
 
+#include <cmath>
+
 #include "vk_framegen.h"
 #include "vk_menu.h"
 #include "vk_texture.h"
@@ -33,6 +35,7 @@
 #endif
 
 namespace mkwvk {
+int display_refresh_mhz();   // (vk_main.cpp: the window's display, mHz; outside the unnamed namespace, where it is defined)
 extern std::atomic<uint32_t> present_kind;   // (vk_replay.cpp: the frame log's)
 namespace {
 
@@ -47,6 +50,11 @@ struct OutSet {
     Img frame[kMaxMult];
     int made = 0;   // (frame[0..made) are made frames; frame[made] is the game's)
     bool busy = false;
+    // (Auto: frame[0..shown) each at its own time, due[k] (steady clock), game[k] the game's own frame; shown < 0: the
+    // fixed multipliers' frames, made + 1 of them, shown evenly)
+    int shown = -1;
+    int64_t due[kMaxMult] = {};
+    bool game[kMaxMult] = {};
 };
 struct Job {
     OutSet* set;
@@ -76,6 +84,8 @@ struct State {
     OutSet* recorded = nullptr;   // (the set the last framegen_record filled, for framegen_queue)
     int64_t recorded_at = 0;
     int mult = 0;   // (the multiplier the sets' frames were made for)
+    int64_t game_ns = 0;     // (Auto: the game's frame time, smoothed: the delay its frames are shown at)
+    int64_t slot_next = 0;   // (Auto: the display's next refresh to be given a frame, on the game's timeline)
 } fg;
 
 // Used only by present_image, whose lock serializes direct and interpolated presents.
@@ -91,7 +101,43 @@ int64_t now_ns() {
     return std::chrono::duration_cast<std::chrono::nanoseconds>(std::chrono::steady_clock::now().time_since_epoch()).count();
 }
 
-int multiplier() { return std::clamp(opts.framegen, 1, kMaxMult); }   // (the Insert menu; 1: the game's frames alone)
+// (hookshot) Frame generation's Auto (the menu's 1): a frame for each refresh of the display, no more. Each game frame
+// is given the display's refreshes that fall between it and the game's frame before (on the game's own timeline, a
+// refresh period apart), and each of them a frame made at its place between the two: 45 fps on a 60 Hz display makes one
+// frame for every three of the game's (3 of 4 shown are made, at 3/4, 1/2, 1/4), on 120 Hz two or three a frame; at
+// 60 fps on 60 Hz none (a refresh near a game frame shows it), and faster than the display a game frame may be given no refresh and is not shown. Shown one
+// game frame time later, each at its refresh's time. The sets' frames are made once for the most a game frame may be
+// given (the refresh over 30 fps, 2 to 4). An unknown refresh counts as 60 Hz.
+int auto_cap() {
+    const int hz = (display_refresh_mhz() + 500) / 1000;
+    return std::clamp(hz > 0 ? (hz + 29) / 30 : 2, 2, 4);
+}
+int multiplier() {   // (the Insert menu; 1 Auto (its most); the game's frames alone at 0)
+    return opts.framegen == 1 ? auto_cap() : std::clamp(opts.framegen, 1, kMaxMult);
+}
+/// Auto: the display's refreshes in the game frame's interval (last, now], next the first not given a frame yet (moved
+/// past them). Each one's place between the two game frames in t (1: this one) and its time in at; most of them at the
+/// most, those past it skipped. A timeline outside the interval (the first frame, a hitch) starts again at now.
+int auto_slots(int64_t last, int64_t now, int64_t period, int64_t& next, float* t, int64_t* at, int most) {
+    if (now <= last || period <= 0) return 0;
+    if (next <= last || next > now + period) next = now;
+    int n = 0;
+    for (; next <= now; next += period)
+        if (n < most) t[n] = (float)(next - last) / (float)(now - last), at[n++] = next;
+    // A refresh within a quarter period of the game's frame shows that frame itself: at the game's rate the frames come
+    // a little early or late (jitter), and without this nearly every refresh got a made frame at t near 1 and some game
+    // frames none (their neighbour two). The timeline is pulled an eighth of the way to the frame, so that a game at the
+    // display's rate (or a whole fraction of it) stays on it while the two clocks drift apart, and other rates hardly
+    // change how many frames the display is given
+    const int64_t near = period / 4;
+    if (next - now < near) {
+        n = std::min(n + 1, most);
+        t[n - 1] = 1.0f, at[n - 1] = now, next += period + (now - next) / 8;
+    } else if (n > 0 && now - (next - period) < near) {
+        t[n - 1] = 1.0f, at[n - 1] = now, next += (now - (next - period)) / 8;
+    }
+    return n;
+}
 
 /// A compute pipeline over bindings of these types (push descriptors), push constants of push bytes
 bool make_pipe(Pipe& p, const uint32_t* code, size_t bytes, std::initializer_list<VkDescriptorType> types, uint32_t push,
@@ -149,7 +195,11 @@ bool init() {
     si.magFilter = si.minFilter = VK_FILTER_LINEAR;
     vkCreateSampler(dev.device, &si, nullptr, &fg.linear);
     fg.failed = false;
-    logf("vk: frame interpolation ready: %dx the game's frames (made from its last two, shown evenly)", multiplier());
+    if (opts.framegen == 1)
+        logf("vk: frame interpolation ready: Auto, a frame for each of the display's refreshes (%d Hz; up to %d a game frame)",
+             (display_refresh_mhz() + 500) / 1000, multiplier());
+    else
+        logf("vk: frame interpolation ready: %dx the game's frames (made from its last two, shown evenly)", multiplier());
     return true;
 }
 
@@ -324,19 +374,22 @@ void display_main() {
         // apart, a frame early waiting for it: never more than one game frame behind what the game handed over, and a line
         // that has fallen behind what came starts again from now. RX_VK_PACE=0: as they come.
         const bool pace = opts.pace != 0;
-        const int shown = j.set->made + 1;
-        const int64_t step = interval / shown;
-        for (int k = 0; k <= j.set->made; k++) {
-            int64_t due = pace ? next_show : 0;
+        const bool timed = j.set->shown >= 0;   // (Auto: each frame at its own refresh's time)
+        const int shown = timed ? j.set->shown : j.set->made + 1;
+        const int64_t step = interval / std::max(shown, 1);
+        if (timed) menu_note_frame(true, 0);   // (the overlay's game rate: Auto counts each game frame, given a refresh or not)
+        for (int k = 0; k < shown; k++) {
+            int64_t due = timed ? j.set->due[k] : pace ? next_show : 0;
             const int64_t now = now_ns();
             if (due < now) due = now;
-            if (due > j.at + interval + step) due = j.at + interval + step;
+            if (!timed && due > j.at + interval + step) due = j.at + interval + step;
+            if (timed && due > now + 100000000) due = now;   // (a game frame time gone wrong: not held back)
             if (due - now > 1000000) wait_until_near(due - now - 1000000);
             while (now_ns() < due) SwitchToThread();
             const int64_t t = now_ns();
-            present_kind.store(k < j.set->made ? 1 : 0);
+            present_kind.store(timed ? (j.set->game[k] ? 0 : 1) : k < j.set->made ? 1 : 0);
             present_image(j.set->frame[k].image, fg.w, fg.h, nullptr, j.set->frame[k].view);
-            menu_note_frame(k == j.set->made);
+            menu_note_frame(!timed && k == j.set->made);
             if (shown_at) {
                 const int64_t d = t - shown_at;
                 hist[d < 7000000 ? 0 : d < 9000000 ? 1 : d < 12000000 ? 2 : d < 17000000 ? 3 : d < 25000000 ? 4 : 5]++;
@@ -355,9 +408,9 @@ void display_main() {
 
 // The Insert menu's Frame generation is off at 0. While it was ignored, every present still copied the frame and
 // reduced it to luma (framegen_record), then a second thread showed it. Open areas paid for that on a full GPU and
-// gained no made frames. 2x and up is the interpolation; RX_VK_FRAMEGEN=0 keeps it off entirely.
+// gained no made frames. Auto (1) and 2x and up are the interpolation; RX_VK_FRAMEGEN=0 keeps it off entirely.
 bool framegen_wanted(int menu_value, int allow) {
-    return allow && menu_value >= 2;
+    return allow && menu_value >= 1;   // (1: Auto)
 }
 
 bool framegen_on() {
@@ -440,8 +493,10 @@ bool framegen_record(VkCommandBuffer cmd, VkImageView src, uint32_t sw, uint32_t
         fg.thread_started = true;
         std::thread(display_main).detach();
     }
-    const int64_t now = now_ns();
+    const int64_t now = now_ns(), before = fg.last_at;
     const bool gap = fg.last_at && now - fg.last_at > 100000000;   // (a loading hitch: nothing to blend)
+    if (fg.last_at && !gap)
+        fg.game_ns = fg.game_ns ? (fg.game_ns * 7 + (now - fg.last_at)) / 8 : now - fg.last_at;
     fg.last_at = now;
     OutSet* set;
     {
@@ -460,6 +515,48 @@ bool framegen_record(VkCommandBuffer cmd, VkImageView src, uint32_t sw, uint32_t
     pass(cmd, fg.copy, {{src, fg.nearest}, {fg.frames[cur].view, VK_NULL_HANDLE}}, uw, uh, pc, sizeof pc);
     pass(cmd, fg.luma, {{fg.frames[cur].view, fg.nearest}, {fg.lumas[cur].view, VK_NULL_HANDLE}}, qw, qh);
     set->made = 0;
+    set->shown = -1;
+    if (opts.framegen == 1) {   // (Auto: a frame for each of the display's refreshes, at its own time)
+        const int mhz = display_refresh_mhz();
+        const int64_t period = (int64_t)(1e12 / (mhz > 0 ? mhz : 60000));
+        float t[kMaxMult];
+        int64_t at[kMaxMult];
+        int n;
+        if (blend) {
+            n = auto_slots(before, now, period, fg.slot_next, t, at, std::min(fg.mult, kMaxMult));
+        } else {   // (nothing to blend from: the game's frame, and the timeline starts at it)
+            n = 1, t[0] = 1.0f, at[0] = now, fg.slot_next = now + period;
+        }
+        bool flow = false;
+        for (int k = 0; k < n; k++) {
+            set->game[k] = t[k] > 0.999f;
+            set->due[k] = at[k] + fg.game_ns;
+            if (set->game[k]) {
+                const VkImageCopy c{{VK_IMAGE_ASPECT_COLOR_BIT, 0, 0, 1}, {0, 0, 0}, {VK_IMAGE_ASPECT_COLOR_BIT, 0, 0, 1},
+                                    {0, 0, 0}, {uw, uh, 1}};
+                vkCmdCopyImage(cmd, fg.frames[cur].image, VK_IMAGE_LAYOUT_GENERAL, set->frame[k].image,
+                               VK_IMAGE_LAYOUT_GENERAL, 1, &c);
+                continue;
+            }
+            if (!flow) {
+                flow = true;
+                pass(cmd, fg.flow, {{fg.lumas[prev].view, fg.nearest}, {fg.lumas[cur].view, fg.nearest},
+                                    {fg.flows[0].view, VK_NULL_HANDLE}}, cw, ch);
+                pass(cmd, fg.median, {{fg.flows[0].view, fg.nearest}, {fg.flows[1].view, VK_NULL_HANDLE}}, cw, ch);
+            }
+            pass(cmd, fg.synth, {{fg.frames[prev].view, fg.linear}, {fg.frames[cur].view, fg.linear},
+                                 {fg.flows[1].view, fg.nearest}, {set->frame[k].view, VK_NULL_HANDLE}},
+                 uw, uh, &t[k], 4);
+        }
+        set->shown = n;
+        set->made = n > 0 && !set->game[n - 1] ? n : std::max(0, n - 1);   // (for framegen_made_image: any made)
+        full_barrier(cmd);
+        fg.have_prev = true;
+        fg.cur = prev;
+        fg.recorded = set;
+        fg.recorded_at = now;
+        return true;
+    }
     if (blend && m > 1) {
         pass(cmd, fg.flow, {{fg.lumas[prev].view, fg.nearest}, {fg.lumas[cur].view, fg.nearest},
                             {fg.flows[0].view, VK_NULL_HANDLE}}, cw, ch);

gpu/vk/vk_main.cpp

@@ -824,7 +824,7 @@ int start(void* hwnd) {
 #else
     const MkwVkWindow* w = (const MkwVkWindow*)hwnd;
     prep_window = w;
-    if (w->version != MKWVK_WINDOW_VERSION) {
+    if (w->version < 1 || w->version > MKWVK_WINDOW_VERSION) {   // (1: no refresh_mhz)
         logf("vk: unsupported SDL window bridge version %u (expected %u)", w->version, MKWVK_WINDOW_VERSION);
         return 0;
     }
@@ -936,4 +936,34 @@ const MkwVkRenderer* mkwvk_init(const MkwVkHost* h) {
 
 namespace mkwvk {
 HWND app_window() { return main_window; }   // (hookshot: the Insert menu's Fullscreen row)
+
+/// (hookshot) The refresh rate of the display the window is on, in mHz; 0 when not known (frame generation's Auto). Asked
+/// again once a second at most: the window may move to another display, and Windows' answer costs a few system calls
+int display_refresh_mhz() {
+    static std::atomic<int> cached{0};
+    static std::atomic<int64_t> asked_at{0};
+    const int64_t now = std::chrono::duration_cast<std::chrono::milliseconds>(
+        std::chrono::steady_clock::now().time_since_epoch()).count();
+    const int64_t last = asked_at.load(std::memory_order_relaxed);
+    if (last && now - last < 1000) return cached.load(std::memory_order_relaxed);
+    asked_at.store(now, std::memory_order_relaxed);
+    int mhz = 0;
+#ifdef _WIN32
+    MONITORINFOEXW mi{};
+    mi.cbSize = sizeof mi;
+    DEVMODEW dm{};
+    dm.dmSize = sizeof dm;
+    if (main_window && GetMonitorInfoW(MonitorFromWindow(main_window, MONITOR_DEFAULTTONEAREST), &mi) &&
+        EnumDisplaySettingsW(mi.szDevice, ENUM_CURRENT_SETTINGS, &dm) && dm.dmDisplayFrequency > 1)   // (0 and 1: the default)
+        // (whole Hz: a 59.94 Hz mode often says 59, and Auto would then make a frame too few a second. One under a
+        // multiple of 6 (24, 30, 60, 120, 144, 240 Hz) is taken as that rate's /1.001)
+        mhz = (dm.dmDisplayFrequency + 1) % 6 == 0 ? (int)((dm.dmDisplayFrequency + 1) * 1000000ull / 1001)
+                                                    : (int)dm.dmDisplayFrequency * 1000;
+#else
+    const MkwVkWindow* w = (const MkwVkWindow*)main_window;
+    if (w && w->version >= 2 && w->refresh_mhz) mhz = w->refresh_mhz(w->window);
+#endif
+    cached.store(mhz, std::memory_order_relaxed);
+    return mhz;
+}
 } // namespace mkwvk

gpu/vk/vk_menu.cpp

@@ -76,7 +76,7 @@ const char* hint_in(const char* const* t, int n, int lo, int v, int variant) {
         return hint_in(t, (int)(sizeof t / sizeof t[0]), lo, v, variant);                                                    \
     }
 const char* tx_framegen(int v) {
-    static const char* t[] = {"Off", "", "2x", "3x", "4x", "5x", "6x", "7x", "8x"};
+    static const char* t[] = {"Off", "Auto", "2x", "3x", "4x", "5x", "6x", "7x", "8x"};
     return t[v];
 }
 const char* tx_onoff(int v) { return v ? "On" : "Off"; }
@@ -389,8 +389,9 @@ MENU_HINTS(hn_preset, 0, "Each row as you set it. Changing most rows yourself co
 MENU_HINTS(hn_gpu, 0, "The best dedicated GPU.")
 /// Frame generation: how many frames are made for each of the game's
 const char* hn_framegen(int v, int variant) {
-    if (variant > 0 || v == 1) return nullptr;
+    if (variant > 0) return nullptr;
     if (v <= 0) return "Only the game's own frames: no added delay.";
+    if (v == 1) return "A frame for each of the display's refreshes: none made when the game keeps up. Frame pacing does not apply.";
     static char b[9][96];
     snprintf(b[v], sizeof b[v], "%s between each two of the game's.", v == 2 ? "One made frame" : v == 3 ? "Two made frames" : v == 4 ? "Three made frames"
              : v == 5 ? "Four made frames" : v == 6 ? "Five made frames" : v == 7 ? "Six made frames" : "Seven made frames");
@@ -855,7 +856,6 @@ void set_row(Row& r, int v, int dir, bool keep = true) {
         v = step_output_choice(*r.value, dir);
         if (v == *r.value) return;
     }
-    if (r.value == &opts.framegen && v == 1) v = dir > 0 ? 2 : 0;   // (off, 2x ... 8x)
     if (stops_at_ends(r)) v = std::min(std::max(v, r.lo), r.hi);   // (a scale: it stops at its ends)
     if (r.value == &opts.pad_layout) {
         if (v > 2) v = dir > 0 ? 0 : 2;
@@ -1642,7 +1642,6 @@ void menu_init() {
         opts.sharpness = 0;
     }
     if (opts.preset > 0) sync_preset_budgets(opts.preset);
-    if (opts.framegen == 1) opts.framegen = 2;
     if (const char* e = getenv("RX_VK_FRAMEGEN_MULT")) opts.framegen = std::min(std::max(atoi(e), 2), 8);
     if (const char* e = getenv("RX_VK_PACE")) opts.pace = atoi(e) != 0;
     if (const char* e = getenv("RX_VK_UPSCALE"))
@@ -1682,8 +1681,8 @@ void menu_note_present_sizes(uint32_t picture_w, uint32_t picture_h, uint32_t wi
     const auto c = [](uint32_t v) { return (uint64_t)std::min(v, 0xffffu); };
     present_sizes.store(c(picture_w) << 48 | c(picture_h) << 32 | c(window_w) << 16 | c(window_h), std::memory_order_relaxed);
 }
-void menu_note_frame(bool game_frame) {
-    shown_frames.fetch_add(1, std::memory_order_relaxed);
+void menu_note_frame(bool game_frame, int shown) {
+    if (shown) shown_frames.fetch_add(shown, std::memory_order_relaxed);
     if (game_frame) game_frames.fetch_add(1, std::memory_order_relaxed);
 }
 
@@ -2210,7 +2209,7 @@ void menu_tick() {
         double shown = 0, game = 0;
         if (fps_take(now, shown_frames.load(), game_frames.load(), &next, &last_shown, &last_game, &shown, &game)) {
             snprintf(fps_text, sizeof fps_text, "%.0f", shown);
-            if (opts.framegen >= 2) snprintf(fps_sub, sizeof fps_sub, "game %.0f", game);
+            if (opts.framegen >= 1) snprintf(fps_sub, sizeof fps_sub, "game %.0f", game);
             else fps_sub[0] = 0;
             fps_bmp.keyed = true;
             gdi_draw(fps_bmp, 120u * scale, 54u * scale, paint_fps, scale);

gpu/vk/vk_menu.h

@@ -7,7 +7,7 @@ namespace mkwvk {
 
 // (out of the box every feature is off: what the game does on its own. The ini beside the executable keeps a player's choices)
 struct Options {
-    int framegen = 0;     // made frames: 0 off, or 2, 3, 4 times the game's frames
+    int framegen = 0;     // made frames: 0 off, 1 Auto (to the display's refresh), or 2 to 8 times the game's frames
     int pace = 0;         // frames shown on an even line (vk_framegen.cpp)
     int resolution = 0;   // the 3D's (Render resolution): 0 Auto (the game's dynamic resolution), else a share of the output,
                           // 1..6 = 100%, 75%, 67%, 50%, 58%, 33% (upscaler_rows.h; nx2/oot_patches.py)
@@ -62,7 +62,7 @@ int menu_heart_protection();
 int menu_player_flight();
 int menu_free_camera();
 int menu_take_cheat_action();
-void menu_note_frame(bool game_frame);   // (the frame rates the FPS counter shows)
+void menu_note_frame(bool game_frame, int shown = 1);   // (the frame rates the FPS counter shows)
 /// The game's picture's size (the Output resolution, its crop) and the window's, at each present: the Window scaling filter
 /// row says when it is not needed
 void menu_note_present_sizes(uint32_t picture_w, uint32_t picture_h, uint32_t window_w, uint32_t window_h);