Recording/streaming: no more game slowdown, correct A/V timing, Twitch-ready presets

audio/audio_driver.c

@@ -9092,9 +9092,15 @@ void audio_driver_menu_sample(bool by_clock)
    if ((AUDIO_FLAGS_GET(audio_st) & AUDIO_FLAG_SUSPENDED))
       check_flush                         = false;
 
+   /* Record Game Only: the menu's filler silence (and so the time spent
+    * in the menu) stays out of the recording, like its frames do. */
+   if (settings->bools.video_record_game_only)
+      recording_st                        = NULL;
+
    while (sample_count > 1024)
    {
-      if (  recording_st->data   &&
+      if (  recording_st         &&
+            recording_st->data   &&
             recording_st->driver &&
             recording_st->driver->push_audio)
       {
@@ -9117,7 +9123,8 @@ void audio_driver_menu_sample(bool by_clock)
       sample_count -= 1024;
    }
 
-   if (     recording_st->data
+   if (     recording_st
+         && recording_st->data
          && recording_st->driver
          && recording_st->driver->push_audio)
    {

config.def.h

@@ -1196,6 +1196,27 @@
 /* Record post-shaded GPU output instead of raw game footage if available. */
 #define DEFAULT_GPU_RECORD false
 
+/* Drop recorded video frames when the encoder queue is full, instead
+ * of stalling the frontend until it drains. */
+#define DEFAULT_VIDEO_RECORD_ALLOW_FRAME_DROP true
+
+/* Record only the game: GPU recordings leave out the menu, OSD,
+ * widgets and overlays, and menu time is left out of recordings. */
+#define DEFAULT_VIDEO_RECORD_GAME_ONLY false
+
+/* Use a hardware H.264 encoder for recording/streaming when the FFmpeg
+ * build has one that works, falling back to libx264. */
+#ifdef HAVE_LAKKA_SWITCH
+#define DEFAULT_VIDEO_RECORD_HW_ENCODER true
+#else
+#define DEFAULT_VIDEO_RECORD_HW_ENCODER false
+#endif
+
+/* Depth of the recording video queue, in frames. */
+#define DEFAULT_VIDEO_RECORD_FIFO_FRAMES 32
+#define MIN_VIDEO_RECORD_FIFO_FRAMES 8
+#define MAX_VIDEO_RECORD_FIFO_FRAMES 128
+
 /* Watch shader files for changes and auto-apply as necessary. */
 #define DEFAULT_VIDEO_SHADER_WATCH_FILES false
 

configuration.h

@@ -426,6 +426,7 @@ typedef struct settings
       unsigned window_auto_dims_max;
 
       unsigned video_record_threads;
+      unsigned video_record_fifo_frames;
 
       unsigned libnx_overclock;
       unsigned ai_service_mode;
@@ -669,6 +670,9 @@ typedef struct settings
       bool video_disable_composition;
       bool video_post_filter_record;
       bool video_gpu_record;
+      bool video_record_allow_frame_drop;
+      bool video_record_game_only;
+      bool video_record_hw_encoder;
       bool video_gpu_screenshot;
       bool video_allow_rotate;
       bool video_shared_context;

gfx/drivers/gl1.c

@@ -2867,13 +2867,12 @@ static void gl1_viewport_info(void *data, struct video_viewport *vp)
    VIDEO_POS_PUT_Y(vp->pos, top_dist);
 }
 
-static bool gl1_read_viewport(void *data, uint8_t *buffer, bool is_idle)
+/* bgrx: write bottom-up 32-bit BGRX (read_viewport_bgrx) instead of
+ * bottom-up BGR24. */
+static bool gl1_read_viewport_internal(gl1_t *gl1, uint8_t *buffer,
+      bool is_idle, bool bgrx)
 {
    unsigned num_pixels = 0;
-   gl1_t *gl1          = (gl1_t*)data;
-
-   if (!gl1)
-      return false;
 
    num_pixels                      = VIDEO_SCALE_AREA(gl1->vp.dims);
    gl1->readback_buffer_screenshot = malloc(num_pixels * sizeof(uint32_t));
@@ -2895,13 +2894,19 @@ static bool gl1_read_viewport(void *data, uint8_t *buffer, bool is_idle)
       unsigned vd_h = VIDEO_SCALE_H(gl1->vp.full_dims);
       unsigned rb_w = (VIDEO_SCALE_W(gl1->vp.dims)  > vd_w) ? vd_w : VIDEO_SCALE_W(gl1->vp.dims);
       unsigned rb_h = (VIDEO_SCALE_H(gl1->vp.dims) > vd_h) ? vd_h : VIDEO_SCALE_H(gl1->vp.dims);
-      video_frame_convert_rgba_to_bgr(
-            (const void*)gl1->readback_buffer_screenshot,
-            buffer,
-            rb_w * sizeof(uint32_t),
-            rb_w * 3,
-            rb_w,
-            rb_h);
+      if (bgrx)
+         video_frame_copy_to_bgrx(buffer, VIDEO_SCALE_W(gl1->vp.dims),
+               VIDEO_SCALE_H(gl1->vp.dims),
+               (const uint8_t*)gl1->readback_buffer_screenshot,
+               rb_w * 4, rb_w, rb_h, true);
+      else
+         video_frame_convert_rgba_to_bgr(
+               (const void*)gl1->readback_buffer_screenshot,
+               buffer,
+               rb_w * sizeof(uint32_t),
+               rb_w * 3,
+               rb_w,
+               rb_h);
    }
 
    free(gl1->readback_buffer_screenshot);
@@ -2910,6 +2915,25 @@ static bool gl1_read_viewport(void *data, uint8_t *buffer, bool is_idle)
    return true;
 }
 
+static bool gl1_read_viewport(void *data, uint8_t *buffer, bool is_idle)
+{
+   gl1_t *gl1 = (gl1_t*)data;
+   if (!gl1)
+      return false;
+   return gl1_read_viewport_internal(gl1, buffer, is_idle, false);
+}
+
+static bool gl1_read_viewport_bgrx(void *data, uint8_t *buffer,
+      bool is_idle, bool *bottom_up)
+{
+   gl1_t *gl1 = (gl1_t*)data;
+   if (!gl1)
+      return false;
+   /* glReadPixels returns the bottom row first. */
+   *bottom_up = true;
+   return gl1_read_viewport_internal(gl1, buffer, is_idle, true);
+}
+
 static void gl1_set_texture_frame(void *data,
       const void *frame, bool rgb32, unsigned dims,
       float alpha)
@@ -3664,7 +3688,13 @@ video_driver_t video_gl1 = {
 #endif
    NULL, /* invalidate_hw_render_cache */
    gl1_read_viewport_hdr,
-   &gl1_raster_font
+   &gl1_raster_font,
+#ifdef HAVE_OPENXR
+   NULL, /* get_vr_frame_state */
+   NULL, /* set_vr_content_info */
+   NULL, /* get_video_views_status */
+#endif
+   gl1_read_viewport_bgrx
 };
 
 gfx_display_ctx_driver_t gfx_display_ctx_gl1 = {

gfx/drivers/gl2.c

@@ -2554,7 +2554,8 @@ static void gl2_renderchain_bind_prev_texture(
 }
 
 #ifdef HAVE_GL_ASYNC_READBACK
-static bool gl2_read_pbo(gl2_t *gl, uint8_t *buffer)
+/* bgrx: write 32-bit BGRX (read_viewport_bgrx) instead of BGR24. */
+static bool gl2_read_pbo(gl2_t *gl, uint8_t *buffer, bool bgrx)
 {
    const uint8_t *ptr = NULL;
 #ifdef HAVE_OPENGLES3
@@ -2583,17 +2584,31 @@ static bool gl2_read_pbo(gl2_t *gl, uint8_t *buffer)
             VIDEO_SCALE_W(gl->video_dims));
       unsigned rb_h = MIN(VIDEO_SCALE_H(gl->vp.dims),
             VIDEO_SCALE_H(gl->video_dims));
-      video_frame_convert_rgba_to_bgr(
-            (const void*)ptr,
-            buffer,
-            rb_w * sizeof(uint32_t),
-            rb_w * 3,
-            rb_w,
-            rb_h);
+      if (bgrx)
+         video_frame_copy_to_bgrx(buffer, VIDEO_SCALE_W(gl->vp.dims),
+               VIDEO_SCALE_H(gl->vp.dims), ptr, rb_w * 4, rb_w, rb_h, true);
+      else
+         video_frame_convert_rgba_to_bgr(
+               (const void*)ptr,
+               buffer,
+               rb_w * sizeof(uint32_t),
+               rb_w * 3,
+               rb_w,
+               rb_h);
    }
 #else
    ptr = (const uint8_t*)glMapBuffer(GL_PIXEL_PACK_BUFFER, GL_READ_ONLY);
-   if (ptr)
+   if (ptr && bgrx)
+   {
+      /* Already BGRA (GL_BGRA readback): rows are copied as-is. */
+      unsigned rb_w = MIN(VIDEO_SCALE_W(gl->vp.dims),
+            VIDEO_SCALE_W(gl->video_dims));
+      unsigned rb_h = MIN(VIDEO_SCALE_H(gl->vp.dims),
+            VIDEO_SCALE_H(gl->video_dims));
+      video_frame_copy_to_bgrx(buffer, VIDEO_SCALE_W(gl->vp.dims),
+            VIDEO_SCALE_H(gl->vp.dims), ptr, rb_w * 4, rb_w, rb_h, false);
+   }
+   else if (ptr)
    {
       struct scaler_ctx *ctx = &gl->pbo_readback_scaler;
       scaler_ctx_scale_direct(ctx, buffer, ptr);
@@ -2619,7 +2634,7 @@ static bool gl2_read_pbo(gl2_t *gl, uint8_t *buffer)
 
 static bool gl2_renderchain_read_viewport(
       gl2_t *gl,
-      uint8_t *buffer, bool is_idle)
+      uint8_t *buffer, bool is_idle, bool bgrx)
 {
    size_t num_pixels      = 0;
 
@@ -2653,7 +2668,7 @@ static bool gl2_renderchain_read_viewport(
 #ifdef HAVE_GL_ASYNC_READBACK
    if (gl->flags & GL2_FLAG_PBO_READBACK_ENABLE)
    {
-      if (!gl2_read_pbo(gl, buffer))
+      if (!gl2_read_pbo(gl, buffer, bgrx))
          goto error;
    }
    else
@@ -2686,13 +2701,19 @@ static bool gl2_renderchain_read_viewport(
                VIDEO_SCALE_W(gl->video_dims));
          unsigned rb_h = MIN(VIDEO_SCALE_H(gl->vp.dims),
                VIDEO_SCALE_H(gl->video_dims));
-         video_frame_convert_rgba_to_bgr(
-               (const void*)gl->readback_buffer_screenshot,
-               buffer,
-               rb_w * sizeof(uint32_t),
-               rb_w * 3,
-               rb_w,
-               rb_h);
+         if (bgrx)
+            video_frame_copy_to_bgrx(buffer, VIDEO_SCALE_W(gl->vp.dims),
+                  VIDEO_SCALE_H(gl->vp.dims),
+                  (const uint8_t*)gl->readback_buffer_screenshot,
+                  rb_w * 4, rb_w, rb_h, true);
+         else
+            video_frame_convert_rgba_to_bgr(
+                  (const void*)gl->readback_buffer_screenshot,
+                  buffer,
+                  rb_w * sizeof(uint32_t),
+                  rb_w * 3,
+                  rb_w,
+                  rb_h);
       }
 
       free(gl->readback_buffer_screenshot);
@@ -4018,6 +4039,30 @@ static INLINE void gl2_draw_texture(gl2_t *gl)
 }
 #endif
 
+static void gl2_pbo_async_readback(gl2_t *gl);
+
+/* The recording's per-frame async readback, or its teardown once
+ * recording has stopped. skip_in_menu: don't read back while the menu
+ * is drawn. */
+static void gl2_record_readback(gl2_t *gl, bool skip_in_menu)
+{
+   if (!(gl->flags & GL2_FLAG_PBO_READBACK_ENABLE))
+      return;
+
+   if (!(gl->flags & GL2_FLAG_GPU_RECORDING))
+   {
+      glDeleteBuffers(4, gl->pbo_readback);
+      scaler_ctx_gen_reset(&gl->pbo_readback_scaler);
+      gl->flags &= ~GL2_FLAG_PBO_READBACK_ENABLE;
+   }
+#ifdef HAVE_MENU
+   else if (skip_in_menu && (gl->flags & GL2_FLAG_MENU_TEXTURE_ENABLE))
+      ;
+#endif
+   else
+      gl2_pbo_async_readback(gl);
+}
+
 static void gl2_pbo_async_readback(gl2_t *gl)
 {
 #ifdef HAVE_OPENGLES
@@ -5024,8 +5069,15 @@ static bool gl2_frame(void *data, const void *frame,
     * displayable image. Same choke-point pattern as glcore. */
    if (gl->scrgb.active)
       gl2_bind_fb(gl2_ui_target_fbo(gl));
-   else if (gl2_needs_pq_downconvert(gl))
-      gl2_encode_pq_to_sdr(gl);
+   else
+   {
+      if (gl2_needs_pq_downconvert(gl))
+         gl2_encode_pq_to_sdr(gl);
+      /* Record Game Only: read back before any UI is drawn. Under
+       * scRGB the image only lands at the final composite. */
+      if (video_info->record_game_only)
+         gl2_record_readback(gl, false);
+   }
 
 #ifdef HAVE_OVERLAY
    if ((gl->flags & GL2_FLAG_OVERLAY_ENABLE) && overlay_behind_menu)
@@ -5182,24 +5234,8 @@ static bool gl2_frame(void *data, const void *frame,
             4, GL_RGBA, GL_UNSIGNED_BYTE,
             gl->readback_buffer_screenshot);
 
-   else if (gl->flags & GL2_FLAG_PBO_READBACK_ENABLE)
-   {
-      /* If recording has stopped, tear down PBO readback */
-      if (!(gl->flags & GL2_FLAG_GPU_RECORDING))
-      {
-         glDeleteBuffers(4, gl->pbo_readback);
-         scaler_ctx_gen_reset(&gl->pbo_readback_scaler);
-         gl->flags &= ~GL2_FLAG_PBO_READBACK_ENABLE;
-      }
-      else
-      {
-#ifdef HAVE_MENU
-         /* Don't readback if we're in menu mode. */
-         if (!(gl->flags & GL2_FLAG_MENU_TEXTURE_ENABLE))
-#endif
-            gl2_pbo_async_readback(gl);
-      }
-   }
+   else if (!video_info->record_game_only || gl->scrgb.active)
+      gl2_record_readback(gl, !video_info->record_game_only);
 
    /* The backbuffer is what it is until the swap, so the copy is
     * taken now. Not from the BFI light dupes, which recurse in here
@@ -6616,7 +6652,20 @@ static bool gl2_read_viewport(void *data, uint8_t *buffer, bool is_idle)
    if (!gl)
       return false;
 
-   return gl2_renderchain_read_viewport(gl, buffer, is_idle);
+   return gl2_renderchain_read_viewport(gl, buffer, is_idle, false);
+}
+
+static bool gl2_read_viewport_bgrx(void *data, uint8_t *buffer,
+      bool is_idle, bool *bottom_up)
+{
+   gl2_t *gl = (gl2_t*)data;
+
+   if (!gl)
+      return false;
+
+   /* glReadPixels returns the bottom row first. */
+   *bottom_up = true;
+   return gl2_renderchain_read_viewport(gl, buffer, is_idle, true);
 }
 
 #if defined(HAVE_GL_ASYNC_READBACK) && !defined(HAVE_OPENGLES) && !defined(HAVE_PSGL)
@@ -6639,7 +6688,7 @@ static bool gl2_record_read(void *data, uint8_t *buffer)
          gl->flags &= ~GL2_FLAG_PBO_READBACK_ENABLE;
       return false;
    }
-   return gl2_read_pbo(gl, buffer);
+   return gl2_read_pbo(gl, buffer, false);
 }
 #endif
 
@@ -7575,8 +7624,9 @@ video_driver_t video_gl2 = {
 #ifdef HAVE_OPENXR
    gl2_get_vr_frame_state,
    gl2_set_vr_content_info,
-   gl2_get_video_views_status
+   gl2_get_video_views_status,
 #endif
+   gl2_read_viewport_bgrx
 };
 
 gfx_display_ctx_driver_t gfx_display_ctx_gl = {