Skip to content

Add background capture thread and live mode UI to DebugLayer #13

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Konstantysz merged 1 commit into from
Mar 9, 2026
Merged

Add background capture thread and live mode UI to DebugLayer #13

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
149 changes: 119 additions & 30 deletions src/App/DebugLayer.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,8 @@ namespace SH3DS::App
detector(std::move(detector)),
shinyRoi(std::move(shinyRoi)),
shinyCheckState(std::move(shinyCheckState)),
playback(totalFrames, targetFps)
playback(totalFrames, targetFps),
isLiveSource(seeker == nullptr)
{
// Initialize ImGui
IMGUI_CHECKVERSION();
Expand All @@ -49,14 +50,32 @@ namespace SH3DS::App
topScreenTexture = TextureUploader::CreateTexture();
bottomScreenTexture = TextureUploader::CreateTexture();

// Process first frame
ProcessCurrentFrame();

LOG_INFO("DebugLayer initialized ({} frames, {:.1f} FPS)", totalFrames, targetFps);
if (isLiveSource)
{
this->source->Open();
lastFpsTime = std::chrono::steady_clock::now();
StartCaptureThread();
LOG_INFO("DebugLayer initialized (LIVE mode, {:.1f} FPS target)", static_cast<double>(targetFps));
}
else
{
GrabCurrentReplayFrame();
LOG_INFO("DebugLayer initialized ({} frames, {:.1f} FPS)", totalFrames, static_cast<double>(targetFps));
}
}

DebugLayer::~DebugLayer()
{
// Stop capture thread before tearing down OpenGL/ImGui
if (captureRunning)
{
captureRunning = false;
if (captureThread.joinable())
{
captureThread.join();
}
}

ImGui_ImplOpenGL3_Shutdown();
ImGui_ImplGlfw_Shutdown();
ImGui::DestroyContext();
Expand All @@ -77,10 +96,38 @@ namespace SH3DS::App

void DebugLayer::OnUpdate(float deltaTime)
{
if (isLiveSource)
{
std::optional<Core::Frame> frame;
{
std::lock_guard<std::mutex> lock(latestFrameMutex);
frame = std::move(latestFrame);
latestFrame = std::nullopt;
}

if (frame)
{
ProcessFrame(*frame);
}

// Update FPS estimate once per second
auto now = std::chrono::steady_clock::now();
double elapsed = std::chrono::duration<double>(now - lastFpsTime).count();
if (elapsed >= 1.0)
{
size_t grabbed = totalFramesGrabbed.load();
liveGrabFps = static_cast<float>(static_cast<double>(grabbed - lastFpsCount) / elapsed);
lastFpsCount = grabbed;
lastFpsTime = now;
}

return;
}

bool advanced = playback.Update(deltaTime);
if (advanced)
{
ProcessCurrentFrame();
GrabCurrentReplayFrame();
}
}

Expand Down Expand Up @@ -131,38 +178,22 @@ namespace SH3DS::App
ImGui_ImplOpenGL3_RenderDrawData(ImGui::GetDrawData());
}

// TODO: move pipeline processing to a background thread for live sources (v0.2.0)
void DebugLayer::ProcessCurrentFrame()
void DebugLayer::ProcessFrame(const Core::Frame &frame)
{
size_t frameIndex = playback.GetCurrentFrameIndex();
if (frameIndex == lastProcessedFrame)
{
return;
}

seeker->Seek(frameIndex);

auto frame = source->Grab();
if (!frame)
{
return;
}

currentRawFrame = frame->image.clone();
currentRawFrame = frame.image.clone();
rawWidth = currentRawFrame.cols;
rawHeight = currentRawFrame.rows;

TextureUploader::Upload(currentRawFrame, rawFrameTexture);

if (screenDetector)
{
screenDetector->ApplyTo(*preprocessor, frame->image);
screenDetector->ApplyTo(*preprocessor, frame.image);
}

auto dualScreenResult = preprocessor->ProcessDualScreen(frame->image);
auto dualScreenResult = preprocessor->ProcessDualScreen(frame.image);
if (!dualScreenResult)
{
lastProcessedFrame = frameIndex;
return;
}

Expand Down Expand Up @@ -219,10 +250,52 @@ namespace SH3DS::App
currentShinyResult = detector->Detect(it->second);
}
}
}

void DebugLayer::GrabCurrentReplayFrame()
{
size_t frameIndex = playback.GetCurrentFrameIndex();
if (frameIndex == lastProcessedFrame)
{
return;
}

seeker->Seek(frameIndex);

auto frame = source->Grab();
if (!frame)
{
return;
}

ProcessFrame(*frame);
lastProcessedFrame = frameIndex;
}

void DebugLayer::StartCaptureThread()
{
captureRunning = true;
captureThread = std::thread([this]() {
while (captureRunning)
{
auto frame = source->Grab();
if (frame)
{
{
std::lock_guard<std::mutex> lock(latestFrameMutex);
latestFrame = std::move(frame);
}
++totalFramesGrabbed;
}
else
{
// Avoid busy-spin on error / stream end
std::this_thread::sleep_for(std::chrono::milliseconds(10));
}
}
});
}

void DebugLayer::RenderImagePanel(const char *title, GLuint textureId, int width, int height)
{
ImGui::Begin(title);
Expand Down Expand Up @@ -254,7 +327,14 @@ namespace SH3DS::App
{
ImGui::Begin("State Info");

ImGui::Text("Frame: %zu / %zu", playback.GetCurrentFrameIndex() + 1, playback.GetTotalFrames());
if (isLiveSource)
{
ImGui::Text("Frames: %zu", totalFramesGrabbed.load());
}
else
{
ImGui::Text("Frame: %zu / %zu", playback.GetCurrentFrameIndex() + 1, playback.GetTotalFrames());
}

ImGui::Separator();

Expand Down Expand Up @@ -312,11 +392,20 @@ namespace SH3DS::App
{
ImGui::Begin("Playback");

if (isLiveSource)
{
ImGui::TextColored(ImVec4(1.0f, 0.2f, 0.2f, 1.0f), "● LIVE");
ImGui::SameLine();
ImGui::Text(" %.1f fps | %zu frames", static_cast<double>(liveGrabFps), totalFramesGrabbed.load());
ImGui::End();
return;
}

// Transport buttons
if (ImGui::Button("<<"))
{
playback.StepBackward();
ProcessCurrentFrame();
GrabCurrentReplayFrame();
}
ImGui::SameLine();

Expand All @@ -339,7 +428,7 @@ namespace SH3DS::App
if (ImGui::Button(">>"))
{
playback.StepForward();
ProcessCurrentFrame();
GrabCurrentReplayFrame();
}

// Frame scrubber
Expand All @@ -354,7 +443,7 @@ namespace SH3DS::App
if (ImGui::SliderInt("##frame", &frameIdx, 0, maxFrame, "Frame %d"))
{
playback.SetFrameIndex(static_cast<size_t>(frameIdx));
ProcessCurrentFrame();
GrabCurrentReplayFrame();
}

// Speed control
Expand Down
45 changes: 36 additions & 9 deletions src/App/DebugLayer.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,9 +11,13 @@
#include "PlaybackController.h"
#include "Vision/ShinyDetector.h"

#include <atomic>
#include <chrono>
#include <memory>
#include <mutex>
#include <optional>
#include <string>
#include <thread>

// clang-format off
#include <glad/glad.h>
Expand All @@ -23,7 +27,7 @@
namespace SH3DS::App
{
/**
* @brief ImGui debug GUI layer for offline frame replay and pipeline visualization.
* @brief ImGui debug GUI layer for offline frame replay and live camera pipeline visualization.
*/
class DebugLayer : public Kappa::Layer
{
Expand All @@ -32,13 +36,14 @@ namespace SH3DS::App
* @brief Constructs the debug layer.
* @param window GLFW window handle for ImGui initialization.
* @param source Frame source for streaming.
* @param seeker Non-owning seek interface (same object as source).
* @param seeker Non-owning seek interface — nullptr signals live (non-seekable) mode.
* @param screenDetector Automatic screen corner detector.
* @param preprocessor Frame preprocessor for perspective warp.
* @param fsm Game state FSM.
* @param detector Shiny detector (may be null).
* @param shinyRoi ROI name for shiny detection.
* @param shinyCheckState FSM state in which shiny detection runs.
* @param totalFrames Total number of frames in the source.
* @param totalFrames Total number of frames in replay source (0 for live).
* @param targetFps Target playback FPS.
*/
DebugLayer(GLFWwindow *window,
Expand All @@ -61,9 +66,20 @@ namespace SH3DS::App

private:
/**
* @brief Processes the current frame through the pipeline.
* @brief Processes a single frame through the full pipeline (screen detection, warp, FSM, detector).
* @param frame Frame to process.
*/
void ProcessCurrentFrame();
void ProcessFrame(const Core::Frame &frame);

/**
* @brief Seeks to the playback controller's current index, grabs, and processes (replay only).
*/
void GrabCurrentReplayFrame();

/**
* @brief Starts the background capture thread (live mode only).
*/
void StartCaptureThread();

/**
* @brief Renders an image panel with ImGui::Image.
Expand All @@ -80,23 +96,34 @@ namespace SH3DS::App
void RenderStatePanel();

/**
* @brief Renders the playback controls bar.
* @brief Renders the playback controls bar (scrubber in replay mode, LIVE badge in live mode).
*/
void RenderPlaybackControls();

// Pipeline components
std::unique_ptr<Capture::FrameSource> source; ///< Frame source (streaming)
std::shared_ptr<Capture::FrameSeeker> seeker = nullptr; ///< Non-owning seek interface
std::shared_ptr<Capture::FrameSeeker> seeker = nullptr; ///< Non-owning seek interface; nullptr = live
std::unique_ptr<Capture::ScreenDetector> screenDetector; ///< Automatic screen detection
std::unique_ptr<Capture::FramePreprocessor> preprocessor; ///< Perspective warp
std::unique_ptr<FSM::GameStateFSM> fsm; ///< Game state FSM
std::unique_ptr<Vision::ShinyDetector> detector; ///< Shiny detector
std::string shinyRoi; ///< ROI name for shiny detection
std::string shinyCheckState; ///< FSM state in which shiny detection runs

// Playback
// Playback (replay mode)
PlaybackController playback; ///< Playback state controller

// Live mode
bool isLiveSource = false; ///< True when seeker == nullptr (live camera)
std::thread captureThread; ///< Background capture thread (live only)
std::atomic<bool> captureRunning{ false }; ///< Signals capture thread to stop
std::mutex latestFrameMutex; ///< Guards latestFrame
std::optional<Core::Frame> latestFrame; ///< Latest frame from capture thread
std::atomic<size_t> totalFramesGrabbed{ 0 }; ///< Monotonically increasing grab counter
float liveGrabFps = 0.0f; ///< Estimated grab FPS (render thread)
std::chrono::steady_clock::time_point lastFpsTime; ///< Timestamp for FPS estimation
size_t lastFpsCount = 0; ///< totalFramesGrabbed at last FPS sample

// OpenGL textures
GLuint rawFrameTexture = 0; ///< Texture for raw camera frame
GLuint topScreenTexture = 0; ///< Texture for warped top screen
Expand All @@ -114,7 +141,7 @@ namespace SH3DS::App
std::string currentStateName = "unknown"; ///< Current FSM state
float timeInState = 0.0f; ///< Time in current state (seconds)
std::optional<Core::ShinyResult> currentShinyResult; ///< Latest shiny detection result
size_t lastProcessedFrame = SIZE_MAX; ///< Last processed frame index
size_t lastProcessedFrame = SIZE_MAX; ///< Last processed frame index (replay only)

// Frame dimensions (for display)
int rawWidth = 0; ///< Raw frame width
Expand Down
Loading