From 3c37403ce2ddcb4a507eab274beabfbca4eab4f2 Mon Sep 17 00:00:00 2001 From: johnfxhayes Date: Thu, 8 Jan 2026 17:28:54 -0500 Subject: [PATCH] add object detection user can select an object in a video using a marquee tool and it will track the object --- CMakeLists.txt | 1 + Modules/CMakeLists.txt | 3 +- Modules/m1_objectdetection/.gitignore | 41 + Modules/m1_objectdetection/README.md | 43 + Modules/m1_objectdetection/example_usage.cpp | 151 +++ .../m1_objectdetection/m1_objectdetection.cpp | 864 ++++++++++++++++++ .../m1_objectdetection/m1_objectdetection.h | 186 ++++ Source/CMakeLists.txt | 3 + Source/MainComponent.cpp | 321 ++++++- Source/MainComponent.h | 7 + Source/ObjectTracker.cpp | 266 ++++++ Source/ObjectTracker.h | 160 ++++ Source/UI/MarqueeSelection.h | 314 +++++++ Source/UI/VideoPlayerWidget.h | 139 +++ 14 files changed, 2492 insertions(+), 7 deletions(-) create mode 100644 Modules/m1_objectdetection/.gitignore create mode 100644 Modules/m1_objectdetection/README.md create mode 100644 Modules/m1_objectdetection/example_usage.cpp create mode 100644 Modules/m1_objectdetection/m1_objectdetection.cpp create mode 100644 Modules/m1_objectdetection/m1_objectdetection.h create mode 100644 Source/ObjectTracker.cpp create mode 100644 Source/ObjectTracker.h create mode 100644 Source/UI/MarqueeSelection.h diff --git a/CMakeLists.txt b/CMakeLists.txt index e257996..2a295ab 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -145,6 +145,7 @@ target_link_libraries(${CMAKE_PROJECT_NAME} PRIVATE m1_orientation_client m1_mathematics juce_libvlc + m1_objectdetection ) # definitions to replace the `JucePluginDefines.h` diff --git a/Modules/CMakeLists.txt b/Modules/CMakeLists.txt index 8874c2b..63bd636 100644 --- a/Modules/CMakeLists.txt +++ b/Modules/CMakeLists.txt @@ -26,7 +26,8 @@ list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/juce_libvlc/cmake") juce_add_modules( juce_murka m1_orientation_client - juce_libvlc) + juce_libvlc + m1_objectdetection) # Mach1 Spatial SDK # Use these options to block dependencies we do not need diff --git a/Modules/m1_objectdetection/.gitignore b/Modules/m1_objectdetection/.gitignore new file mode 100644 index 0000000..d4fb281 --- /dev/null +++ b/Modules/m1_objectdetection/.gitignore @@ -0,0 +1,41 @@ +# Prerequisites +*.d + +# Compiled Object files +*.slo +*.lo +*.o +*.obj + +# Precompiled Headers +*.gch +*.pch + +# Linker files +*.ilk + +# Debugger Files +*.pdb + +# Compiled Dynamic libraries +*.so +*.dylib +*.dll + +# Fortran module files +*.mod +*.smod + +# Compiled Static libraries +*.lai +*.la +*.a +*.lib + +# Executables +*.exe +*.out +*.app + +# debug information files +*.dwo diff --git a/Modules/m1_objectdetection/README.md b/Modules/m1_objectdetection/README.md new file mode 100644 index 0000000..b16cde1 --- /dev/null +++ b/Modules/m1_objectdetection/README.md @@ -0,0 +1,43 @@ +# m1_objectdetection +Dependency for JUCE projects to add image or video object detection for auto-panning use cases after a user has supplied a reference object image for the target object to be detected and tracked. + +## Implementation +This module provides a JUCE-compatible object detection system using template matching algorithms. The implementation includes: + +### Core Classes +- `Mach1::ObjectDetector`: Main detection class +- `Mach1::DetectedObject`: Structure representing a detected object with position, bounds, and confidence + +### Key Features +- **Template Matching**: Uses normalized cross-correlation for object detection +- **Proximity-Based Tracking**: Weights detections based on proximity to last known position +- **JUCE Integration**: Native support for `juce::Image` and JUCE data structures +- **Configurable Parameters**: Adjustable confidence threshold and proximity weighting +- **Multi-Object Detection**: Support for detecting multiple instances of the reference object + +### Usage +```cpp +#include "m1_objectdetection.h" + +// Create detector +Mach1::ObjectDetector detector; + +// Set reference object from user selection +juce::Rectangle selection(100, 100, 50, 50); +juce::Image referenceObject = videoFrame.getClippedImage(selection); +detector.setReferenceObject(referenceObject); + +// Process video frames +for (auto& frame : videoFrames) { + auto objectCenter = detector.detectObjectCenter(frame); + if (!objectCenter.isOrigin()) { + // Use objectCenter for auto-panning + float panPosition = (objectCenter.getX() / frame.getWidth() - 0.5f) * 2.0f; + } +} +``` + +### Configuration +- `setConfidenceThreshold(float)`: Set minimum confidence for valid detections (0.0-1.0) +- `setProximityWeight(float)`: Set weight for proximity-based tracking (0.0-1.0) +- `resetTracking()`: Reset tracking state diff --git a/Modules/m1_objectdetection/example_usage.cpp b/Modules/m1_objectdetection/example_usage.cpp new file mode 100644 index 0000000..aed090c --- /dev/null +++ b/Modules/m1_objectdetection/example_usage.cpp @@ -0,0 +1,151 @@ +/******************************************************************************* + Example usage of M1 Object Detection module + This file demonstrates how to use the ObjectDetector class for auto-panning +*******************************************************************************/ + +#include "m1_objectdetection.h" +#include +#include + +// Example of how to use the ObjectDetector in a video player context +class VideoPlayerWithObjectDetection +{ +public: + VideoPlayerWithObjectDetection() + { + // Initialize the object detector + detector = std::make_unique(); + + // Set detection parameters + detector->setConfidenceThreshold(0.6f); // 60% confidence threshold + detector->setProximityWeight(0.4f); // 40% weight for proximity-based tracking + } + + void setReferenceObjectFromSelection(const juce::Image& frameImage, + juce::Rectangle selection) + { + // Extract the selected region as reference object + if (selection.isEmpty() || !frameImage.isValid()) + return; + + // Create a sub-image from the selection + juce::Image referenceObject = frameImage.getClippedImage(selection); + + // Set as reference object for detection + if (detector->setReferenceObject(referenceObject)) + { + juce::Logger::writeToLog("Reference object set successfully"); + } + else + { + juce::Logger::writeToLog("Failed to set reference object"); + } + } + + juce::Point processVideoFrame(const juce::Image& currentFrame) + { + if (!currentFrame.isValid()) + return juce::Point(); + + // Detect the object in the current frame + auto objectCenter = detector->detectObjectCenter(currentFrame); + + if (!objectCenter.isOrigin()) + { + // Convert to normalized coordinates (0.0 to 1.0) + float normalizedX = objectCenter.getX() / currentFrame.getWidth(); + float normalizedY = objectCenter.getY() / currentFrame.getHeight(); + + // Log the detection + juce::Logger::writeToLog( + "Object detected at: (" + + juce::String(normalizedX, 3) + ", " + + juce::String(normalizedY, 3) + ")" + ); + + return juce::Point(normalizedX, normalizedY); + } + + return juce::Point(); + } + + std::vector findMultipleObjects(const juce::Image& currentFrame, + int maxObjects = 3) + { + if (!currentFrame.isValid()) + return {}; + + // Find multiple instances of the object + auto detectedObjects = detector->detectObjects(currentFrame, maxObjects); + + // Log all detections + for (size_t i = 0; i < detectedObjects.size(); ++i) + { + const auto& obj = detectedObjects[i]; + juce::Logger::writeToLog( + "Object " + juce::String(i + 1) + + " detected at: (" + + juce::String(obj.centerPosition.getX(), 1) + ", " + + juce::String(obj.centerPosition.getY(), 1) + + ") with confidence: " + + juce::String(obj.confidence, 3) + ); + } + + return detectedObjects; + } + + void resetObjectTracking() + { + detector->resetTracking(); + juce::Logger::writeToLog("Object tracking reset"); + } + + bool hasReferenceObject() const + { + return detector->hasReferenceObject(); + } + +private: + std::unique_ptr detector; +}; + +// Example of how to integrate with audio panning +class AutoPanningController +{ +public: + AutoPanningController(VideoPlayerWithObjectDetection& videoPlayer) + : videoPlayer(videoPlayer) + { + } + + void updatePanningFromVideoFrame(const juce::Image& frame) + { + if (!videoPlayer.hasReferenceObject()) + return; + + // Get the object position + auto objectPos = videoPlayer.processVideoFrame(frame); + + if (!objectPos.isOrigin()) + { + // Convert normalized position to pan value (-1.0 to 1.0) + float panValue = (objectPos.getX() - 0.5f) * 2.0f; + panValue = juce::jlimit(-1.0f, 1.0f, panValue); + + // Apply panning (this would connect to your audio engine) + applyAudioPanning(panValue); + } + } + +private: + VideoPlayerWithObjectDetection& videoPlayer; + + void applyAudioPanning(float panValue) + { + // This would connect to your Mach1 Spatial System or audio engine + juce::Logger::writeToLog( + "Auto-panning audio to: " + juce::String(panValue, 3) + ); + } +}; \ No newline at end of file diff --git a/Modules/m1_objectdetection/m1_objectdetection.cpp b/Modules/m1_objectdetection/m1_objectdetection.cpp new file mode 100644 index 0000000..ce71739 --- /dev/null +++ b/Modules/m1_objectdetection/m1_objectdetection.cpp @@ -0,0 +1,864 @@ +/******************************************************************************* + Implementation of M1 Object Detection module. + Copyright (c) 2025 - Mach1 +*******************************************************************************/ + +#include "m1_objectdetection.h" + +#if M1_OBJECTDETECTION_ENABLED + +namespace Mach1 +{ + +//============================================================================== +struct ObjectDetector::Impl +{ + // Reference object data (stored at scaled size for matching) + juce::Image referenceImage; // Original reference + juce::Image scaledReferenceImage; // Scaled reference for matching + int refWidth = 0; // Scaled reference width + int refHeight = 0; // Scaled reference height + int originalRefWidth = 0; // Original reference width + int originalRefHeight = 0; // Original reference height + + // Detection parameters + float confidenceThreshold = 0.30f; // Slightly higher to avoid false positives + float proximityWeight = 0.05f; // Very low - almost no proximity bias + + // Scale factor for processing + static constexpr int SCALE_FACTOR = 4; + + // Tracking quality metrics + int consecutiveLowConfidenceFrames = 0; + float lastGoodConfidence = 0.0f; + + // Tracking state + juce::Point lastKnownPosition; // In scaled coordinates + juce::Point velocity; // Movement velocity for prediction + bool hasLastKnownPosition = false; + int framesSinceGoodMatch = 0; // Frames since we had a confident match + + // Multi-scale templates (wider range for significant size changes) + static constexpr int NUM_SCALES = 9; + struct ScaleTemplate { + juce::Image image; + int width = 0; + int height = 0; + float scale = 1.0f; + }; + ScaleTemplate scaleTemplates[NUM_SCALES]; // 50%, 65%, 80%, 100%, 130%, 170%, 220%, 280%, 350% + + // Best matching scale from last frame (to prioritize nearby scales) + int lastBestScaleIndex = 3; // Start with 100% + + // Threading support + class DetectionThread : public juce::Thread + { + public: + DetectionThread(Impl* parent) : Thread("ObjectDetectionThread"), parentImpl(parent) {} + + void run() override + { + while (!threadShouldExit()) + { + // Wait for new frame to process + if (frameReadyEvent.wait(100)) // 100ms timeout + { + if (threadShouldExit()) break; + + // Process the frame on this background thread + processFrame(); + } + } + } + + void submitFrame(const juce::Image& frame) + { + const juce::ScopedLock lock(frameLock); + if (frame.isValid()) + { + frameSkipCounter++; + if (frameSkipCounter >= frameSkipCount) + { + frameSkipCounter = 0; + currentFrame = frame.createCopy(); + frameReadyEvent.signal(); + } + } + } + + void setFrameSkipCount(int count) { frameSkipCount = count; } + void setCallback(ObjectDetector::DetectionCallback cb) { callback = cb; } + + private: + void processFrame() + { + juce::Image frameToProcess; + { + const juce::ScopedLock lock(frameLock); + if (!currentFrame.isValid()) return; + frameToProcess = currentFrame.createCopy(); + } + + if (!parentImpl || !parentImpl->hasReferenceObjectInternal()) + return; + + auto startTime = juce::Time::getMillisecondCounterHiRes(); + + // Scale down the frame for faster processing + int scaledWidth = frameToProcess.getWidth() / SCALE_FACTOR; + int scaledHeight = frameToProcess.getHeight() / SCALE_FACTOR; + + juce::Image scaledFrame = frameToProcess.rescaled(scaledWidth, scaledHeight, + juce::Graphics::lowResamplingQuality); + + // Find best match using template matching + auto detected = parentImpl->findBestMatchFast(scaledFrame); + + auto endTime = juce::Time::getMillisecondCounterHiRes(); + double processingTime = endTime - startTime; + + DBG("[ObjectDetection] Processing: " + juce::String(processingTime, 1) + + "ms, confidence: " + juce::String(detected.confidence, 3)); + + // Scale the detected coordinates back up to original frame size + juce::Point detectedCenter; + if (detected.confidence >= parentImpl->confidenceThreshold) + { + detectedCenter.setX(detected.centerPosition.getX() * SCALE_FACTOR); + detectedCenter.setY(detected.centerPosition.getY() * SCALE_FACTOR); + + // Update tracking state (in scaled coordinates) + parentImpl->lastKnownPosition = detected.centerPosition; + parentImpl->hasLastKnownPosition = true; + + DBG("[ObjectDetection] FOUND at (" + juce::String(detectedCenter.getX(), 0) + + ", " + juce::String(detectedCenter.getY(), 0) + ")"); + } + else + { + DBG("[ObjectDetection] Not found (confidence " + + juce::String(detected.confidence, 3) + " < " + + juce::String(parentImpl->confidenceThreshold, 3) + ")"); + } + + // Call the callback on the message thread + if (callback) + { + juce::MessageManager::callAsync([this, detectedCenter, frameToProcess, processingTime]() { + if (callback) + { + callback(detectedCenter, frameToProcess.getWidth(), frameToProcess.getHeight(), processingTime); + } + }); + } + } + + Impl* parentImpl; + juce::CriticalSection frameLock; + juce::WaitableEvent frameReadyEvent; + juce::Image currentFrame; + int frameSkipCount = 5; + int frameSkipCounter = 0; + ObjectDetector::DetectionCallback callback; + }; + + std::unique_ptr detectionThread; + bool asyncDetectionActive = false; + + // Fast template matching using direct pixel comparison + DetectedObject findBestMatchFast(const juce::Image& scaledFrame); + + // Calculate similarity between two image regions + float calculateSimilarityFast(const juce::Image& frame, int frameX, int frameY, + const juce::Image& reference); + + // Color histogram for reference image (helps distinguish objects with same luminance) + std::array referenceHueHistogram; // 16-bin hue histogram + float referenceAvgSaturation = 0.0f; + + // Calculate hue histogram for a region + void calculateHueHistogram(const juce::Image& img, int x, int y, int w, int h, + std::array& histogram, float& avgSaturation); + + // Compare two histograms + float compareHistograms(const std::array& hist1, const std::array& hist2); + + // Calculate variance of a region (to reject uniform areas like walls) + float calculateRegionVariance(const juce::Image& frame, int x, int y, int w, int h); + + // Reference image variance (for comparison) + float referenceVariance = 0.0f; + + bool hasReferenceObjectInternal() const + { + return scaledReferenceImage.isValid() && refWidth > 0 && refHeight > 0; + } +}; + +//============================================================================== +ObjectDetector::ObjectDetector() + : pimpl(std::make_unique()) +{ +} + +ObjectDetector::~ObjectDetector() +{ + // Stop async processing if active + stopAsyncDetection(); +} + +//============================================================================== +bool ObjectDetector::setReferenceObject(const std::vector>& imageData, + int width, int height, int channels) +{ + // Create a JUCE image from the raw data + juce::Image img(channels == 4 ? juce::Image::ARGB : juce::Image::RGB, width, height, true); + juce::Image::BitmapData bitmapData(img, juce::Image::BitmapData::writeOnly); + + for (int y = 0; y < height; ++y) + { + for (int x = 0; x < width; ++x) + { + int idx = y * width + x; + if (idx < imageData.size() && imageData[idx].size() >= 3) + { + juce::Colour c(imageData[idx][0], imageData[idx][1], imageData[idx][2], + static_cast(channels == 4 && imageData[idx].size() >= 4 ? imageData[idx][3] : 255)); + bitmapData.setPixelColour(x, y, c); + } + } + } + + return setReferenceObject(img); +} + +bool ObjectDetector::setReferenceObject(const juce::Image& referenceImage) +{ + if (!referenceImage.isValid()) + return false; + + pimpl->referenceImage = referenceImage; + pimpl->originalRefWidth = referenceImage.getWidth(); + pimpl->originalRefHeight = referenceImage.getHeight(); + + // Scale reference to match the scaled frame size + pimpl->refWidth = referenceImage.getWidth() / Impl::SCALE_FACTOR; + pimpl->refHeight = referenceImage.getHeight() / Impl::SCALE_FACTOR; + + // Ensure minimum size for matching + pimpl->refWidth = juce::jmax(8, pimpl->refWidth); + pimpl->refHeight = juce::jmax(8, pimpl->refHeight); + + pimpl->scaledReferenceImage = referenceImage.rescaled(pimpl->refWidth, pimpl->refHeight, + juce::Graphics::lowResamplingQuality); + + // Create multi-scale templates for handling significant object distance changes + // Scales: 50%, 65%, 80%, 100%, 130%, 170%, 220%, 280%, 350% + // This allows tracking objects that grow up to 3.5x their original size + const float scales[Impl::NUM_SCALES] = { 0.50f, 0.65f, 0.80f, 1.0f, 1.30f, 1.70f, 2.20f, 2.80f, 3.50f }; + + for (int i = 0; i < Impl::NUM_SCALES; i++) + { + pimpl->scaleTemplates[i].scale = scales[i]; + pimpl->scaleTemplates[i].width = juce::jmax(6, (int)(pimpl->refWidth * scales[i])); + pimpl->scaleTemplates[i].height = juce::jmax(6, (int)(pimpl->refHeight * scales[i])); + pimpl->scaleTemplates[i].image = referenceImage.rescaled( + pimpl->scaleTemplates[i].width, + pimpl->scaleTemplates[i].height, + juce::Graphics::lowResamplingQuality); + } + + pimpl->lastBestScaleIndex = 3; // Reset to 100% (index 3 in our scale array) + + // Reset tracking state + pimpl->hasLastKnownPosition = false; + pimpl->velocity = juce::Point(0, 0); + pimpl->framesSinceGoodMatch = 0; + + // Calculate color histogram for reference + pimpl->calculateHueHistogram(pimpl->scaledReferenceImage, 0, 0, pimpl->refWidth, pimpl->refHeight, + pimpl->referenceHueHistogram, pimpl->referenceAvgSaturation); + + // Calculate variance for reference (people have more texture than walls) + pimpl->referenceVariance = pimpl->calculateRegionVariance(pimpl->scaledReferenceImage, + 0, 0, pimpl->refWidth, pimpl->refHeight); + + DBG("[ObjectDetection] Reference set: original " + + juce::String(pimpl->originalRefWidth) + "x" + juce::String(pimpl->originalRefHeight) + + " -> scaled " + juce::String(pimpl->refWidth) + "x" + juce::String(pimpl->refHeight)); + DBG("[ObjectDetection] Reference variance: " + juce::String(pimpl->referenceVariance, 4)); + DBG("[ObjectDetection] Multi-scale templates: " + + juce::String(pimpl->scaleTemplates[0].width) + "x" + juce::String(pimpl->scaleTemplates[0].height) + " (50%) to " + + juce::String(pimpl->scaleTemplates[Impl::NUM_SCALES-1].width) + "x" + + juce::String(pimpl->scaleTemplates[Impl::NUM_SCALES-1].height) + " (350%)"); + + return true; +} + +//============================================================================== +// ASYNCHRONOUS API (threaded processing) + +bool ObjectDetector::startAsyncDetection(DetectionCallback callback, int frameSkipCount) +{ + if (pimpl->asyncDetectionActive) + return false; // Already running + + if (!callback) + return false; // Invalid callback + + DBG("[ObjectDetection] Starting async detection"); + + pimpl->detectionThread = std::make_unique(pimpl.get()); + pimpl->detectionThread->setCallback(callback); + pimpl->detectionThread->setFrameSkipCount(frameSkipCount); + pimpl->detectionThread->startThread(); + pimpl->asyncDetectionActive = true; + + return true; +} + +void ObjectDetector::stopAsyncDetection() +{ + if (!pimpl->asyncDetectionActive) + return; + + DBG("[ObjectDetection] Stopping async detection"); + + if (pimpl->detectionThread) + { + pimpl->detectionThread->signalThreadShouldExit(); + pimpl->detectionThread->stopThread(1000); + pimpl->detectionThread.reset(); + } + + pimpl->asyncDetectionActive = false; +} + +bool ObjectDetector::submitFrame(const juce::Image& frameImage) +{ + if (!pimpl->asyncDetectionActive || !pimpl->detectionThread) + return false; + + pimpl->detectionThread->submitFrame(frameImage); + return true; +} + +bool ObjectDetector::isAsyncDetectionActive() const +{ + return pimpl->asyncDetectionActive; +} + +//============================================================================== +std::vector ObjectDetector::detectObjects(const std::vector>& frameData, + int frameWidth, int frameHeight, int channels, + int maxMatches) +{ + // Convert to JUCE Image and use that + juce::Image img(channels == 4 ? juce::Image::ARGB : juce::Image::RGB, frameWidth, frameHeight, true); + juce::Image::BitmapData bitmapData(img, juce::Image::BitmapData::writeOnly); + + for (int y = 0; y < frameHeight; ++y) + { + for (int x = 0; x < frameWidth; ++x) + { + int idx = y * frameWidth + x; + if (idx < frameData.size() && frameData[idx].size() >= 3) + { + juce::Colour c(frameData[idx][0], frameData[idx][1], frameData[idx][2], + static_cast(channels == 4 && frameData[idx].size() >= 4 ? frameData[idx][3] : 255)); + bitmapData.setPixelColour(x, y, c); + } + } + } + + return detectObjects(img, maxMatches); +} + +std::vector ObjectDetector::detectObjects(const juce::Image& frameImage, int maxMatches) +{ + std::vector results; + + if (!frameImage.isValid() || !hasReferenceObject()) + return results; + + // Scale the frame + int scaledWidth = frameImage.getWidth() / Impl::SCALE_FACTOR; + int scaledHeight = frameImage.getHeight() / Impl::SCALE_FACTOR; + juce::Image scaledFrame = frameImage.rescaled(scaledWidth, scaledHeight, + juce::Graphics::lowResamplingQuality); + + auto detected = pimpl->findBestMatchFast(scaledFrame); + + if (detected.confidence >= pimpl->confidenceThreshold) + { + // Scale coordinates back up + detected.centerPosition.setX(detected.centerPosition.getX() * Impl::SCALE_FACTOR); + detected.centerPosition.setY(detected.centerPosition.getY() * Impl::SCALE_FACTOR); + detected.bounds = juce::Rectangle( + detected.bounds.getX() * Impl::SCALE_FACTOR, + detected.bounds.getY() * Impl::SCALE_FACTOR, + detected.bounds.getWidth() * Impl::SCALE_FACTOR, + detected.bounds.getHeight() * Impl::SCALE_FACTOR + ); + results.push_back(detected); + } + + return results; +} + +//============================================================================== +void ObjectDetector::setConfidenceThreshold(float threshold) +{ + pimpl->confidenceThreshold = juce::jlimit(0.0f, 1.0f, threshold); +} + +float ObjectDetector::getConfidenceThreshold() const +{ + return pimpl->confidenceThreshold; +} + +void ObjectDetector::setProximityWeight(float weight) +{ + pimpl->proximityWeight = juce::jlimit(0.0f, 1.0f, weight); +} + +float ObjectDetector::getProximityWeight() const +{ + return pimpl->proximityWeight; +} + +void ObjectDetector::resetTracking() +{ + pimpl->hasLastKnownPosition = false; + pimpl->lastKnownPosition = juce::Point(); + pimpl->velocity = juce::Point(0, 0); + pimpl->framesSinceGoodMatch = 0; +} + +bool ObjectDetector::hasReferenceObject() const +{ + return pimpl->scaledReferenceImage.isValid() && pimpl->refWidth > 0 && pimpl->refHeight > 0; +} + +//============================================================================== +// Color histogram functions + +void ObjectDetector::Impl::calculateHueHistogram(const juce::Image& img, int x, int y, int w, int h, + std::array& histogram, float& avgSaturation) +{ + histogram.fill(0); + avgSaturation = 0.0f; + int pixelCount = 0; + + if (!img.isValid()) return; + + juce::Image::BitmapData bitmap(img, juce::Image::BitmapData::readOnly); + + // Sample every 3rd pixel for speed + for (int py = y; py < y + h && py < img.getHeight(); py += 3) + { + for (int px = x; px < x + w && px < img.getWidth(); px += 3) + { + juce::Colour c = bitmap.getPixelColour(px, py); + float hue = c.getHue(); + float saturation = c.getSaturation(); + + // Only count pixels with some saturation (ignore grays) + if (saturation > 0.1f) + { + int bin = (int)(hue * 15.99f); // 0-15 + histogram[bin]++; + avgSaturation += saturation; + pixelCount++; + } + } + } + + if (pixelCount > 0) + { + avgSaturation /= pixelCount; + } +} + +float ObjectDetector::Impl::compareHistograms(const std::array& hist1, const std::array& hist2) +{ + // Calculate histogram intersection (normalized) + int sum1 = 0, sum2 = 0, intersection = 0; + + for (int i = 0; i < 16; i++) + { + sum1 += hist1[i]; + sum2 += hist2[i]; + intersection += juce::jmin(hist1[i], hist2[i]); + } + + if (sum1 == 0 || sum2 == 0) + return 1.0f; // No color info, consider it a match + + return (float)intersection / (float)juce::jmax(sum1, sum2); +} + +float ObjectDetector::Impl::calculateRegionVariance(const juce::Image& frame, int x, int y, int w, int h) +{ + if (!frame.isValid() || w <= 0 || h <= 0) + return 0.0f; + + // Clamp bounds + int endX = juce::jmin(x + w, frame.getWidth()); + int endY = juce::jmin(y + h, frame.getHeight()); + x = juce::jmax(0, x); + y = juce::jmax(0, y); + + if (endX <= x || endY <= y) + return 0.0f; + + juce::Image::BitmapData bitmap(frame, juce::Image::BitmapData::readOnly); + + // First pass: calculate mean luminance (sample every 3rd pixel for speed) + double sum = 0.0; + int count = 0; + + for (int py = y; py < endY; py += 3) + { + for (int px = x; px < endX; px += 3) + { + juce::Colour c = bitmap.getPixelColour(px, py); + float lum = c.getFloatRed() * 0.299f + c.getFloatGreen() * 0.587f + c.getFloatBlue() * 0.114f; + sum += lum; + count++; + } + } + + if (count == 0) return 0.0f; + + double mean = sum / count; + + // Second pass: calculate variance + double variance = 0.0; + for (int py = y; py < endY; py += 3) + { + for (int px = x; px < endX; px += 3) + { + juce::Colour c = bitmap.getPixelColour(px, py); + float lum = c.getFloatRed() * 0.299f + c.getFloatGreen() * 0.587f + c.getFloatBlue() * 0.114f; + double diff = lum - mean; + variance += diff * diff; + } + } + + return static_cast(variance / count); +} + +//============================================================================== +// Fast template matching implementation + +float ObjectDetector::Impl::calculateSimilarityFast(const juce::Image& frame, int frameX, int frameY, + const juce::Image& reference) +{ + if (!frame.isValid() || !reference.isValid()) + return 0.0f; + + int refW = reference.getWidth(); + int refH = reference.getHeight(); + + // Check bounds + if (frameX < 0 || frameY < 0 || + frameX + refW > frame.getWidth() || + frameY + refH > frame.getHeight()) + return 0.0f; + + juce::Image::BitmapData frameBitmap(frame, juce::Image::BitmapData::readOnly); + juce::Image::BitmapData refBitmap(reference, juce::Image::BitmapData::readOnly); + + // Calculate Sum of Squared Differences (SSD) normalized + // Also compute color similarity and frame region variance + double sumDiff = 0.0; + double sumRef = 0.0; + double colorDiff = 0.0; // Track color channel differences + double frameLumSum = 0.0; + double frameLumSqSum = 0.0; + int pixelCount = 0; + + // Sample every 2nd pixel for speed + for (int y = 0; y < refH; y += 2) + { + for (int x = 0; x < refW; x += 2) + { + juce::Colour framePixel = frameBitmap.getPixelColour(frameX + x, frameY + y); + juce::Colour refPixel = refBitmap.getPixelColour(x, y); + + // Use luminance for primary comparison + float frameLum = framePixel.getFloatRed() * 0.299f + + framePixel.getFloatGreen() * 0.587f + + framePixel.getFloatBlue() * 0.114f; + float refLum = refPixel.getFloatRed() * 0.299f + + refPixel.getFloatGreen() * 0.587f + + refPixel.getFloatBlue() * 0.114f; + + float diff = frameLum - refLum; + sumDiff += diff * diff; + sumRef += refLum * refLum; + + // Track frame luminance for variance calculation + frameLumSum += frameLum; + frameLumSqSum += frameLum * frameLum; + + // Add color channel comparison (helps distinguish objects with same brightness) + float rDiff = framePixel.getFloatRed() - refPixel.getFloatRed(); + float gDiff = framePixel.getFloatGreen() - refPixel.getFloatGreen(); + float bDiff = framePixel.getFloatBlue() - refPixel.getFloatBlue(); + colorDiff += (rDiff * rDiff + gDiff * gDiff + bDiff * bDiff) / 3.0f; + + pixelCount++; + } + } + + if (sumRef < 0.001 || pixelCount == 0) + return 0.0f; + + // Calculate variance of the frame region + double frameMean = frameLumSum / pixelCount; + double frameVariance = (frameLumSqSum / pixelCount) - (frameMean * frameMean); + + // REJECT LOW-VARIANCE REGIONS (walls, floors, etc.) + // If the frame region has much less texture than the reference, it's probably not the object + float minVarianceRatio = 0.3f; // Frame must have at least 30% of reference variance + if (referenceVariance > 0.001f) + { + float varianceRatio = static_cast(frameVariance) / referenceVariance; + if (varianceRatio < minVarianceRatio) + { + // This region is too uniform - likely a wall or floor + return 0.0f; + } + + // Penalize regions with lower variance than reference + // (they're less likely to be the textured object we're looking for) + if (varianceRatio < 0.7f) + { + float penalty = (varianceRatio - minVarianceRatio) / (0.7f - minVarianceRatio); + // Apply a soft penalty based on variance difference + sumDiff *= (2.0 - penalty); // Increase the difference for low-variance regions + } + } + + // Normalized luminance similarity (1.0 = perfect match, 0.0 = no match) + float lumSimilarity = static_cast(1.0 - std::sqrt(sumDiff / pixelCount)); + + // Normalized color similarity + float colorSimilarity = static_cast(1.0 - std::sqrt(colorDiff / pixelCount)); + + // Combine luminance and color (70% luminance, 30% color) + float similarity = lumSimilarity * 0.7f + colorSimilarity * 0.3f; + + return juce::jmax(0.0f, similarity); +} + +DetectedObject ObjectDetector::Impl::findBestMatchFast(const juce::Image& scaledFrame) +{ + DetectedObject bestMatch; + float bestSimilarity = 0.0f; + int bestScaleIndex = lastBestScaleIndex; + + if (!scaledReferenceImage.isValid() || refWidth == 0 || refHeight == 0) + return bestMatch; + + int frameW = scaledFrame.getWidth(); + int frameH = scaledFrame.getHeight(); + + // Search step size - larger = faster but less accurate + int step = juce::jmax(2, juce::jmin(refWidth, refHeight) / 4); + + // Lambda to search a region with a given template + auto searchRegion = [&](int startX, int startY, int endX, int endY, int searchStep, + int scaleIdx, bool applyProximityBonus) { + const auto& scale = scaleTemplates[scaleIdx]; + if (!scale.image.isValid()) return; + + for (int y = startY; y <= endY; y += searchStep) + { + for (int x = startX; x <= endX; x += searchStep) + { + if (x + scale.width > frameW || y + scale.height > frameH) + continue; + + float similarity = calculateSimilarityFast(scaledFrame, x, y, scale.image); + + // Only apply proximity bonus if we're confident in tracking + if (applyProximityBonus && hasLastKnownPosition && framesSinceGoodMatch < 3) + { + juce::Point currentCenter(x + scale.width / 2.0f, y + scale.height / 2.0f); + juce::Point predictedPos = lastKnownPosition + velocity; + float distance = predictedPos.getDistanceFrom(currentCenter); + float maxDistance = (float)juce::jmax(refWidth, refHeight) * 4.0f; + if (distance < maxDistance) + { + float proximityBonus = (1.0f - distance / maxDistance) * proximityWeight * 0.05f; + similarity += proximityBonus; + } + } + + if (similarity > bestSimilarity) + { + bestSimilarity = similarity; + bestMatch.centerPosition = juce::Point(x + scale.width / 2.0f, y + scale.height / 2.0f); + // Report bounds using the actual matched template size + bestMatch.bounds = juce::Rectangle((float)x, (float)y, + (float)scale.width, (float)scale.height); + bestMatch.confidence = similarity; + bestScaleIndex = scaleIdx; + } + } + } + }; + + // Determine search strategy based on tracking history + // Be VERY aggressive about doing full search when tracking is uncertain + // Only use focused search if we've had a match in the last frame + bool doFullSearch = !hasLastKnownPosition || framesSinceGoodMatch > 0; + + if (!doFullSearch) + { + // Focused search around predicted position (last position + velocity) + juce::Point predictedPos = lastKnownPosition + velocity; + + // Larger base search radius to handle fast movement + int baseRadius = juce::jmax(refWidth, refHeight) * 4; + int searchRadius = baseRadius + (framesSinceGoodMatch * refWidth * 2); + searchRadius = juce::jmin(searchRadius, juce::jmax(frameW, frameH) / 2); + + // Get the largest template size to ensure we don't go out of bounds + int maxTemplateW = scaleTemplates[NUM_SCALES - 1].width; + int maxTemplateH = scaleTemplates[NUM_SCALES - 1].height; + + int focusStartX = juce::jmax(0, (int)predictedPos.getX() - refWidth/2 - searchRadius); + int focusEndX = juce::jmin(frameW - maxTemplateW, (int)predictedPos.getX() - refWidth/2 + searchRadius); + int focusStartY = juce::jmax(0, (int)predictedPos.getY() - refHeight/2 - searchRadius); + int focusEndY = juce::jmin(frameH - maxTemplateH, (int)predictedPos.getY() - refHeight/2 + searchRadius); + + // Search scales near the last best scale first, then expand + // This prioritizes the current size but allows for size changes + int focusStep = juce::jmax(1, step / 2); + + // Search in order of likelihood: last best scale, then neighbors, then all + std::vector scaleOrder; + scaleOrder.push_back(lastBestScaleIndex); + for (int offset = 1; offset < NUM_SCALES; offset++) + { + if (lastBestScaleIndex + offset < NUM_SCALES) + scaleOrder.push_back(lastBestScaleIndex + offset); + if (lastBestScaleIndex - offset >= 0) + scaleOrder.push_back(lastBestScaleIndex - offset); + } + + for (int scaleIdx : scaleOrder) + { + searchRegion(focusStartX, focusStartY, focusEndX, focusEndY, focusStep, scaleIdx, true); + } + + // If we found a good match, refine it + if (bestSimilarity >= confidenceThreshold) + { + // Pixel-level refinement + int refineX = (int)bestMatch.bounds.getX(); + int refineY = (int)bestMatch.bounds.getY(); + int refineRadius = step; + const auto& bestScale = scaleTemplates[bestScaleIndex]; + + for (int y = juce::jmax(0, refineY - refineRadius); + y <= juce::jmin(frameH - bestScale.height, refineY + refineRadius); y++) + { + for (int x = juce::jmax(0, refineX - refineRadius); + x <= juce::jmin(frameW - bestScale.width, refineX + refineRadius); x++) + { + float similarity = calculateSimilarityFast(scaledFrame, x, y, bestScale.image); + if (similarity > bestSimilarity) + { + bestSimilarity = similarity; + bestMatch.centerPosition = juce::Point(x + bestScale.width / 2.0f, y + bestScale.height / 2.0f); + bestMatch.bounds = juce::Rectangle((float)x, (float)y, + (float)bestScale.width, (float)bestScale.height); + bestMatch.confidence = similarity; + } + } + } + + // Update velocity based on movement + juce::Point newVelocity = bestMatch.centerPosition - lastKnownPosition; + velocity = velocity * 0.5f + newVelocity * 0.5f; // Smooth the velocity + framesSinceGoodMatch = 0; + lastBestScaleIndex = bestScaleIndex; + + DBG("[ObjectDetection] Tracked at scale " + juce::String(scaleTemplates[bestScaleIndex].scale, 2) + + " (" + juce::String(bestScaleIndex) + ")"); + + return bestMatch; + } + } + + // Full frame search (either first frame, or lost tracking) + framesSinceGoodMatch++; + + DBG("[ObjectDetection] Full frame search (framesSinceGoodMatch=" + juce::String(framesSinceGoodMatch) + ")"); + + // When doing full search, DON'T apply any proximity bonus - start fresh + for (int scaleIdx = 0; scaleIdx < NUM_SCALES; scaleIdx++) + { + const auto& scale = scaleTemplates[scaleIdx]; + if (scale.image.isValid()) + { + searchRegion(0, 0, frameW - scale.width, frameH - scale.height, step, scaleIdx, false); + } + } + + // Refine if we found something + if (bestSimilarity > 0.2f) + { + int refineX = (int)bestMatch.bounds.getX(); + int refineY = (int)bestMatch.bounds.getY(); + int refineRadius = step; + const auto& bestScale = scaleTemplates[bestScaleIndex]; + + for (int y = juce::jmax(0, refineY - refineRadius); + y <= juce::jmin(frameH - bestScale.height, refineY + refineRadius); y++) + { + for (int x = juce::jmax(0, refineX - refineRadius); + x <= juce::jmin(frameW - bestScale.width, refineX + refineRadius); x++) + { + float similarity = calculateSimilarityFast(scaledFrame, x, y, bestScale.image); + if (similarity > bestSimilarity) + { + bestSimilarity = similarity; + bestMatch.centerPosition = juce::Point(x + bestScale.width / 2.0f, y + bestScale.height / 2.0f); + bestMatch.bounds = juce::Rectangle((float)x, (float)y, + (float)bestScale.width, (float)bestScale.height); + bestMatch.confidence = similarity; + } + } + } + } + + // If this is a good match after full search, reset tracking state + if (bestSimilarity >= confidenceThreshold) + { + if (hasLastKnownPosition) + { + velocity = bestMatch.centerPosition - lastKnownPosition; + } + else + { + velocity = juce::Point(0, 0); + } + framesSinceGoodMatch = 0; + lastBestScaleIndex = bestScaleIndex; + + DBG("[ObjectDetection] Re-acquired at scale " + juce::String(scaleTemplates[bestScaleIndex].scale, 2)); + } + + return bestMatch; +} + +} // namespace Mach1 + +#endif // M1_OBJECTDETECTION_ENABLED \ No newline at end of file diff --git a/Modules/m1_objectdetection/m1_objectdetection.h b/Modules/m1_objectdetection/m1_objectdetection.h new file mode 100644 index 0000000..5dff642 --- /dev/null +++ b/Modules/m1_objectdetection/m1_objectdetection.h @@ -0,0 +1,186 @@ +/******************************************************************************* + The block below describes the properties of this module, and is read by + the Projucer to automatically generate project code that uses it. + For details about the syntax and how to create or use a module, see the + JUCE Module Format.md file. + + BEGIN_JUCE_MODULE_DECLARATION + + ID: m1_objectdetection + vendor: Mach1 + version: 0.0.1 + name: Mach1 Object Detection + description: Object detection and tracking for auto-panning use cases + website: https://mach1.tech + license: Proprietary + dependencies: juce_core juce_graphics juce_gui_basics + + END_JUCE_MODULE_DECLARATION + +*******************************************************************************/ + +#pragma once + +#include +#include +#include + +#include +#include +#include + +//============================================================================== +/** Config: M1_OBJECTDETECTION_ENABLED + Enables the object detection functionality. Set to 0 to disable. +*/ +#ifndef M1_OBJECTDETECTION_ENABLED + #define M1_OBJECTDETECTION_ENABLED 1 +#endif + +#if M1_OBJECTDETECTION_ENABLED + +namespace Mach1 +{ + +//============================================================================== +/** A structure representing a detected object with its position and confidence score. */ +struct DetectedObject +{ + juce::Point centerPosition; /**< Center X,Y coordinate of the detected object */ + juce::Rectangle bounds; /**< Bounding rectangle of the detected object */ + float confidence; /**< Confidence score (0.0 to 1.0) */ + + DetectedObject() : confidence(0.0f) {} + DetectedObject(juce::Point center, juce::Rectangle rect, float conf) + : centerPosition(center), bounds(rect), confidence(conf) {} +}; + +//============================================================================== +/** + Object detection and tracking class for auto-panning use cases. + + This class provides functionality to detect and track objects in video frames + based on a reference object image supplied by the user. + + Supports asynchronous (threaded) operation mode. +*/ +class ObjectDetector +{ +public: + //============================================================================== + /** Callback function type for asynchronous object detection results. + + @param detectedCenter Center coordinate of detected object (0,0 if not found) + @param frameWidth Width of the processed frame + @param frameHeight Height of the processed frame + @param processingTimeMs Time taken to process the frame in milliseconds + */ + using DetectionCallback = std::function detectedCenter, + int frameWidth, int frameHeight, + double processingTimeMs)>; + + //============================================================================== + ObjectDetector(); + ~ObjectDetector(); + + //============================================================================== + /** Sets the reference object image for detection. + + @param imageData 2D rectangular vector of pixels representing the target object + @param width Width of the reference image + @param height Height of the reference image + @param channels Number of color channels (1 for grayscale, 3 for RGB, 4 for RGBA) + @returns true if the reference image was successfully set + */ + bool setReferenceObject(const std::vector>& imageData, + int width, int height, int channels); + + /** Sets the reference object image from a JUCE Image. + + @param referenceImage JUCE Image containing the reference object + @returns true if the reference image was successfully set + */ + bool setReferenceObject(const juce::Image& referenceImage); + + //============================================================================== + // ASYNCHRONOUS API (threaded processing) + + /** Starts asynchronous object detection processing. + + @param callback Function to call when object detection completes + @param frameSkipCount Process every Nth frame (default: 5) + @returns true if threading was started successfully + */ + bool startAsyncDetection(DetectionCallback callback, int frameSkipCount = 5); + + /** Stops asynchronous object detection processing. */ + void stopAsyncDetection(); + + /** Submits a frame for asynchronous processing. + + @param frameImage JUCE Image containing the current frame + @returns true if frame was successfully queued + */ + bool submitFrame(const juce::Image& frameImage); + + /** Returns whether asynchronous processing is currently active. */ + bool isAsyncDetectionActive() const; + + //============================================================================== + /** Returns a vector of closest matching objects with confidence scores. + + @param frameData 2D rectangular vector of pixels representing the current frame + @param frameWidth Width of the current frame + @param frameHeight Height of the current frame + @param channels Number of color channels + @param maxMatches Maximum number of matches to return + @returns Vector of DetectedObject structures with confidence scores + */ + std::vector detectObjects(const std::vector>& frameData, + int frameWidth, int frameHeight, int channels, + int maxMatches = 5); + + /** Returns a vector of closest matching objects from a JUCE Image. + + @param frameImage JUCE Image containing the current frame + @param maxMatches Maximum number of matches to return + @returns Vector of DetectedObject structures with confidence scores + */ + std::vector detectObjects(const juce::Image& frameImage, int maxMatches = 5); + + //============================================================================== + /** Sets the minimum confidence threshold for object detection. + + @param threshold Minimum confidence score (0.0 to 1.0) for a detection to be considered valid + */ + void setConfidenceThreshold(float threshold); + + /** Gets the current confidence threshold. */ + float getConfidenceThreshold() const; + + /** Sets the proximity weight factor for tracking continuity. + + @param weight Weight factor (0.0 to 1.0) for proximity-based tracking + */ + void setProximityWeight(float weight); + + /** Gets the current proximity weight factor. */ + float getProximityWeight() const; + + /** Resets the tracking state (clears last known object position). */ + void resetTracking(); + + /** Returns whether a reference object has been set. */ + bool hasReferenceObject() const; + +private: + //============================================================================== + struct Impl; + std::unique_ptr pimpl; + + JUCE_DECLARE_NON_COPYABLE_WITH_LEAK_DETECTOR(ObjectDetector) +}; + +} // namespace Mach1 + +#endif // M1_OBJECTDETECTION_ENABLED \ No newline at end of file diff --git a/Source/CMakeLists.txt b/Source/CMakeLists.txt index 38a54b5..229caee 100644 --- a/Source/CMakeLists.txt +++ b/Source/CMakeLists.txt @@ -4,6 +4,8 @@ file(GLOB source_files Config.h MainComponent.cpp MediaPlayer.h MediaPlayer.cpp + ObjectTracker.h + ObjectTracker.cpp TypesForDataExchange.h SphereMeshGenerator.h PlayerOSC.h @@ -13,6 +15,7 @@ file(GLOB source_files Config.h UI/M1DropdownButton.h UI/M1DropdownMenu.h UI/VideoPlayerWidget.h + UI/MarqueeSelection.h UI/RadioGroupWidget.h UI/M1PlayerControlButton.h UI/M1PlayerControls.h) diff --git a/Source/MainComponent.cpp b/Source/MainComponent.cpp index ddffecc..7295d63 100644 --- a/Source/MainComponent.cpp +++ b/Source/MainComponent.cpp @@ -30,6 +30,19 @@ MainComponent::MainComponent() : m_decode_strategy(&MainComponent::nullStrategy) // Setup OSC playerOSC = std::make_unique(); + + // Setup Object Tracker + objectTracker = std::make_unique(); + objectTracker->onTrackingUpdate = [this](const ObjectTracker::TrackingResult& result) { + // This callback is called on the message thread when tracking updates + DBG("[ObjectTracker] Object found at: " + + juce::String(result.normalizedPosition.x, 3) + ", " + + juce::String(result.normalizedPosition.y, 3) + + " (confidence: " + juce::String(result.confidence, 2) + ")"); + }; + objectTracker->onTrackingLost = [this]() { + DBG("[ObjectTracker] Tracking lost"); + }; // print build time for debug juce::String date(__DATE__); @@ -833,6 +846,9 @@ void MainComponent::draw() syncWithDAWPlayhead(); } + // Update object tracking with current video frame + updateObjectTracking(); + // update video frame if (currentMedia.clipLoaded() && currentMedia.hasVideo()) { @@ -861,6 +877,43 @@ void MainComponent::draw() m.setFontFromRawData(PLUGIN_FONT, BINARYDATA_FONT, BINARYDATA_FONT_SIZE, DEFAULT_FONT_SIZE); auto& videoPlayerWidget = m.prepare({ 0, 0, m.getWindowWidth(), m.getWindowHeight() }); + + // Configure object tracking for the video player widget + videoPlayerWidget.objectSelectionEnabled = objectSelectionModeEnabled; + videoPlayerWidget.objectTrackingActive = objectTracker && objectTracker->isTracking(); + + // Set video frame dimensions for coordinate conversion + if (currentMedia.clipLoaded() && currentMedia.hasVideo()) + { + juce::Image& frame = currentMedia.getFrame(); + if (frame.isValid()) + { + videoPlayerWidget.videoFrameWidth = frame.getWidth(); + videoPlayerWidget.videoFrameHeight = frame.getHeight(); + } + } + + // Update tracking result visualization + if (objectTracker && objectTracker->isTracking()) + { + auto result = objectTracker->getLatestResult(); + videoPlayerWidget.setTrackingResult( + result.objectFound, + result.centerPosition, + result.bounds, + result.confidence, + result.processingTimeMs + ); + } + + // Handle object selection callback + videoPlayerWidget.onObjectSelected = [this](juce::Rectangle selection) { + handleObjectSelection(selection); + }; + + videoPlayerWidget.onSelectionCancelled = [this]() { + objectSelectionModeEnabled = false; + }; auto vid_rot = Mach1::Float3{ videoPlayerWidget.rotationCurrent.x, videoPlayerWidget.rotationCurrent.y, videoPlayerWidget.rotationCurrent.z }.EulerRadians(); currentOrientation.SetRotation(vid_rot); @@ -1165,6 +1218,21 @@ void MainComponent::draw() if (m.isKeyPressed('o')) { videoPlayerWidget.drawOverlay = !videoPlayerWidget.drawOverlay; } + + // Toggle object tracking selection mode + if (m.isKeyPressed('t')) { + if (currentMedia.clipLoaded() && currentMedia.hasVideo() && videoPlayerWidget.drawFlat) { + if (objectTracker && objectTracker->isTracking()) { + // Stop tracking if already tracking + objectTracker->stopTracking(); + objectTracker->clearReference(); + objectSelectionModeEnabled = false; + } else { + // Toggle selection mode + objectSelectionModeEnabled = !objectSelectionModeEnabled; + } + } + } if (m.isKeyPressed('d')) { // Cycle through stereoscopic modes: OFF -> TB -> LR @@ -1280,14 +1348,15 @@ void MainComponent::draw() m.getCurrentFont()->drawString("[g] - Overlay 2D Reference", 10, 210); m.getCurrentFont()->drawString("[o] - Overlay Reference", 10, 230); m.getCurrentFont()->drawString("[d] - Cycle stereoscopic modes (Off/TB/LR)", 10, 250); - m.getCurrentFont()->drawString("[h] - Hide UI", 10, 290); - m.getCurrentFont()->drawString("[Arrow Keys] - Orientation Resets", 10, 310); + m.getCurrentFont()->drawString("[t] - Object tracking selection (2D mode)", 10, 270); + m.getCurrentFont()->drawString("[h] - Hide UI", 10, 310); + m.getCurrentFont()->drawString("[Arrow Keys] - Orientation Resets", 10, 330); auto ori_deg = currentOrientation.GetGlobalRotationAsEulerDegrees(); - m.getCurrentFont()->drawString("OverlayCoords:", 10, 350); - m.getCurrentFont()->drawString("Y: " + std::to_string(ori_deg.GetYaw()), 10, 370); - m.getCurrentFont()->drawString("P: " + std::to_string(ori_deg.GetPitch()), 10, 390); - m.getCurrentFont()->drawString("R: " + std::to_string(ori_deg.GetRoll()), 10, 410); + m.getCurrentFont()->drawString("OverlayCoords:", 10, 370); + m.getCurrentFont()->drawString("Y: " + std::to_string(ori_deg.GetYaw()), 10, 390); + m.getCurrentFont()->drawString("P: " + std::to_string(ori_deg.GetPitch()), 10, 410); + m.getCurrentFont()->drawString("R: " + std::to_string(ori_deg.GetRoll()), 10, 430); } std::function deleteTheSettingsButton = [&]() { @@ -1683,6 +1752,189 @@ void MainComponent::draw() .draw(); m.setColor(ENABLED_PARAM); } + + // OBJECT TRACKING SECTION + float objectTracking_y_position = stereo_y_position + 60; + + juceFontStash::Rectangle ot_label_box = m.getCurrentFont()->getStringBoundingBox("OBJECT TRACKING", 0, 0); + m.setColor(ENABLED_PARAM); + m.prepare({ + leftSide_LeftBound_x, + objectTracking_y_position, + ot_label_box.width + 20, ot_label_box.height + }) + .text("OBJECT TRACKING") + .withAlignment(TEXT_LEFT) + .draw(); + + // Object tracking controls - only available in 2D mode with video + bool trackingAvailable = currentMedia.clipLoaded() && currentMedia.hasVideo() && videoPlayerWidget.drawFlat; + + if (trackingAvailable) + { + // Select Object Button + float button_y = objectTracking_y_position + 25; + bool isTracking = objectTracker && objectTracker->isTracking(); + bool hasReference = objectTracker && objectTracker->hasReference(); + + // "SELECT OBJECT FOR TRACKING" / "CANCEL SELECTION" / "STOP TRACKING" button + std::string trackButtonLabel = objectSelectionModeEnabled ? "CANCEL SELECTION" : + (isTracking ? "STOP TRACKING" : "SELECT OBJECT FOR TRACKING"); + + // Set border color based on state + MurkaColor borderColor; + if (objectSelectionModeEnabled) { + borderColor = MurkaColor(255, 136, 68); // Orange + } else if (isTracking) { + borderColor = MurkaColor(68, 255, 136); // Green + } else { + borderColor = MurkaColor(ENABLED_PARAM); // Light gray + } + + m.prepare({ + leftSide_LeftBound_x, button_y, + 210, 24 + }) + .withText(trackButtonLabel) + .withTextAlignment(TEXT_CENTER) + .withVerticalTextOffset(4) + .withStrokeBorder(borderColor) + .withBackgroundFill(MurkaColor(BACKGROUND_COMPONENT), MurkaColor(BACKGROUND_GREY)) + .withOnClickFlash() + .withOnClickCallback([&]() { + if (objectSelectionModeEnabled) + { + // Cancel selection mode + objectSelectionModeEnabled = false; + } + else if (isTracking) + { + // Stop tracking + objectTracker->stopTracking(); + objectTracker->clearReference(); + } + else + { + // Close settings menu and enter selection mode + showSettingsMenu = false; + objectSelectionModeEnabled = true; + // Make sure we're in 2D mode for selection + videoPlayerWidget.drawFlat = true; + } + }) + .draw(); + + // Show tracking status indicator below the button + if (isTracking) + { + auto trackResult = objectTracker->getLatestResult(); + std::string statusText = trackResult.objectFound ? + "Tracking: " + juce::String(trackResult.processingTimeMs, 1).toStdString() + "ms" : + "Searching..."; + + m.setColor(trackResult.objectFound ? 0x88FF88FF : 0xFFFF88FF); + m.setFontFromRawData(PLUGIN_FONT, BINARYDATA_FONT, BINARYDATA_FONT_SIZE, DEFAULT_FONT_SIZE - 3); + m.prepare({ + leftSide_LeftBound_x, button_y + 28, + 200, 18 + }) + .text(statusText) + .withAlignment(TEXT_LEFT) + .draw(); + m.setFontFromRawData(PLUGIN_FONT, BINARYDATA_FONT, BINARYDATA_FONT_SIZE, DEFAULT_FONT_SIZE); + } + else if (objectSelectionModeEnabled) + { + m.setColor(0xFFFF8844); + m.setFontFromRawData(PLUGIN_FONT, BINARYDATA_FONT, BINARYDATA_FONT_SIZE, DEFAULT_FONT_SIZE - 3); + m.prepare({ + leftSide_LeftBound_x, button_y + 28, + 200, 18 + }) + .text("Draw a box around object") + .withAlignment(TEXT_LEFT) + .draw(); + m.setFontFromRawData(PLUGIN_FONT, BINARYDATA_FONT, BINARYDATA_FONT_SIZE, DEFAULT_FONT_SIZE); + } + + m.setColor(ENABLED_PARAM); + } + else + { + float button_y = objectTracking_y_position + 25; + + // Check what's preventing tracking + bool hasVideo = currentMedia.clipLoaded() && currentMedia.hasVideo(); + bool isIn2DMode = videoPlayerWidget.drawFlat; + + if (!currentMedia.clipLoaded()) + { + // No video loaded - show disabled message + m.setColor(DISABLED_PARAM); + m.setFontFromRawData(PLUGIN_FONT, BINARYDATA_FONT, BINARYDATA_FONT_SIZE, DEFAULT_FONT_SIZE - 2); + m.prepare({ + leftSide_LeftBound_x, button_y, + 200, 18 + }) + .text("Load a video first") + .withAlignment(TEXT_LEFT) + .draw(); + m.setFontFromRawData(PLUGIN_FONT, BINARYDATA_FONT, BINARYDATA_FONT_SIZE, DEFAULT_FONT_SIZE); + m.setColor(ENABLED_PARAM); + } + else if (!hasVideo) + { + // Audio-only file - show disabled message + m.setColor(DISABLED_PARAM); + m.setFontFromRawData(PLUGIN_FONT, BINARYDATA_FONT, BINARYDATA_FONT_SIZE, DEFAULT_FONT_SIZE - 2); + m.prepare({ + leftSide_LeftBound_x, button_y, + 200, 18 + }) + .text("Audio-only file") + .withAlignment(TEXT_LEFT) + .draw(); + m.setFontFromRawData(PLUGIN_FONT, BINARYDATA_FONT, BINARYDATA_FONT_SIZE, DEFAULT_FONT_SIZE); + m.setColor(ENABLED_PARAM); + } + else + { + // Video loaded but in 3D mode - show clickable button + m.prepare({ + leftSide_LeftBound_x, button_y, + 210, 24 + }) + .withText("SELECT OBJECT FOR TRACKING") + .withTextAlignment(TEXT_CENTER) + .withVerticalTextOffset(4) + .withStrokeBorder(MurkaColor(ENABLED_PARAM)) + .withBackgroundFill(MurkaColor(BACKGROUND_COMPONENT), MurkaColor(BACKGROUND_GREY)) + .withOnClickFlash() + .withOnClickCallback([&]() { + // Close settings menu + showSettingsMenu = false; + // Switch to 2D mode + videoPlayerWidget.drawFlat = true; + drawReference = false; + // Enable object selection mode + objectSelectionModeEnabled = true; + }) + .draw(); + + // Show hint text + m.setColor(DISABLED_PARAM); + m.setFontFromRawData(PLUGIN_FONT, BINARYDATA_FONT, BINARYDATA_FONT_SIZE, DEFAULT_FONT_SIZE - 3); + m.prepare({ + leftSide_LeftBound_x, button_y + 28, + 210, 14 + }) + .text("(Will switch to 2D view)") + .withAlignment(TEXT_LEFT) + .draw(); + m.setFontFromRawData(PLUGIN_FONT, BINARYDATA_FONT, BINARYDATA_FONT_SIZE, DEFAULT_FONT_SIZE); + m.setColor(ENABLED_PARAM); + } + } /// RIGHT SIDE @@ -2240,3 +2492,60 @@ void MainComponent::loadRecentFileList() DBG("Loaded " + juce::String(recentFiles.size()) + " recent files"); } + +//============================================================================== +// Object Tracking Methods + +void MainComponent::handleObjectSelection(const juce::Rectangle& selection) +{ + if (!currentMedia.clipLoaded() || !currentMedia.hasVideo()) + { + DBG("[ObjectTracker] Cannot set reference - no video loaded"); + return; + } + + // Get the current video frame + juce::Image& frame = currentMedia.getFrame(); + if (!frame.isValid()) + { + DBG("[ObjectTracker] Cannot set reference - invalid frame"); + return; + } + + // Set the reference object from the selection + if (objectTracker->setReferenceFromSelection(frame, selection)) + { + DBG("[ObjectTracker] Reference object set successfully"); + + // Start tracking automatically + if (objectTracker->startTracking(3)) // Process every 3rd frame + { + objectSelectionModeEnabled = false; // Exit selection mode + DBG("[ObjectTracker] Tracking started"); + } + } + else + { + // Show error + showErrorPopup = true; + errorMessage = "OBJECT TRACKING ERROR"; + errorMessageInfo = "Failed to set reference object. Try selecting a larger area."; + errorStartTime = std::chrono::steady_clock::now(); + } +} + +void MainComponent::updateObjectTracking() +{ + if (!objectTracker || !objectTracker->isTracking()) + return; + + if (!currentMedia.clipLoaded() || !currentMedia.hasVideo()) + return; + + // Get the current video frame and submit it for tracking + juce::Image& frame = currentMedia.getFrame(); + if (frame.isValid()) + { + objectTracker->submitFrame(frame); + } +} diff --git a/Source/MainComponent.h b/Source/MainComponent.h index 509a13a..100fa0e 100644 --- a/Source/MainComponent.h +++ b/Source/MainComponent.h @@ -14,6 +14,7 @@ #include "PlayerOSC.h" #include "MediaPlayer.h" +#include "ObjectTracker.h" #include "UI/M1PlayerControls.h" #include "UI/M1Checkbox.h" @@ -198,6 +199,12 @@ class MainComponent : public murka::JuceMurkaBaseComponent, // Consolidate the media and transport into a single object class MediaPlayer currentMedia; + + // Object tracking for auto-panning + std::unique_ptr objectTracker; + bool objectSelectionModeEnabled = false; + void handleObjectSelection(const juce::Rectangle& selection); + void updateObjectTracking(); bool b_standalone_mode = false; bool b_wants_to_switch_to_standalone = false; diff --git a/Source/ObjectTracker.cpp b/Source/ObjectTracker.cpp new file mode 100644 index 0000000..824dfb3 --- /dev/null +++ b/Source/ObjectTracker.cpp @@ -0,0 +1,266 @@ +#include "ObjectTracker.h" + +//============================================================================== +ObjectTracker::ObjectTracker() + : detector(std::make_unique()) +{ + // Set default parameters optimized for video tracking + // Lower threshold for better detection (template matching typically gives lower scores) + detector->setConfidenceThreshold(0.3f); + detector->setProximityWeight(0.4f); +} + +ObjectTracker::~ObjectTracker() +{ + stopTracking(); +} + +//============================================================================== +bool ObjectTracker::setReferenceFromSelection(const juce::Image& videoFrame, + const juce::Rectangle& selectionRect) +{ + if (!videoFrame.isValid() || selectionRect.isEmpty()) + return false; + + // Ensure selection is within bounds + auto clampedRect = selectionRect.getIntersection(videoFrame.getBounds()); + if (clampedRect.isEmpty() || clampedRect.getWidth() < 10 || clampedRect.getHeight() < 10) + { + DBG("[ObjectTracker] Selection too small or out of bounds"); + return false; + } + + // Extract the selected region as reference + referenceImage = videoFrame.getClippedImage(clampedRect); + + if (!detector->setReferenceObject(referenceImage)) + { + DBG("[ObjectTracker] Failed to set reference object"); + referenceImage = juce::Image(); + return false; + } + + DBG("[ObjectTracker] Reference object set: " + + juce::String(clampedRect.getWidth()) + "x" + + juce::String(clampedRect.getHeight())); + + return true; +} + +void ObjectTracker::clearReference() +{ + stopTracking(); + detector = std::make_unique(); + detector->setConfidenceThreshold(0.3f); + detector->setProximityWeight(0.4f); + referenceImage = juce::Image(); + trackingMode = TrackingMode::Disabled; + currentSelection = juce::Rectangle(); + + { + const juce::ScopedLock lock(resultLock); + latestResult = TrackingResult(); + } + + DBG("[ObjectTracker] Reference cleared"); +} + +bool ObjectTracker::hasReference() const +{ + return detector->hasReferenceObject(); +} + +//============================================================================== +bool ObjectTracker::startTracking(int frameSkipCount) +{ + if (!hasReference()) + { + DBG("[ObjectTracker] Cannot start tracking without reference object"); + return false; + } + + if (isTracking()) + return true; // Already tracking + + auto callback = [this](juce::Point center, int width, int height, double time) { + handleDetectionResult(center, width, height, time); + }; + + if (detector->startAsyncDetection(callback, frameSkipCount)) + { + trackingMode = TrackingMode::Tracking; + DBG("[ObjectTracker] Tracking started (frameSkip: " + juce::String(frameSkipCount) + ")"); + return true; + } + + DBG("[ObjectTracker] Failed to start tracking"); + return false; +} + +void ObjectTracker::stopTracking() +{ + if (detector->isAsyncDetectionActive()) + { + detector->stopAsyncDetection(); + DBG("[ObjectTracker] Tracking stopped"); + } + + if (trackingMode == TrackingMode::Tracking) + trackingMode = TrackingMode::Disabled; +} + +bool ObjectTracker::isTracking() const +{ + return detector->isAsyncDetectionActive(); +} + +//============================================================================== +bool ObjectTracker::submitFrame(const juce::Image& videoFrame) +{ + if (!isTracking() || !videoFrame.isValid()) + return false; + + lastFrameWidth = videoFrame.getWidth(); + lastFrameHeight = videoFrame.getHeight(); + + return detector->submitFrame(videoFrame); +} + +ObjectTracker::TrackingResult ObjectTracker::getLatestResult() const +{ + const juce::ScopedLock lock(resultLock); + return latestResult; +} + +//============================================================================== +void ObjectTracker::setConfidenceThreshold(float threshold) +{ + detector->setConfidenceThreshold(threshold); +} + +float ObjectTracker::getConfidenceThreshold() const +{ + return detector->getConfidenceThreshold(); +} + +void ObjectTracker::setProximityWeight(float weight) +{ + detector->setProximityWeight(weight); +} + +float ObjectTracker::getProximityWeight() const +{ + return detector->getProximityWeight(); +} + +void ObjectTracker::resetTrackingState() +{ + detector->resetTracking(); + + { + const juce::ScopedLock lock(resultLock); + latestResult.objectFound = false; + } + + DBG("[ObjectTracker] Tracking state reset"); +} + +//============================================================================== +void ObjectTracker::beginSelection(juce::Point startPoint) +{ + trackingMode = TrackingMode::Selecting; + selectionStartPoint = startPoint; + currentSelection = juce::Rectangle(startPoint.x, startPoint.y, 0, 0); +} + +void ObjectTracker::updateSelection(juce::Point currentPoint) +{ + if (trackingMode != TrackingMode::Selecting) + return; + + // Create rectangle from start and current points + float left = juce::jmin(selectionStartPoint.x, currentPoint.x); + float top = juce::jmin(selectionStartPoint.y, currentPoint.y); + float right = juce::jmax(selectionStartPoint.x, currentPoint.x); + float bottom = juce::jmax(selectionStartPoint.y, currentPoint.y); + + currentSelection = juce::Rectangle(left, top, right - left, bottom - top); +} + +void ObjectTracker::endSelection() +{ + if (trackingMode != TrackingMode::Selecting) + return; + + // Selection will be processed when user confirms + // Keep the selection visible until the reference is set + trackingMode = TrackingMode::Disabled; +} + +void ObjectTracker::cancelSelection() +{ + trackingMode = TrackingMode::Disabled; + currentSelection = juce::Rectangle(); + selectionStartPoint = juce::Point(); +} + +//============================================================================== +void ObjectTracker::handleDetectionResult(juce::Point detectedCenter, + int frameWidth, int frameHeight, + double processingTimeMs) +{ + TrackingResult result; + result.processingTimeMs = processingTimeMs; + + if (!detectedCenter.isOrigin() && frameWidth > 0 && frameHeight > 0) + { + result.objectFound = true; + result.centerPosition = detectedCenter; + result.normalizedPosition.setX(detectedCenter.x / static_cast(frameWidth)); + result.normalizedPosition.setY(detectedCenter.y / static_cast(frameHeight)); + result.confidence = detector->getConfidenceThreshold(); // Approximate + + // Estimate bounds based on reference size + if (referenceImage.isValid()) + { + float refW = static_cast(referenceImage.getWidth()); + float refH = static_cast(referenceImage.getHeight()); + result.bounds = juce::Rectangle( + detectedCenter.x - refW / 2.0f, + detectedCenter.y - refH / 2.0f, + refW, refH + ); + } + } + else + { + result.objectFound = false; + } + + { + const juce::ScopedLock lock(resultLock); + latestResult = result; + } + + // Notify callbacks on message thread + if (result.objectFound) + { + if (onTrackingUpdate) + { + juce::MessageManager::callAsync([this, result]() { + if (onTrackingUpdate) + onTrackingUpdate(result); + }); + } + } + else + { + if (onTrackingLost) + { + juce::MessageManager::callAsync([this]() { + if (onTrackingLost) + onTrackingLost(); + }); + } + } +} diff --git a/Source/ObjectTracker.h b/Source/ObjectTracker.h new file mode 100644 index 0000000..d6ce448 --- /dev/null +++ b/Source/ObjectTracker.h @@ -0,0 +1,160 @@ +#pragma once + +#include +#include + +/** + * ObjectTracker - A wrapper class for integrating object detection with the M1-Player. + * + * This class manages the object detection lifecycle, provides a simple API for: + * - Setting a reference object from a marquee selection + * - Processing video frames asynchronously + * - Getting tracked object position for visualization + * - Converting tracked positions to audio panning values + */ +class ObjectTracker +{ +public: + //============================================================================== + /** Tracking mode enumeration */ + enum class TrackingMode + { + Disabled, // No tracking active + Selecting, // User is drawing marquee selection + Tracking // Actively tracking the selected object + }; + + /** Tracking result structure */ + struct TrackingResult + { + bool objectFound = false; + juce::Point centerPosition; // Pixel position in video frame + juce::Point normalizedPosition; // 0.0 to 1.0 normalized position + juce::Rectangle bounds; // Bounding rectangle + float confidence = 0.0f; + double processingTimeMs = 0.0; + }; + + //============================================================================== + ObjectTracker(); + ~ObjectTracker(); + + //============================================================================== + /** Sets the reference object from a selection on a video frame. + @param videoFrame The current video frame image + @param selectionRect The rectangle selected by the user (in video frame coordinates) + @returns true if the reference was set successfully + */ + bool setReferenceFromSelection(const juce::Image& videoFrame, + const juce::Rectangle& selectionRect); + + /** Clears the current reference object and stops tracking. */ + void clearReference(); + + /** Returns whether a reference object has been set. */ + bool hasReference() const; + + //============================================================================== + /** Starts asynchronous tracking. + @param frameSkipCount Process every Nth frame (default: 3 for better responsiveness) + @returns true if tracking started successfully + */ + bool startTracking(int frameSkipCount = 3); + + /** Stops asynchronous tracking. */ + void stopTracking(); + + /** Returns the current tracking mode. */ + TrackingMode getTrackingMode() const { return trackingMode; } + + /** Sets the tracking mode. */ + void setTrackingMode(TrackingMode mode) { trackingMode = mode; } + + /** Returns whether tracking is currently active. */ + bool isTracking() const; + + //============================================================================== + /** Submits a video frame for asynchronous processing. + @param videoFrame The current video frame + @returns true if the frame was submitted successfully + */ + bool submitFrame(const juce::Image& videoFrame); + + /** Gets the latest tracking result. Thread-safe. */ + TrackingResult getLatestResult() const; + + //============================================================================== + /** Sets the confidence threshold for detection. + @param threshold Value between 0.0 and 1.0 (default: 0.5) + */ + void setConfidenceThreshold(float threshold); + + /** Gets the current confidence threshold. */ + float getConfidenceThreshold() const; + + /** Sets the proximity weight for tracking continuity. + @param weight Value between 0.0 and 1.0 (default: 0.4) + */ + void setProximityWeight(float weight); + + /** Gets the current proximity weight. */ + float getProximityWeight() const; + + /** Resets the tracking state (useful when seeking in video). */ + void resetTrackingState(); + + //============================================================================== + // Selection State Management + + /** Called when user starts drawing a selection. */ + void beginSelection(juce::Point startPoint); + + /** Called as user drags to update selection. */ + void updateSelection(juce::Point currentPoint); + + /** Called when user finishes drawing selection. */ + void endSelection(); + + /** Cancels the current selection. */ + void cancelSelection(); + + /** Returns the current selection rectangle (in normalized 0-1 coordinates). */ + juce::Rectangle getCurrentSelection() const { return currentSelection; } + + /** Returns whether a selection is currently being drawn. */ + bool isSelecting() const { return trackingMode == TrackingMode::Selecting; } + + //============================================================================== + /** Callback when tracking result is updated. Called on message thread. */ + std::function onTrackingUpdate; + + /** Callback when tracking is lost. Called on message thread. */ + std::function onTrackingLost; + +private: + //============================================================================== + std::unique_ptr detector; + TrackingMode trackingMode = TrackingMode::Disabled; + + // Selection state + juce::Point selectionStartPoint; + juce::Rectangle currentSelection; + + // Thread-safe result storage + mutable juce::CriticalSection resultLock; + TrackingResult latestResult; + + // Frame dimensions (updated with each submitted frame) + std::atomic lastFrameWidth { 0 }; + std::atomic lastFrameHeight { 0 }; + + // Reference image for visualization + juce::Image referenceImage; + + //============================================================================== + void handleDetectionResult(juce::Point detectedCenter, + int frameWidth, int frameHeight, + double processingTimeMs); + + JUCE_DECLARE_NON_COPYABLE_WITH_LEAK_DETECTOR(ObjectTracker) +}; diff --git a/Source/UI/MarqueeSelection.h b/Source/UI/MarqueeSelection.h new file mode 100644 index 0000000..1d936fb --- /dev/null +++ b/Source/UI/MarqueeSelection.h @@ -0,0 +1,314 @@ +#pragma once + +#include "../JuceLibraryCode/JuceHeader.h" +#include "MurkaBasicWidgets.h" +#include "juce_murka/JuceMurkaBaseComponent.h" +#include "../ObjectTracker.h" + +/** + * MarqueeSelection - A Murka widget for drawing selection rectangles on video frames. + * + * Features: + * - Click and drag to create selection rectangle + * - Visual feedback with animated dashed border + * - Clear button to cancel selection + * - Callbacks for selection events + */ +class MarqueeSelection : public View +{ +public: + // Styling options + juce::Colour selectionColor = juce::Colour(0xFF00D4FF); // Cyan selection border + juce::Colour selectionFillColor = juce::Colour(0x3000D4FF); // Semi-transparent cyan fill + juce::Colour trackingColor = juce::Colour(0xFF00FF88); // Green for active tracking + juce::Colour lostTrackingColor = juce::Colour(0xFFFF4444); // Red when tracking lost + float borderWidth = 2.0f; + float cornerRadius = 3.0f; + + // State + bool selectionEnabled = false; // Set to true to enable selection mode + bool isSelecting = false; + bool hasSelection = false; + bool hideSelectionWhenTracking = true; // Hide selection rectangle when tracking is active + juce::Rectangle selectionRect; + + // Tracking visualization + bool showTrackingResult = false; + juce::Rectangle trackingRect; + float trackingConfidence = 0.0f; + bool trackingLost = false; + + // Animation + float dashOffset = 0.0f; + float pulsePhase = 0.0f; + + // Callbacks + std::function)> onSelectionComplete; + std::function onSelectionCancelled; + + void internalDraw(Murka& m) + { + // Update animation + dashOffset += 0.5f; + if (dashOffset > 20.0f) dashOffset = 0.0f; + pulsePhase += 0.1f; + if (pulsePhase > 2.0f * M_PI) pulsePhase -= 2.0f * M_PI; + + auto viewSize = getSize(); + + // Handle mouse interaction for selection + if (selectionEnabled) + { + if (mouseDownPressed(0) && inside()) + { + // Start new selection + isSelecting = true; + hasSelection = false; + selectionStartPoint = mousePosition(); + selectionRect = juce::Rectangle( + selectionStartPoint.x, selectionStartPoint.y, 0, 0 + ); + } + + if (isSelecting && mouseDown(0)) + { + // Update selection rectangle + auto currentPos = mousePosition(); + float left = juce::jmin(selectionStartPoint.x, currentPos.x); + float top = juce::jmin(selectionStartPoint.y, currentPos.y); + float right = juce::jmax(selectionStartPoint.x, currentPos.x); + float bottom = juce::jmax(selectionStartPoint.y, currentPos.y); + + // Clamp to view bounds + left = juce::jmax(0.0f, left); + top = juce::jmax(0.0f, top); + right = juce::jmin(viewSize.x, right); + bottom = juce::jmin(viewSize.y, bottom); + + selectionRect = juce::Rectangle(left, top, right - left, bottom - top); + } + + if (isSelecting && !mouseDown(0)) + { + // Finish selection + isSelecting = false; + + // Only keep selection if it's large enough + if (selectionRect.getWidth() > 20 && selectionRect.getHeight() > 20) + { + hasSelection = true; + if (onSelectionComplete) + { + // Convert to normalized coordinates (0-1) + juce::Rectangle normalizedRect( + selectionRect.getX() / viewSize.x, + selectionRect.getY() / viewSize.y, + selectionRect.getWidth() / viewSize.x, + selectionRect.getHeight() / viewSize.y + ); + onSelectionComplete(normalizedRect); + } + } + else + { + selectionRect = juce::Rectangle(); + } + } + } + + // Draw active tracking result + if (showTrackingResult && !trackingRect.isEmpty()) + { + float pulse = 0.7f + 0.3f * std::sin(pulsePhase); + + if (trackingLost) + { + // Draw lost tracking indicator (red pulsing) + m.setColor(lostTrackingColor.getRed(), lostTrackingColor.getGreen(), + lostTrackingColor.getBlue(), (int)(180 * pulse)); + } + else + { + // Draw active tracking indicator (green) + m.setColor(trackingColor.getRed(), trackingColor.getGreen(), + trackingColor.getBlue(), (int)(200 * pulse)); + } + + // Draw tracking rectangle + drawDashedRect(m, trackingRect, borderWidth * 1.5f); + + // Draw crosshair at center + float cx = trackingRect.getCentreX(); + float cy = trackingRect.getCentreY(); + float crossSize = juce::jmin(trackingRect.getWidth(), trackingRect.getHeight()) * 0.2f; + + m.setColor(255, 255, 255, (int)(200 * pulse)); + m.drawLine(cx - crossSize, cy, cx + crossSize, cy); + m.drawLine(cx, cy - crossSize, cx, cy + crossSize); + + // Draw confidence indicator + if (!trackingLost && trackingConfidence > 0) + { + m.setFontFromRawData(PLUGIN_FONT, BINARYDATA_FONT, BINARYDATA_FONT_SIZE, DEFAULT_FONT_SIZE - 4); + std::string confText = juce::String(trackingConfidence * 100, 0).toStdString() + "%"; + m.setColor(255, 255, 255, 200); + m.getCurrentFont()->drawString(confText, + trackingRect.getX() + 5, + trackingRect.getY() - 5); + } + } + + // Draw selection rectangle if selecting or has selection + // But hide it when tracking is active (showTrackingResult means we're tracking) + bool shouldShowSelection = (isSelecting || hasSelection) && !selectionRect.isEmpty(); + if (hideSelectionWhenTracking && showTrackingResult) + shouldShowSelection = false; + + if (shouldShowSelection) + { + // Draw semi-transparent fill + m.setColor(selectionFillColor.getRed(), selectionFillColor.getGreen(), + selectionFillColor.getBlue(), selectionFillColor.getAlpha()); + m.drawRectangle(selectionRect.getX(), selectionRect.getY(), + selectionRect.getWidth(), selectionRect.getHeight()); + + // Draw animated dashed border + m.setColor(selectionColor.getRed(), selectionColor.getGreen(), + selectionColor.getBlue(), 255); + drawDashedRect(m, selectionRect, borderWidth); + + // Draw corner handles + float handleSize = 8.0f; + drawHandle(m, selectionRect.getTopLeft(), handleSize); + drawHandle(m, selectionRect.getTopRight(), handleSize); + drawHandle(m, selectionRect.getBottomLeft(), handleSize); + drawHandle(m, selectionRect.getBottomRight(), handleSize); + + // Draw dimension text + if (isSelecting) + { + m.setFontFromRawData(PLUGIN_FONT, BINARYDATA_FONT, BINARYDATA_FONT_SIZE, DEFAULT_FONT_SIZE - 2); + std::string dimText = juce::String((int)selectionRect.getWidth()).toStdString() + + " x " + + juce::String((int)selectionRect.getHeight()).toStdString(); + m.setColor(255, 255, 255, 220); + m.getCurrentFont()->drawString(dimText, + selectionRect.getX() + 5, + selectionRect.getBottom() + 15); + } + } + + // Draw selection mode indicator + if (selectionEnabled && !isSelecting && !hasSelection) + { + // Draw crosshair cursor hint + auto mousePos = mousePosition(); + if (inside()) + { + m.setColor(selectionColor.getRed(), selectionColor.getGreen(), + selectionColor.getBlue(), 150); + m.drawLine(mousePos.x - 10, mousePos.y, mousePos.x + 10, mousePos.y); + m.drawLine(mousePos.x, mousePos.y - 10, mousePos.x, mousePos.y + 10); + } + } + } + + /** Sets the tracking result for visualization. + @param rect The bounding rectangle of the tracked object (in view coordinates) + @param confidence Confidence value 0-1 + @param lost Whether tracking was lost + */ + void setTrackingResult(const juce::Rectangle& rect, float confidence, bool lost) + { + trackingRect = rect; + trackingConfidence = confidence; + trackingLost = lost; + showTrackingResult = true; + } + + /** Clears the tracking result visualization. */ + void clearTrackingResult() + { + showTrackingResult = false; + trackingRect = juce::Rectangle(); + } + + /** Clears the current selection. */ + void clearSelection() + { + hasSelection = false; + isSelecting = false; + selectionRect = juce::Rectangle(); + if (onSelectionCancelled) + onSelectionCancelled(); + } + + /** Gets the selection rectangle in normalized coordinates (0-1). */ + juce::Rectangle getNormalizedSelection() + { + auto viewSize = getSize(); + if (viewSize.x <= 0 || viewSize.y <= 0) + return juce::Rectangle(); + + return juce::Rectangle( + selectionRect.getX() / viewSize.x, + selectionRect.getY() / viewSize.y, + selectionRect.getWidth() / viewSize.x, + selectionRect.getHeight() / viewSize.y + ); + } + +private: + MurkaPoint selectionStartPoint; + + void drawDashedRect(Murka& m, const juce::Rectangle& rect, float lineWidth) + { + // Draw dashed rectangle + float dashLen = 8.0f; + float gapLen = 4.0f; + float totalLen = dashLen + gapLen; + + // Top edge + float offset = std::fmod(dashOffset, totalLen); + for (float x = rect.getX() - offset; x < rect.getRight(); x += totalLen) + { + float startX = juce::jmax(rect.getX(), x); + float endX = juce::jmin(rect.getRight(), x + dashLen); + if (startX < endX) + m.drawLine(startX, rect.getY(), endX, rect.getY()); + } + + // Bottom edge + for (float x = rect.getX() - offset; x < rect.getRight(); x += totalLen) + { + float startX = juce::jmax(rect.getX(), x); + float endX = juce::jmin(rect.getRight(), x + dashLen); + if (startX < endX) + m.drawLine(startX, rect.getBottom(), endX, rect.getBottom()); + } + + // Left edge + for (float y = rect.getY() - offset; y < rect.getBottom(); y += totalLen) + { + float startY = juce::jmax(rect.getY(), y); + float endY = juce::jmin(rect.getBottom(), y + dashLen); + if (startY < endY) + m.drawLine(rect.getX(), startY, rect.getX(), endY); + } + + // Right edge + for (float y = rect.getY() - offset; y < rect.getBottom(); y += totalLen) + { + float startY = juce::jmax(rect.getY(), y); + float endY = juce::jmin(rect.getBottom(), y + dashLen); + if (startY < endY) + m.drawLine(rect.getRight(), startY, rect.getRight(), endY); + } + } + + void drawHandle(Murka& m, juce::Point point, float size) + { + m.enableFill(); + m.drawRectangle(point.x - size/2, point.y - size/2, size, size); + } +}; diff --git a/Source/UI/VideoPlayerWidget.h b/Source/UI/VideoPlayerWidget.h index f170955..841cca6 100644 --- a/Source/UI/VideoPlayerWidget.h +++ b/Source/UI/VideoPlayerWidget.h @@ -6,6 +6,7 @@ #include "m1_orientation_client/UI/M1Label.h" #include "../MeshGenerator.h" #include "../TypesForDataExchange.h" +#include "MarqueeSelection.h" class VideoPlayerSurface : public View { private: @@ -271,6 +272,83 @@ class VideoPlayerWidget : public View { rotationOffsetMouse = videoPlayerSurface.rotationOffsetMouse; isUpdatedRotation = videoPlayerSurface.isUpdatedRotation; + + // Draw object tracking selection overlay (only in 2D/flat mode for now) + if (objectSelectionEnabled || objectTrackingActive) + { + auto& marquee = m.prepare({ 0, 0, getSize().x, getSize().y }); + marquee.selectionEnabled = objectSelectionEnabled && drawFlat; + marquee.selectionColor = juce::Colour(0xFF00D4FF); + marquee.trackingColor = juce::Colour(0xFF00FF88); + + // Update tracking visualization + if (objectTrackingActive && trackingResult.objectFound) + { + // Clear the selection rectangle when tracking is active and object found + marquee.hasSelection = false; + marquee.selectionRect = juce::Rectangle(); + + // Convert normalized tracking position to view coordinates + float viewW = getSize().x; + float viewH = getSize().y; + + juce::Rectangle trackRect( + trackingResult.bounds.getX() / static_cast(videoFrameWidth) * viewW, + trackingResult.bounds.getY() / static_cast(videoFrameHeight) * viewH, + trackingResult.bounds.getWidth() / static_cast(videoFrameWidth) * viewW, + trackingResult.bounds.getHeight() / static_cast(videoFrameHeight) * viewH + ); + + marquee.setTrackingResult(trackRect, trackingResult.confidence, false); + } + else if (objectTrackingActive && !trackingResult.objectFound && trackingResult.processingTimeMs > 0) + { + // Clear the selection when tracking (even if searching/lost) + marquee.hasSelection = false; + marquee.selectionRect = juce::Rectangle(); + + // Show "lost" state if we were tracking but lost the object + marquee.setTrackingResult(lastKnownTrackingRect, 0.0f, true); + } + else if (objectTrackingActive) + { + // Still searching - hide selection but don't show tracking rect yet + marquee.hasSelection = false; + marquee.selectionRect = juce::Rectangle(); + } + else + { + marquee.clearTrackingResult(); + } + + // Handle selection callbacks + marquee.onSelectionComplete = [this](juce::Rectangle normalizedSelection) { + if (onObjectSelected) + { + // Convert from view coordinates to video frame coordinates + juce::Rectangle frameSelection( + static_cast(normalizedSelection.getX() * videoFrameWidth), + static_cast(normalizedSelection.getY() * videoFrameHeight), + static_cast(normalizedSelection.getWidth() * videoFrameWidth), + static_cast(normalizedSelection.getHeight() * videoFrameHeight) + ); + onObjectSelected(frameSelection); + } + }; + + marquee.onSelectionCancelled = [this]() { + if (onSelectionCancelled) + onSelectionCancelled(); + }; + + marquee.draw(); + + // Store if user made a new selection + if (marquee.hasSelection) + { + currentSelection = marquee.getNormalizedSelection(); + } + } } bool drawFlat = false; @@ -293,4 +371,65 @@ class VideoPlayerWidget : public View { bool isUpdatedRotation = false; float playheadPosition = 0.0; + + //============================================================================== + // Object Tracking Support + + /** Enable/disable object selection mode (marquee drawing) */ + bool objectSelectionEnabled = false; + + /** Whether object tracking is currently active */ + bool objectTrackingActive = false; + + /** Current tracking result for visualization */ + struct TrackingResultView { + bool objectFound = false; + juce::Point centerPosition; + juce::Rectangle bounds; + float confidence = 0.0f; + double processingTimeMs = 0.0; + } trackingResult; + + /** Video frame dimensions (needed for coordinate conversion) */ + int videoFrameWidth = 0; + int videoFrameHeight = 0; + + /** Current normalized selection rectangle */ + juce::Rectangle currentSelection; + + /** Callback when user completes a selection (provides rect in video frame coordinates) */ + std::function)> onObjectSelected; + + /** Callback when selection is cancelled */ + std::function onSelectionCancelled; + + /** Sets the tracking result for visualization */ + void setTrackingResult(bool found, juce::Point center, + juce::Rectangle bounds, float confidence, double timeMs) + { + trackingResult.objectFound = found; + trackingResult.centerPosition = center; + trackingResult.bounds = bounds; + trackingResult.confidence = confidence; + trackingResult.processingTimeMs = timeMs; + + if (found) + { + lastKnownTrackingRect = juce::Rectangle( + bounds.getX() / static_cast(videoFrameWidth) * getSize().x, + bounds.getY() / static_cast(videoFrameHeight) * getSize().y, + bounds.getWidth() / static_cast(videoFrameWidth) * getSize().x, + bounds.getHeight() / static_cast(videoFrameHeight) * getSize().y + ); + } + } + + /** Clears the tracking result */ + void clearTrackingResult() + { + trackingResult = TrackingResultView(); + } + +private: + juce::Rectangle lastKnownTrackingRect; };