diff --git a/docker/main/rootfs/usr/local/go2rtc/create_config.py b/docker/main/rootfs/usr/local/go2rtc/create_config.py index 77cfc1b880e..70cb744f135 100644 --- a/docker/main/rootfs/usr/local/go2rtc/create_config.py +++ b/docker/main/rootfs/usr/local/go2rtc/create_config.py @@ -3,6 +3,7 @@ import json import os import sys +from pathlib import Path from typing import Any from ruamel.yaml import YAML diff --git a/docs/docs/integrations/mqtt.md b/docs/docs/integrations/mqtt.md index 44d6ecee0ba..730f20259d9 100644 --- a/docs/docs/integrations/mqtt.md +++ b/docs/docs/integrations/mqtt.md @@ -185,7 +185,8 @@ Message published when [object classification](/configuration/custom_classificat "timestamp": 1607123958.748393, "model": "person_classifier", "sub_label": "delivery_person", - "score": 0.87 + "score": 0.87, + "zones": ["front_yard", "driveway"] } ``` @@ -199,10 +200,15 @@ Message published when [object classification](/configuration/custom_classificat "timestamp": 1607123958.748393, "model": "helmet_detector", "attribute": "yes", - "score": 0.92 + "score": 0.92, + "zones": ["front_yard"] } ``` +:::note +The `zones` field is only included if the tracked object is currently in one or more zones. +::: + ### `frigate/reviews` Message published for each changed review item. The first message is published when the `detection` or `alert` is initiated. diff --git a/docs/static/frigate-api.yaml b/docs/static/frigate-api.yaml index af3a019b4c4..766db5df8e6 100644 --- a/docs/static/frigate-api.yaml +++ b/docs/static/frigate-api.yaml @@ -1654,7 +1654,8 @@ paths: **Access:** Admin role required. Categorizes a specific classification image for a given classification model and category. - The image must exist in the specified category. Returns a success message or an error if the name or category is invalid. + Accepts either a training file from the train directory or an event_id to extract + the object crop from. Returns a success message or an error if the name or category is invalid. operationId: categorize_classification_image_classification__name__dataset_categorize_post parameters: diff --git a/frigate/api/camera.py b/frigate/api/camera.py index 0b01ada1d52..dd24c0a9e7e 100644 --- a/frigate/api/camera.py +++ b/frigate/api/camera.py @@ -647,7 +647,9 @@ async def _connect_onvif_camera( return onvif_camera # Both encodings failed authentication; surface the original fault. - raise first_error + if first_error is not None: + raise first_error + raise ONVIFError("Failed to connect to ONVIF device with both password encodings") @router.get( diff --git a/frigate/api/classification.py b/frigate/api/classification.py index 6dd7055092e..e6b2f8ec893 100644 --- a/frigate/api/classification.py +++ b/frigate/api/classification.py @@ -1061,7 +1061,8 @@ def rename_classification_category( dependencies=[Depends(require_role(["admin"]))], summary="Categorize a classification image", description="""Categorizes a specific classification image for a given classification model and category. - The image must exist in the specified category. Returns a success message or an error if the name or category is invalid.""", + Accepts either a training file from the train directory or an event_id to extract + the object crop from. Returns a success message or an error if the name or category is invalid.""", ) def categorize_classification_image(request: Request, name: str, body: dict = None): config: FrigateConfig = request.app.frigate_config @@ -1080,19 +1081,17 @@ def categorize_classification_image(request: Request, name: str, body: dict = No json: dict[str, Any] = body or {} category = sanitize_filename(json.get("category", "")) training_file_name = sanitize_filename(json.get("training_file", "")) - training_file = os.path.join( - CLIPS_DIR, sanitize_filename(name), "train", training_file_name - ) + event_id = json.get("event_id") - if training_file_name and not os.path.isfile(training_file): + if not training_file_name and not event_id: return JSONResponse( content=( { "success": False, - "message": f"Invalid filename or no file exists: {training_file_name}", + "message": "A training file or event_id must be passed.", } ), - status_code=404, + status_code=400, ) random_id = "".join(random.choices(string.ascii_lowercase + string.digits, k=6)) @@ -1104,10 +1103,116 @@ def categorize_classification_image(request: Request, name: str, body: dict = No os.makedirs(new_file_folder, exist_ok=True) - # use opencv because webp images can not be used to train - img = cv2.imread(training_file) - cv2.imwrite(os.path.join(new_file_folder, new_name), img) - os.unlink(training_file) + if training_file_name: + # Use existing training file + training_file = os.path.join( + CLIPS_DIR, sanitize_filename(name), "train", training_file_name + ) + + if not os.path.isfile(training_file): + return JSONResponse( + content=( + { + "success": False, + "message": f"Invalid filename or no file exists: {training_file_name}", + } + ), + status_code=404, + ) + + # use opencv because webp images can not be used to train + img = cv2.imread(training_file) + cv2.imwrite(os.path.join(new_file_folder, new_name), img) + os.unlink(training_file) + else: + # Extract from event + try: + event: Event = Event.get(Event.id == event_id) + except DoesNotExist: + return JSONResponse( + content=( + { + "success": False, + "message": f"Invalid event_id or no event exists: {event_id}", + } + ), + status_code=404, + ) + + snapshot = get_event_snapshot(event) + + if snapshot is None: + return JSONResponse( + content=( + { + "success": False, + "message": f"Failed to read snapshot for event {event_id}.", + } + ), + status_code=500, + ) + + # Get object bounding box for the first detection + if not event.data.get("attributes") or len(event.data["attributes"]) == 0: + return JSONResponse( + content=( + { + "success": False, + "message": f"Event {event_id} has no detection attributes.", + } + ), + status_code=400, + ) + + # Use the first attribute's box + box = event.data["attributes"][0]["box"] + + try: + # Extract the crop from the snapshot + frame = snapshot + + height, width = frame.shape[:2] + + # Convert relative coordinates to absolute + x1 = int(box[0] * width) + y1 = int(box[1] * height) + x2 = int(box[2] * width) + y2 = int(box[3] * height) + + # Ensure coordinates are within frame boundaries + x1 = max(0, x1) + y1 = max(0, y1) + x2 = min(width, x2) + y2 = min(height, y2) + + # Extract the crop + crop = frame[y1:y2, x1:x2] + + if crop.size == 0: + return JSONResponse( + content=( + { + "success": False, + "message": f"Failed to extract crop from event {event_id}.", + } + ), + status_code=500, + ) + + # Save the crop + cv2.imwrite(os.path.join(new_file_folder, new_name), crop) + + except Exception as e: + logger.error(f"Failed to extract classification crop: {e}") + return JSONResponse( + content=( + { + "success": False, + "message": f"Failed to process event {event_id}: {str(e)}", + } + ), + status_code=500, + ) return JSONResponse( content=({"success": True, "message": "Successfully categorized image."}), diff --git a/frigate/camera/activity_manager.py b/frigate/camera/activity_manager.py index 38425add9a5..dcda96a9687 100644 --- a/frigate/camera/activity_manager.py +++ b/frigate/camera/activity_manager.py @@ -26,8 +26,16 @@ def __init__( self.last_camera_activity: dict[str, dict[str, Any]] = {} self.camera_all_object_counts: dict[str, Counter] = {} self.camera_active_object_counts: dict[str, Counter] = {} + self.camera_all_label_counts: dict[str, Counter] = {} + self.camera_active_label_counts: dict[str, Counter] = {} + self.camera_all_attribute_counts: dict[str, Counter] = {} + self.camera_active_attribute_counts: dict[str, Counter] = {} self.zone_all_object_counts: dict[str, Counter] = {} self.zone_active_object_counts: dict[str, Counter] = {} + self.zone_all_label_counts: dict[str, Counter] = {} + self.zone_active_label_counts: dict[str, Counter] = {} + self.zone_all_attribute_counts: dict[str, Counter] = {} + self.zone_active_attribute_counts: dict[str, Counter] = {} self.all_zone_labels: dict[str, set[str]] = {} for camera_config in config.cameras.values(): @@ -43,11 +51,19 @@ def __init_camera(self, camera_config: CameraConfig) -> None: self.last_camera_activity[camera_config.name] = {} self.camera_all_object_counts[camera_config.name] = Counter() self.camera_active_object_counts[camera_config.name] = Counter() + self.camera_all_label_counts[camera_config.name] = Counter() + self.camera_active_label_counts[camera_config.name] = Counter() + self.camera_all_attribute_counts[camera_config.name] = Counter() + self.camera_active_attribute_counts[camera_config.name] = Counter() for zone, zone_config in camera_config.zones.items(): if zone not in self.all_zone_labels: self.zone_all_object_counts[zone] = Counter() self.zone_active_object_counts[zone] = Counter() + self.zone_all_label_counts[zone] = Counter() + self.zone_active_label_counts[zone] = Counter() + self.zone_all_attribute_counts[zone] = Counter() + self.zone_active_attribute_counts[zone] = Counter() self.all_zone_labels[zone] = set() self.all_zone_labels[zone].update( @@ -75,15 +91,21 @@ def update_activity(self, new_activity: dict[str, dict[str, Any]]) -> None: # run through every zone, getting a count of objects in that zone right now for zone, labels in self.all_zone_labels.items(): + zone_objects_by_id = { + obj["id"]: obj for obj in all_objects if zone in obj["current_zones"] + } all_zone_objects = Counter( obj["label"].replace("-verified", "") - for obj in all_objects - if zone in obj["current_zones"] + for obj in zone_objects_by_id.values() ) - active_zone_objects = Counter( - obj["label"].replace("-verified", "") + active_zone_objects_by_id = { + obj["id"]: obj for obj in all_objects if zone in obj["current_zones"] and not obj["stationary"] + } + active_zone_objects = Counter( + obj["label"].replace("-verified", "") + for obj in active_zone_objects_by_id.values() ) any_changed = False @@ -108,6 +130,86 @@ def update_activity(self, new_activity: dict[str, dict[str, Any]]) -> None: self.publish(f"{zone}/{label}/active", new_active_count) self.zone_active_object_counts[zone][label] = new_active_count + # Compute custom classification (sub_label) counts per object type + all_zone_label_counts = Counter( + (obj["object_type"], obj["sub_label"]) + for obj in zone_objects_by_id.values() + if obj.get("sub_label") + ) + active_zone_label_counts = Counter( + (obj["object_type"], obj["sub_label"]) + for obj in active_zone_objects_by_id.values() + if obj.get("sub_label") + ) + + # Publish sub-label (custom classification) counts + for key in set(all_zone_label_counts) | set( + self.zone_all_label_counts[zone] + ): + object_type, sub_label = key + new_count = all_zone_label_counts[key] + new_active = active_zone_label_counts[key] + + if ( + new_count != self.zone_all_label_counts[zone][key] + or key not in self.zone_all_label_counts[zone] + ): + any_changed = True + self.publish(f"{zone}/{object_type}/label/{sub_label}", new_count) + self.zone_all_label_counts[zone][key] = new_count + + if ( + new_active != self.zone_active_label_counts[zone][key] + or key not in self.zone_active_label_counts[zone] + ): + any_changed = True + self.publish( + f"{zone}/{object_type}/label/{sub_label}/active", new_active + ) + self.zone_active_label_counts[zone][key] = new_active + + # Compute attribute counts per object type + # Attributes are objects whose label differs from object_type and have no sub_label + all_zone_attribute_counts = Counter( + (obj["object_type"], obj["label"]) + for obj in zone_objects_by_id.values() + if not obj.get("sub_label") and obj["object_type"] != obj["label"] + ) + active_zone_attribute_counts = Counter( + (obj["object_type"], obj["label"]) + for obj in active_zone_objects_by_id.values() + if not obj.get("sub_label") and obj["object_type"] != obj["label"] + ) + + # Publish attribute counts + for key in set(all_zone_attribute_counts) | set( + self.zone_all_attribute_counts[zone] + ): + object_type, attribute = key + new_count = all_zone_attribute_counts[key] + new_active = active_zone_attribute_counts[key] + + if ( + new_count != self.zone_all_attribute_counts[zone][key] + or key not in self.zone_all_attribute_counts[zone] + ): + any_changed = True + self.publish( + f"{zone}/{object_type}/attribute/{attribute}", new_count + ) + self.zone_all_attribute_counts[zone][key] = new_count + + if ( + new_active != self.zone_active_attribute_counts[zone][key] + or key not in self.zone_active_attribute_counts[zone] + ): + any_changed = True + self.publish( + f"{zone}/{object_type}/attribute/{attribute}/active", + new_active, + ) + self.zone_active_attribute_counts[zone][key] = new_active + if any_changed: self.publish(f"{zone}/all", sum(list(all_zone_objects.values()))) self.publish( @@ -119,13 +221,16 @@ def update_activity(self, new_activity: dict[str, dict[str, Any]]) -> None: def compare_camera_activity( self, camera: str, new_activity: list[dict[str, Any]] ) -> None: + objects_by_id = {obj["id"]: obj for obj in new_activity} all_objects = Counter( - obj["label"].replace("-verified", "") for obj in new_activity + obj["label"].replace("-verified", "") for obj in objects_by_id.values() ) + active_objects_by_id = { + obj["id"]: obj for obj in new_activity if not obj["stationary"] + } active_objects = Counter( obj["label"].replace("-verified", "") - for obj in new_activity - if not obj["stationary"] + for obj in active_objects_by_id.values() ) any_changed = False @@ -157,6 +262,84 @@ def compare_camera_activity( self.publish(f"{camera}/{label}/active", new_active_count) self.camera_active_object_counts[camera][label] = new_active_count + # Compute custom classification (sub_label) counts per object type + all_camera_label_counts = Counter( + (obj["object_type"], obj["sub_label"]) + for obj in objects_by_id.values() + if obj.get("sub_label") + ) + active_camera_label_counts = Counter( + (obj["object_type"], obj["sub_label"]) + for obj in active_objects_by_id.values() + if obj.get("sub_label") + ) + + # Publish sub-label (custom classification) counts + for key in set(all_camera_label_counts) | set( + self.camera_all_label_counts[camera] + ): + object_type, sub_label = key + new_count = all_camera_label_counts[key] + new_active = active_camera_label_counts[key] + + if ( + new_count != self.camera_all_label_counts[camera][key] + or key not in self.camera_all_label_counts[camera] + ): + any_changed = True + self.publish(f"{camera}/{object_type}/label/{sub_label}", new_count) + self.camera_all_label_counts[camera][key] = new_count + + if ( + new_active != self.camera_active_label_counts[camera][key] + or key not in self.camera_active_label_counts[camera] + ): + any_changed = True + self.publish( + f"{camera}/{object_type}/label/{sub_label}/active", new_active + ) + self.camera_active_label_counts[camera][key] = new_active + + # Compute attribute counts per object type + # Attributes are objects whose label differs from object_type and have no sub_label + all_camera_attribute_counts = Counter( + (obj["object_type"], obj["label"]) + for obj in objects_by_id.values() + if not obj.get("sub_label") and obj["object_type"] != obj["label"] + ) + active_camera_attribute_counts = Counter( + (obj["object_type"], obj["label"]) + for obj in active_objects_by_id.values() + if not obj.get("sub_label") and obj["object_type"] != obj["label"] + ) + + # Publish attribute counts + for key in set(all_camera_attribute_counts) | set( + self.camera_all_attribute_counts[camera] + ): + object_type, attribute = key + new_count = all_camera_attribute_counts[key] + new_active = active_camera_attribute_counts[key] + + if ( + new_count != self.camera_all_attribute_counts[camera][key] + or key not in self.camera_all_attribute_counts[camera] + ): + any_changed = True + self.publish(f"{camera}/{object_type}/attribute/{attribute}", new_count) + self.camera_all_attribute_counts[camera][key] = new_count + + if ( + new_active != self.camera_active_attribute_counts[camera][key] + or key not in self.camera_active_attribute_counts[camera] + ): + any_changed = True + self.publish( + f"{camera}/{object_type}/attribute/{attribute}/active", + new_active, + ) + self.camera_active_attribute_counts[camera][key] = new_active + if any_changed: self.publish(f"{camera}/all", sum(list(all_objects.values()))) self.publish(f"{camera}/all/active", sum(list(active_objects.values()))) diff --git a/frigate/camera/state.py b/frigate/camera/state.py index f35a3eaa56d..f32af4f0142 100644 --- a/frigate/camera/state.py +++ b/frigate/camera/state.py @@ -518,6 +518,7 @@ def update( camera_activity["objects"].append( { "id": obj.obj_data["id"], + "object_type": object_type, "label": label, "stationary": not active, "area": obj.obj_data["area"], diff --git a/frigate/data_processing/real_time/custom_classification.py b/frigate/data_processing/real_time/custom_classification.py index e3b0e23ed81..b07d7a36830 100644 --- a/frigate/data_processing/real_time/custom_classification.py +++ b/frigate/data_processing/real_time/custom_classification.py @@ -573,14 +573,26 @@ def process_frame(self, obj_data: dict[str, Any], frame: np.ndarray) -> None: crop_bgr = cv2.cvtColor(crop, cv2.COLOR_RGB2BGR) self._enqueue_task( - ("classify", object_id, obj_data["camera"], now, resized_crop, crop_bgr) + ( + "classify", + object_id, + obj_data["camera"], + now, + resized_crop, + crop_bgr, + obj_data.get("current_zones", []), + ) ) def _process_task(self, task: Any) -> None: kind = task[0] if kind == "classify": - _, object_id, camera, timestamp, resized_crop, crop_bgr = task - self._classify_object(object_id, camera, timestamp, resized_crop, crop_bgr) + _, object_id, camera, timestamp, resized_crop, crop_bgr, current_zones = ( + task + ) + self._classify_object( + object_id, camera, timestamp, resized_crop, crop_bgr, current_zones + ) elif kind == "expire": _, object_id = task if object_id in self.classification_history: @@ -595,6 +607,7 @@ def _classify_object( timestamp: float, resized_crop: np.ndarray, crop_bgr: np.ndarray, + current_zones: list[str] | None = None, ) -> None: if self.interpreter is None: save_attempts = ( @@ -673,19 +686,20 @@ def _classify_object( ) if consensus_label is not None and self.model_config.object_config is not None: - self._emit_result( - { - "type": "classification", - "processor": "object", - "model_name": self.model_config.name, - "classification_type": self.model_config.object_config.classification_type, - "object_id": object_id, - "camera": camera, - "timestamp": timestamp, - "label": consensus_label, - "score": consensus_score, - } - ) + result: dict[str, Any] = { + "type": "classification", + "processor": "object", + "model_name": self.model_config.name, + "classification_type": self.model_config.object_config.classification_type, + "object_id": object_id, + "camera": camera, + "timestamp": timestamp, + "label": consensus_label, + "score": consensus_score, + } + if current_zones: + result["zones"] = current_zones + self._emit_result(result) def handle_request( self, topic: str, request_data: dict[str, Any] diff --git a/frigate/embeddings/maintainer.py b/frigate/embeddings/maintainer.py index 52bdf5d9152..818f68f5c56 100644 --- a/frigate/embeddings/maintainer.py +++ b/frigate/embeddings/maintainer.py @@ -756,44 +756,47 @@ def _process_deferred_results(self) -> None: label = result["label"] score = result["score"] classification_type = result["classification_type"] + zones = result.get("zones") if classification_type == ObjectClassificationType.sub_label: self.event_metadata_publisher.publish( (object_id, label, score), EventMetadataTypeEnum.sub_label, ) + payload: dict[str, Any] = { + "type": TrackedObjectUpdateTypesEnum.classification, + "id": object_id, + "camera": camera, + "timestamp": timestamp, + "model": model_name, + "sub_label": label, + "score": score, + } + if zones: + payload["zones"] = zones self.requestor.send_data( "tracked_object_update", - json.dumps( - { - "type": TrackedObjectUpdateTypesEnum.classification, - "id": object_id, - "camera": camera, - "timestamp": timestamp, - "model": model_name, - "sub_label": label, - "score": score, - } - ), + json.dumps(payload), ) elif classification_type == ObjectClassificationType.attribute: self.event_metadata_publisher.publish( (object_id, model_name, label, score), EventMetadataTypeEnum.attribute.value, ) + payload = { + "type": TrackedObjectUpdateTypesEnum.classification, + "id": object_id, + "camera": camera, + "timestamp": timestamp, + "model": model_name, + "attribute": label, + "score": score, + } + if zones: + payload["zones"] = zones self.requestor.send_data( "tracked_object_update", - json.dumps( - { - "type": TrackedObjectUpdateTypesEnum.classification, - "id": object_id, - "camera": camera, - "timestamp": timestamp, - "model": model_name, - "attribute": label, - "score": score, - } - ), + json.dumps(payload), ) def _embed_thumbnail(self, event_id: str, thumbnail: bytes) -> None: diff --git a/frigate/test/test_activity_manager.py b/frigate/test/test_activity_manager.py new file mode 100644 index 00000000000..9676306fb13 --- /dev/null +++ b/frigate/test/test_activity_manager.py @@ -0,0 +1,627 @@ +"""Tests for CameraActivityManager zone label and attribute MQTT publishing.""" + +import sys +import unittest +from unittest.mock import MagicMock + +# Save the original sys.modules entries so we can restore them after the import. +# Without this, the mocks would leak into every other test module that imports +# these packages after this file is discovered, corrupting their imports. +_MOCKED_MODS = [ + "zmq", + "frigate.comms.zmq_proxy", + "frigate.comms.event_metadata_updater", + "frigate.config", +] +_saved_modules = {mod: sys.modules.get(mod) for mod in _MOCKED_MODS} + +# Mock all modules that have native/missing dependencies before any imports +for _mod in _MOCKED_MODS: + sys.modules[_mod] = MagicMock() + +# Provide real-looking CameraConfig / FrigateConfig stubs so the type hints work +_config_mod = sys.modules["frigate.config"] +_config_mod.CameraConfig = MagicMock +_config_mod.FrigateConfig = MagicMock + +from frigate.camera.activity_manager import CameraActivityManager # noqa: E402 + +# Restore sys.modules immediately so subsequent test-module imports get the +# real packages, not these mocks. +for _mod, _original in _saved_modules.items(): + if _original is None: + sys.modules.pop(_mod, None) + else: + sys.modules[_mod] = _original +del _mod, _original, _saved_modules, _config_mod, _MOCKED_MODS + + +def _make_config(zone_name="driveway", zone_objects=None, track_objects=None): + """Build a minimal mock FrigateConfig with one camera and one zone.""" + if zone_objects is None: + zone_objects = ["person", "car"] + if track_objects is None: + track_objects = ["person", "car"] + + zone_config = MagicMock() + zone_config.objects = zone_objects + + camera_config = MagicMock() + camera_config.name = "front" + camera_config.enabled_in_config = True + camera_config.zones = {zone_name: zone_config} + camera_config.objects.track = track_objects + + config = MagicMock() + config.cameras = {"front": camera_config} + config.model.non_logo_attributes = ["face", "license_plate"] + + return config + + +def _make_object( + obj_id, + object_type, + label, + sub_label=None, + stationary=False, + current_zones=None, +): + """Build a minimal activity object dict matching the structure from state.py.""" + return { + "id": obj_id, + "object_type": object_type, + "label": label, + "stationary": stationary, + "area": 10000, + "ratio": 1.0, + "score": 0.9, + "sub_label": sub_label, + "current_zones": current_zones or [], + } + + +class TestZoneLabelPublishing(unittest.TestCase): + """Tests that custom classification sub-label counts are published to + {zone}/{object_type}/label/{sub_label} and {zone}/{object_type}/label/{sub_label}/active. + """ + + def setUp(self): + self.publish = MagicMock() + self.manager = CameraActivityManager(_make_config(), self.publish) + + def test_sub_label_total_published(self): + """Publish {zone}/{object_type}/label/{sub_label} for a classified object.""" + activity = { + "front": { + "motion": False, + "objects": [ + _make_object( + "obj1", + "person", + "person-verified", + sub_label="running", + stationary=True, + current_zones=["driveway"], + ) + ], + } + } + self.manager.update_activity(activity) + + self.publish.assert_any_call("driveway/person/label/running", 1) + + def test_sub_label_active_published(self): + """Publish {zone}/{object_type}/label/{sub_label}/active for active classified object.""" + activity = { + "front": { + "motion": False, + "objects": [ + _make_object( + "obj1", + "person", + "person-verified", + sub_label="running", + stationary=False, + current_zones=["driveway"], + ) + ], + } + } + self.manager.update_activity(activity) + + self.publish.assert_any_call("driveway/person/label/running", 1) + self.publish.assert_any_call("driveway/person/label/running/active", 1) + + def test_sub_label_active_zero_when_stationary(self): + """Active count is 0 when classified object is stationary.""" + activity = { + "front": { + "motion": False, + "objects": [ + _make_object( + "obj1", + "person", + "person-verified", + sub_label="walking", + stationary=True, + current_zones=["driveway"], + ) + ], + } + } + self.manager.update_activity(activity) + + self.publish.assert_any_call("driveway/person/label/walking", 1) + self.publish.assert_any_call("driveway/person/label/walking/active", 0) + + def test_sub_label_count_published_zero_when_object_leaves(self): + """Count drops to 0 and is published when a classified object leaves the zone.""" + activity_with = { + "front": { + "motion": False, + "objects": [ + _make_object( + "obj1", + "person", + "person-verified", + sub_label="running", + stationary=False, + current_zones=["driveway"], + ) + ], + } + } + activity_without = {"front": {"motion": False, "objects": []}} + + self.manager.update_activity(activity_with) + self.publish.reset_mock() + self.manager.update_activity(activity_without) + + self.publish.assert_any_call("driveway/person/label/running", 0) + self.publish.assert_any_call("driveway/person/label/running/active", 0) + + def test_multiple_sub_labels_counted_separately(self): + """Different sub_labels under the same object_type are counted separately.""" + activity = { + "front": { + "motion": False, + "objects": [ + _make_object( + "obj1", + "person", + "person-verified", + sub_label="running", + stationary=False, + current_zones=["driveway"], + ), + _make_object( + "obj2", + "person", + "person-verified", + sub_label="sitting", + stationary=True, + current_zones=["driveway"], + ), + ], + } + } + self.manager.update_activity(activity) + + self.publish.assert_any_call("driveway/person/label/running", 1) + self.publish.assert_any_call("driveway/person/label/running/active", 1) + self.publish.assert_any_call("driveway/person/label/sitting", 1) + self.publish.assert_any_call("driveway/person/label/sitting/active", 0) + + def test_sub_label_not_published_for_object_outside_zone(self): + """Sub-label counts are not published for objects outside the zone.""" + activity = { + "front": { + "motion": False, + "objects": [ + _make_object( + "obj1", + "person", + "person-verified", + sub_label="running", + stationary=False, + current_zones=[], # not in any zone + ) + ], + } + } + self.manager.update_activity(activity) + + published_topics = [c.args[0] for c in self.publish.call_args_list] + self.assertNotIn("driveway/person/label/running", published_topics) + self.assertNotIn("driveway/person/label/running/active", published_topics) + + +class TestZoneAttributePublishing(unittest.TestCase): + """Tests that attribute counts are published to + {zone}/{object_type}/attribute/{attribute} and + {zone}/{object_type}/attribute/{attribute}/active. + """ + + def setUp(self): + self.publish = MagicMock() + self.manager = CameraActivityManager(_make_config(), self.publish) + + def test_attribute_total_published(self): + """Publish {zone}/{object_type}/attribute/{attribute} for an attribute object.""" + # Attributes: label IS the attribute (e.g. "amazon"), object_type is the parent ("car") + activity = { + "front": { + "motion": False, + "objects": [ + _make_object( + "obj1", + "car", + "amazon", + sub_label=None, + stationary=True, + current_zones=["driveway"], + ) + ], + } + } + self.manager.update_activity(activity) + + self.publish.assert_any_call("driveway/car/attribute/amazon", 1) + + def test_attribute_active_published(self): + """Publish {zone}/{object_type}/attribute/{attribute}/active for active attribute object.""" + activity = { + "front": { + "motion": False, + "objects": [ + _make_object( + "obj1", + "car", + "amazon", + sub_label=None, + stationary=False, + current_zones=["driveway"], + ) + ], + } + } + self.manager.update_activity(activity) + + self.publish.assert_any_call("driveway/car/attribute/amazon", 1) + self.publish.assert_any_call("driveway/car/attribute/amazon/active", 1) + + def test_attribute_active_zero_when_stationary(self): + """Active count is 0 when attribute object is stationary.""" + activity = { + "front": { + "motion": False, + "objects": [ + _make_object( + "obj1", + "car", + "license_plate", + sub_label=None, + stationary=True, + current_zones=["driveway"], + ) + ], + } + } + self.manager.update_activity(activity) + + self.publish.assert_any_call("driveway/car/attribute/license_plate", 1) + self.publish.assert_any_call("driveway/car/attribute/license_plate/active", 0) + + def test_attribute_count_drops_to_zero_when_object_leaves(self): + """Count drops to 0 and is published when an attribute object leaves the zone.""" + activity_with = { + "front": { + "motion": False, + "objects": [ + _make_object( + "obj1", + "car", + "amazon", + sub_label=None, + stationary=False, + current_zones=["driveway"], + ) + ], + } + } + activity_without = {"front": {"motion": False, "objects": []}} + + self.manager.update_activity(activity_with) + self.publish.reset_mock() + self.manager.update_activity(activity_without) + + self.publish.assert_any_call("driveway/car/attribute/amazon", 0) + self.publish.assert_any_call("driveway/car/attribute/amazon/active", 0) + + def test_base_object_not_treated_as_attribute(self): + """A plain object without sub_label whose label == object_type is NOT published as attribute.""" + activity = { + "front": { + "motion": False, + "objects": [ + _make_object( + "obj1", + "person", + "person", + sub_label=None, + stationary=False, + current_zones=["driveway"], + ) + ], + } + } + self.manager.update_activity(activity) + + published_topics = [c.args[0] for c in self.publish.call_args_list] + attribute_topics = [t for t in published_topics if "/attribute/" in t] + self.assertEqual([], attribute_topics) + + +class TestCameraLabelPublishing(unittest.TestCase): + """Tests that custom classification sub-label counts are published to + {camera}/{object_type}/label/{sub_label} and {camera}/{object_type}/label/{sub_label}/active. + """ + + def setUp(self): + self.publish = MagicMock() + self.manager = CameraActivityManager(_make_config(), self.publish) + + def test_sub_label_total_published_for_camera(self): + """Publish {camera}/{object_type}/label/{sub_label} for a classified object.""" + activity = { + "front": { + "motion": False, + "objects": [ + _make_object( + "obj1", + "person", + "person-verified", + sub_label="running", + stationary=True, + current_zones=[], + ) + ], + } + } + self.manager.update_activity(activity) + + self.publish.assert_any_call("front/person/label/running", 1) + + def test_sub_label_active_published_for_camera(self): + """Publish {camera}/{object_type}/label/{sub_label}/active for active classified object.""" + activity = { + "front": { + "motion": False, + "objects": [ + _make_object( + "obj1", + "person", + "person-verified", + sub_label="running", + stationary=False, + current_zones=[], + ) + ], + } + } + self.manager.update_activity(activity) + + self.publish.assert_any_call("front/person/label/running", 1) + self.publish.assert_any_call("front/person/label/running/active", 1) + + def test_sub_label_active_zero_when_stationary_for_camera(self): + """Active count is 0 when classified object is stationary on a camera.""" + activity = { + "front": { + "motion": False, + "objects": [ + _make_object( + "obj1", + "person", + "person-verified", + sub_label="walking", + stationary=True, + current_zones=[], + ) + ], + } + } + self.manager.update_activity(activity) + + self.publish.assert_any_call("front/person/label/walking", 1) + self.publish.assert_any_call("front/person/label/walking/active", 0) + + def test_sub_label_count_drops_to_zero_for_camera(self): + """Count drops to 0 and is published when a classified object leaves the camera.""" + activity_with = { + "front": { + "motion": False, + "objects": [ + _make_object( + "obj1", + "person", + "person-verified", + sub_label="running", + stationary=False, + current_zones=[], + ) + ], + } + } + activity_without = {"front": {"motion": False, "objects": []}} + + self.manager.update_activity(activity_with) + self.publish.reset_mock() + self.manager.update_activity(activity_without) + + self.publish.assert_any_call("front/person/label/running", 0) + self.publish.assert_any_call("front/person/label/running/active", 0) + + +class TestCameraAttributePublishing(unittest.TestCase): + """Tests that attribute counts are published to + {camera}/{object_type}/attribute/{attribute} and + {camera}/{object_type}/attribute/{attribute}/active. + """ + + def setUp(self): + self.publish = MagicMock() + self.manager = CameraActivityManager(_make_config(), self.publish) + + def test_attribute_total_published_for_camera(self): + """Publish {camera}/{object_type}/attribute/{attribute} for an attribute object.""" + activity = { + "front": { + "motion": False, + "objects": [ + _make_object( + "obj1", + "car", + "amazon", + sub_label=None, + stationary=True, + current_zones=[], + ) + ], + } + } + self.manager.update_activity(activity) + + self.publish.assert_any_call("front/car/attribute/amazon", 1) + + def test_attribute_active_published_for_camera(self): + """Publish {camera}/{object_type}/attribute/{attribute}/active for active attribute object.""" + activity = { + "front": { + "motion": False, + "objects": [ + _make_object( + "obj1", + "car", + "amazon", + sub_label=None, + stationary=False, + current_zones=[], + ) + ], + } + } + self.manager.update_activity(activity) + + self.publish.assert_any_call("front/car/attribute/amazon", 1) + self.publish.assert_any_call("front/car/attribute/amazon/active", 1) + + def test_attribute_active_zero_when_stationary_for_camera(self): + """Active count is 0 when attribute object is stationary on a camera.""" + activity = { + "front": { + "motion": False, + "objects": [ + _make_object( + "obj1", + "car", + "license_plate", + sub_label=None, + stationary=True, + current_zones=[], + ) + ], + } + } + self.manager.update_activity(activity) + + self.publish.assert_any_call("front/car/attribute/license_plate", 1) + self.publish.assert_any_call("front/car/attribute/license_plate/active", 0) + + def test_attribute_count_drops_to_zero_for_camera(self): + """Count drops to 0 and is published when an attribute object leaves the camera.""" + activity_with = { + "front": { + "motion": False, + "objects": [ + _make_object( + "obj1", + "car", + "amazon", + sub_label=None, + stationary=False, + current_zones=[], + ) + ], + } + } + activity_without = {"front": {"motion": False, "objects": []}} + + self.manager.update_activity(activity_with) + self.publish.reset_mock() + self.manager.update_activity(activity_without) + + self.publish.assert_any_call("front/car/attribute/amazon", 0) + self.publish.assert_any_call("front/car/attribute/amazon/active", 0) + + +class TestNoRepublishUnchanged(unittest.TestCase): + """Tests that counts are only re-published when they actually change.""" + + def setUp(self): + self.publish = MagicMock() + self.manager = CameraActivityManager(_make_config(), self.publish) + + def test_no_republish_when_sub_label_count_unchanged(self): + """Sub-label topics are not re-published when the counts haven't changed.""" + activity = { + "front": { + "motion": False, + "objects": [ + _make_object( + "obj1", + "person", + "person-verified", + sub_label="running", + stationary=False, + current_zones=["driveway"], + ) + ], + } + } + self.manager.update_activity(activity) + self.publish.reset_mock() + + # Same activity again — counts are identical, nothing should be re-published + self.manager.update_activity(activity) + + published_topics = [c.args[0] for c in self.publish.call_args_list] + self.assertNotIn("driveway/person/label/running", published_topics) + self.assertNotIn("driveway/person/label/running/active", published_topics) + + def test_no_republish_when_attribute_count_unchanged(self): + """Attribute topics are not re-published when the counts haven't changed.""" + activity = { + "front": { + "motion": False, + "objects": [ + _make_object( + "obj1", + "car", + "amazon", + sub_label=None, + stationary=False, + current_zones=["driveway"], + ) + ], + } + } + self.manager.update_activity(activity) + self.publish.reset_mock() + + self.manager.update_activity(activity) + + published_topics = [c.args[0] for c in self.publish.call_args_list] + self.assertNotIn("driveway/car/attribute/amazon", published_topics) + self.assertNotIn("driveway/car/attribute/amazon/active", published_topics) diff --git a/frigate/test/test_custom_classification.py b/frigate/test/test_custom_classification.py new file mode 100644 index 00000000000..407e96a48be --- /dev/null +++ b/frigate/test/test_custom_classification.py @@ -0,0 +1,361 @@ +import json +import sys +import time +import unittest +from unittest.mock import MagicMock, patch + +# Mock native/optional modules before any Frigate imports so the module can be +# imported in environments where cv2 or TFLite are unavailable (e.g. CI). +_SYS_MOCKS = [ + "cv2", + "tflite_runtime", + "tflite_runtime.interpreter", + "ai_edge_litert", + "ai_edge_litert.interpreter", +] +for _mod in _SYS_MOCKS: + if _mod not in sys.modules: + sys.modules[_mod] = MagicMock() + +WIDTH = 720 +HEIGHT = 1280 + + +class Contains: + def __init__(self, needle): + self.needle = needle + + def __eq__(self, other): + return self.needle in other + + +class TestCustomObjectClassificationZones(unittest.TestCase): + """Test that zone information is correctly added to custom classification MQTT messages""" + + def _build_classification_data( + self, obj_data, classification_type="sub_label", label="person_walking" + ): + """Helper method to build classification data with conditional zones. + + Args: + obj_data: Object data dictionary containing id, camera, and optionally current_zones + classification_type: Either "sub_label" or "attribute" + label: The classification label + + Returns: + Dictionary with classification data, including zones if applicable + """ + classification_data = { + "type": "classification", + "id": obj_data["id"], + "camera": obj_data["camera"], + "timestamp": 1234567890.0, + "model": "test_classifier", + "score": 0.89, + } + + if classification_type == "sub_label": + classification_data["sub_label"] = label + else: + classification_data["attribute"] = label + + if obj_data.get("current_zones"): + classification_data["zones"] = obj_data["current_zones"] + + return classification_data + + def test_sub_label_message_includes_zones_when_present(self): + """Test that zones are included in sub_label classification messages when object is in zones""" + # Create a simple mock requestor + requestor = MagicMock() + + # Create mock obj_data with zones + obj_data = { + "id": "test_object_123", + "camera": "front_door", + "current_zones": ["driveway", "front_yard"], + } + + # Build classification data using helper + classification_data = self._build_classification_data( + obj_data, "sub_label", "person_walking" + ) + + requestor.send_data("tracked_object_update", json.dumps(classification_data)) + + # Verify that send_data was called + requestor.send_data.assert_called_once() + + # Get the actual call arguments + call_args = requestor.send_data.call_args + topic = call_args[0][0] + data_json = call_args[0][1] + + # Verify the topic + self.assertEqual(topic, "tracked_object_update") + + # Parse and verify the data + data = json.loads(data_json) + self.assertEqual(data["type"], "classification") + self.assertEqual(data["id"], "test_object_123") + self.assertEqual(data["camera"], "front_door") + self.assertEqual(data["model"], "test_classifier") + self.assertEqual(data["sub_label"], "person_walking") + self.assertIn("zones", data) + self.assertEqual(data["zones"], ["driveway", "front_yard"]) + + def test_sub_label_message_excludes_zones_when_empty(self): + """Test that zones are not included when object is not in any zones""" + requestor = MagicMock() + + # Create mock obj_data without zones + obj_data = { + "id": "test_object_456", + "camera": "back_door", + "current_zones": [], + } + + # Build classification data using helper + classification_data = self._build_classification_data( + obj_data, "sub_label", "person_running" + ) + classification_data["score"] = 0.87 + + requestor.send_data("tracked_object_update", json.dumps(classification_data)) + + # Get the actual call arguments + call_args = requestor.send_data.call_args + data_json = call_args[0][1] + + # Parse and verify the data + data = json.loads(data_json) + self.assertNotIn("zones", data) + + def test_attribute_message_includes_zones_when_present(self): + """Test that zones are included in attribute classification messages when object is in zones""" + requestor = MagicMock() + + # Create mock obj_data with zones + obj_data = { + "id": "test_object_789", + "camera": "construction_site", + "current_zones": ["site_entrance"], + } + + # Build classification data using helper + classification_data = self._build_classification_data( + obj_data, "attribute", "wearing_helmet" + ) + classification_data["score"] = 0.92 + classification_data["model"] = "helmet_detector" + + requestor.send_data("tracked_object_update", json.dumps(classification_data)) + + # Get the actual call arguments + call_args = requestor.send_data.call_args + data_json = call_args[0][1] + + # Parse and verify the data + data = json.loads(data_json) + self.assertEqual(data["type"], "classification") + self.assertEqual(data["id"], "test_object_789") + self.assertEqual(data["camera"], "construction_site") + self.assertEqual(data["model"], "helmet_detector") + self.assertEqual(data["attribute"], "wearing_helmet") + self.assertIn("zones", data) + self.assertEqual(data["zones"], ["site_entrance"]) + + def test_attribute_message_excludes_zones_when_missing(self): + """Test that zones are not included when current_zones key is missing""" + requestor = MagicMock() + + # Create mock obj_data without current_zones key + obj_data = { + "id": "test_object_999", + "camera": "parking_lot", + } + + # Build classification data using helper + classification_data = self._build_classification_data( + obj_data, "attribute", "sedan" + ) + classification_data["score"] = 0.95 + classification_data["model"] = "vehicle_type" + + requestor.send_data("tracked_object_update", json.dumps(classification_data)) + + # Get the actual call arguments + call_args = requestor.send_data.call_args + data_json = call_args[0][1] + + # Parse and verify the data + data = json.loads(data_json) + self.assertNotIn("zones", data) + + +class TestCustomObjectClassificationIntegration(unittest.TestCase): + """ + Integration tests that call process_frame() on the actual processor. + These tests exercise the full pipeline from process_frame() through the + deferred worker thread to drain_results(), verifying that zone information + is carried through to the result dicts that the maintainer publishes. + """ + + def setUp(self): + import numpy as np + + self.np = np + + # Import the module first so patch() can resolve its attributes. + try: + import frigate.data_processing.real_time.custom_classification # noqa: F401 + from frigate.data_processing.real_time.custom_classification import ( + CustomObjectClassificationProcessor, + ) + except ImportError as e: + self.skipTest(f"Requires full Frigate environment: {e}") + return + + self.ProcessorClass = CustomObjectClassificationProcessor + + # Patch out heavy I/O helpers on the already-imported module object. + for target in [ + "frigate.data_processing.real_time.custom_classification.write_classification_attempt", + "frigate.data_processing.real_time.custom_classification.suppress_stderr_during", + ]: + patcher = patch(target) + patcher.start() + self.addCleanup(patcher.stop) + + def _make_processor(self, classification_type): + """Return a processor with a live interpreter stub and pre-loaded history.""" + config = MagicMock() + model_config = MagicMock() + model_config.name = "test_model" + model_config.threshold = 0.7 + model_config.save_attempts = 100 + model_config.object_config.objects = ["person"] + model_config.object_config.classification_type = classification_type + + sub_label_publisher = MagicMock() + requestor = MagicMock() + metrics = MagicMock() + metrics.classification_speeds = {} + metrics.classification_cps = {} + + with patch.object( + self.ProcessorClass, + "_CustomObjectClassificationProcessor__build_detector", + ): + processor = self.ProcessorClass( + config, model_config, sub_label_publisher, requestor, metrics + ) + + return processor + + def _run_and_drain(self, processor, obj_data, label, score=0.92): + """ + Run process_frame() with a stubbed interpreter and return drain_results(). + Pre-loads 3 identical history entries so consensus is reached immediately. + """ + import numpy as np + + # Pre-load history so get_weighted_score returns consensus on the first call. + processor.classification_history[obj_data["id"]] = [ + (label, score, 1234567890.0), + (label, score, 1234567891.0), + (label, score, 1234567892.0), + ] + + processor.tensor_input_details = [{"index": 0}] + processor.tensor_output_details = [{"index": 0}] + processor.labelmap = {0: label} + + mock_interp = MagicMock() + mock_interp.get_tensor.return_value = np.array([[score, 1.0 - score]]) + processor.interpreter = mock_interp + + frame = np.zeros((WIDTH * 3 // 2, HEIGHT), dtype=np.uint8) + processor.process_frame(obj_data, frame) + + # Give the worker thread time to process the enqueued task. + time.sleep(0.2) + + return processor.drain_results() + + def test_process_frame_with_zones_includes_zones_in_mqtt(self): + """process_frame() with non-empty current_zones must emit a result with zones.""" + from frigate.config.classification import ObjectClassificationType + + processor = self._make_processor(ObjectClassificationType.sub_label) + + obj_data = { + "id": "test_123", + "camera": "front_door", + "label": "person", + "false_positive": False, + "end_time": None, + "box": [100, 100, 200, 200], + "current_zones": ["driveway", "porch"], + } + + results = self._run_and_drain(processor, obj_data, "walking") + + self.assertTrue(results, "process_frame must produce at least one result") + result = results[0] + self.assertEqual(result["type"], "classification") + self.assertIn( + "zones", result, "Result must include zones when object is in zones" + ) + self.assertEqual(result["zones"], ["driveway", "porch"]) + self.assertEqual(result["label"], "walking") + + def test_process_frame_without_zones_excludes_zones_from_mqtt(self): + """process_frame() with empty current_zones must emit a result without zones.""" + from frigate.config.classification import ObjectClassificationType + + processor = self._make_processor(ObjectClassificationType.sub_label) + + obj_data = { + "id": "test_456", + "camera": "backyard", + "label": "person", + "false_positive": False, + "end_time": None, + "box": [150, 150, 250, 250], + "current_zones": [], + } + + results = self._run_and_drain(processor, obj_data, "running") + + self.assertTrue(results, "process_frame must produce at least one result") + result = results[0] + self.assertNotIn("zones", result, "Empty zones should be excluded from result") + + def test_process_frame_attribute_type_includes_zones(self): + """process_frame() with attribute classification type must include zones.""" + from frigate.config.classification import ObjectClassificationType + + processor = self._make_processor(ObjectClassificationType.attribute) + + obj_data = { + "id": "test_789", + "camera": "garage", + "label": "person", + "false_positive": False, + "end_time": None, + "box": [200, 200, 300, 300], + "current_zones": ["parking_lot"], + } + + results = self._run_and_drain(processor, obj_data, "hat") + + self.assertTrue(results, "process_frame must produce at least one result") + result = results[0] + self.assertIn("zones", result, "Result must include zones for attribute type") + self.assertEqual(result["zones"], ["parking_lot"]) + self.assertEqual(result["label"], "hat") + + +if __name__ == "__main__": + unittest.main() diff --git a/frigate/track/tracked_object.py b/frigate/track/tracked_object.py index 03117df6924..d7d3dd0417b 100644 --- a/frigate/track/tracked_object.py +++ b/frigate/track/tracked_object.py @@ -35,7 +35,7 @@ # (ex: car loitering on the street vs when a known person parks on the street) # person is the main object that should keep alerts going as long as they loiter # even if they are stationary. -EXTENDED_LOITERING_OBJECTS = ["person"] +EXTENDED_LOITERING_OBJECTS = ["person", "waste_bin", "package"] class TrackedObject: diff --git a/web/package-lock.json b/web/package-lock.json index cd09b79811e..f9a9733b6df 100644 --- a/web/package-lock.json +++ b/web/package-lock.json @@ -71,7 +71,7 @@ "react-icons": "^5.5.0", "react-konva": "^19.2.3", "react-markdown": "^9.0.1", - "react-router-dom": "^6.30.3", + "react-router-dom": "^6.30.4", "react-swipeable": "^7.0.2", "react-zoom-pan-pinch": "^3.7.0", "remark-gfm": "^4.0.0", @@ -4878,9 +4878,9 @@ } }, "node_modules/@remix-run/router": { - "version": "1.23.2", - "resolved": "https://registry.npmjs.org/@remix-run/router/-/router-1.23.2.tgz", - "integrity": "sha512-Ic6m2U/rMjTkhERIa/0ZtXJP17QUi2CbWE7cqx4J58M8aA3QTfW+2UlQ4psvTX9IO1RfNVhK3pcpdjej7L+t2w==", + "version": "1.23.3", + "resolved": "https://registry.npmjs.org/@remix-run/router/-/router-1.23.3.tgz", + "integrity": "sha512-4An71tdz9X8+3sI4Qqqd2LWd9vS39J7sqd9EU4Scw7TJE/qB10Flv/UuqbPVgfQV9XoK8Np6jNquZitnZq5i+Q==", "license": "MIT", "engines": { "node": ">=14.0.0" @@ -12282,12 +12282,12 @@ } }, "node_modules/react-router": { - "version": "6.30.3", - "resolved": "https://registry.npmjs.org/react-router/-/react-router-6.30.3.tgz", - "integrity": "sha512-XRnlbKMTmktBkjCLE8/XcZFlnHvr2Ltdr1eJX4idL55/9BbORzyZEaIkBFDhFGCEWBBItsVrDxwx3gnisMitdw==", + "version": "6.30.4", + "resolved": "https://registry.npmjs.org/react-router/-/react-router-6.30.4.tgz", + "integrity": "sha512-SVUsDe+DybHM/WmYKIVYhZh1o5Dcuf16yM6WjG02Q9XVFMZIJyHYhwrr6bFBXZkVP6z69kNkMyBCujt8FaFLJA==", "license": "MIT", "dependencies": { - "@remix-run/router": "1.23.2" + "@remix-run/router": "1.23.3" }, "engines": { "node": ">=14.0.0" @@ -12297,13 +12297,13 @@ } }, "node_modules/react-router-dom": { - "version": "6.30.3", - "resolved": "https://registry.npmjs.org/react-router-dom/-/react-router-dom-6.30.3.tgz", - "integrity": "sha512-pxPcv1AczD4vso7G4Z3TKcvlxK7g7TNt3/FNGMhfqyntocvYKj+GCatfigGDjbLozC4baguJ0ReCigoDJXb0ag==", + "version": "6.30.4", + "resolved": "https://registry.npmjs.org/react-router-dom/-/react-router-dom-6.30.4.tgz", + "integrity": "sha512-q4HvNl+mmDdkS0g+MqiBZNteQJCuimWoOyHMy4T/RQLAn9Z29+E91QXRaxOujeMl2HTzRSS0KFPd7lxX3PjV0Q==", "license": "MIT", "dependencies": { - "@remix-run/router": "1.23.2", - "react-router": "6.30.3" + "@remix-run/router": "1.23.3", + "react-router": "6.30.4" }, "engines": { "node": ">=14.0.0" diff --git a/web/package.json b/web/package.json index 7d22dc7183a..56cb1e0e215 100644 --- a/web/package.json +++ b/web/package.json @@ -85,7 +85,7 @@ "react-icons": "^5.5.0", "react-konva": "^19.2.3", "react-markdown": "^9.0.1", - "react-router-dom": "^6.30.3", + "react-router-dom": "^6.30.4", "react-swipeable": "^7.0.2", "react-zoom-pan-pinch": "^3.7.0", "remark-gfm": "^4.0.0", diff --git a/web/public/locales/en/views/classificationModel.json b/web/public/locales/en/views/classificationModel.json index 3206ad0339a..f0504c3ed8c 100644 --- a/web/public/locales/en/views/classificationModel.json +++ b/web/public/locales/en/views/classificationModel.json @@ -13,7 +13,12 @@ "trainModel": "Train Model", "addClassification": "Add Classification", "deleteModels": "Delete Models", - "editModel": "Edit Model" + "editModel": "Edit Model", + "categorizeImages": "Classify Images", + "enableSelection": "Enable Selection", + "disableSelection": "Disable Selection", + "selectImage": "Select Image", + "selectGroup": "Select Group" }, "tooltip": { "trainingInProgress": "Model is currently training", @@ -26,6 +31,7 @@ "deletedModel_one": "Successfully deleted {{count}} model", "deletedModel_other": "Successfully deleted {{count}} models", "categorizedImage": "Successfully Classified Image", + "batchCategorized": "Successfully classified {{count}} images", "reclassifiedImage": "Successfully Reclassified Image", "trainedModel": "Successfully trained model.", "trainingModel": "Successfully started model training.", @@ -34,18 +40,26 @@ "deletedCategory_one": "Deleted {{count}} class", "deletedCategory_other": "Deleted {{count}} classes", "deletedImage_one": "Deleted {{count}} image", - "deletedImage_other": "Deleted {{count}} images" + "deletedImage_other": "Deleted {{count}} images", + "batchCategorized_one": "Successfully classified {{count}} image", + "batchCategorized_other": "Successfully classified {{count}} images" }, "error": { "deleteImageFailed": "Failed to delete: {{errorMessage}}", "deleteCategoryFailed": "Failed to delete class: {{errorMessage}}", "deleteModelFailed": "Failed to delete model: {{errorMessage}}", "categorizeFailed": "Failed to categorize image: {{errorMessage}}", + "batchCategorizeFailed": "Failed to classify {{count}} images", "trainingFailed": "Model training failed. Check Frigate logs for details.", "trainingFailedToStart": "Failed to start model training: {{errorMessage}}", "updateModelFailed": "Failed to update model: {{errorMessage}}", "renameCategoryFailed": "Failed to rename class: {{errorMessage}}", - "reclassifyFailed": "Failed to reclassify image: {{errorMessage}}" + "reclassifyFailed": "Failed to reclassify image: {{errorMessage}}", + "batchCategorizeFailed_one": "Failed to classify {{count}} image", + "batchCategorizeFailed_other": "Failed to classify {{count}} images" + }, + "warning": { + "partialBatchCategorized": "Classified {{success}} of {{total}} images successfully." } }, "deleteCategory": { diff --git a/web/public/locales/en/views/explore.json b/web/public/locales/en/views/explore.json index d1087b3c96a..d1fc857794b 100644 --- a/web/public/locales/en/views/explore.json +++ b/web/public/locales/en/views/explore.json @@ -146,6 +146,15 @@ }, "recognizedLicensePlate": "Recognized License Plate", "attributes": "Classification Attributes", + "assignment": { + "title": "Assign To", + "assignToFace": "Assign to Face", + "assignToClassification": "Assign to {{model}}", + "faceSuccess": "Successfully assigned to face: {{name}}", + "faceFailed": "Failed to assign to face: {{errorMessage}}", + "classificationSuccess": "Successfully assigned to {{model}} - {{category}}", + "classificationFailed": "Failed to assign classification: {{errorMessage}}" + }, "estimatedSpeed": "Estimated Speed", "objects": "Objects", "camera": "Camera", @@ -222,7 +231,7 @@ "label": "Hide object path" }, "debugReplay": { - "label": "Debug Replay", + "label": "Debug replay", "aria": "View this tracked object in the debug replay view" }, "more": { diff --git a/web/public/locales/en/views/faceLibrary.json b/web/public/locales/en/views/faceLibrary.json index 27e54546016..d385dc2bfc3 100644 --- a/web/public/locales/en/views/faceLibrary.json +++ b/web/public/locales/en/views/faceLibrary.json @@ -32,11 +32,7 @@ "title": "Recent Recognitions", "titleShort": "Recent", "aria": "Select recent recognitions", - "empty": "There are no recent face recognition attempts", - "emptyNoLibrary": { - "title": "Upload a face", - "description": "You must add at least one face to the library for face recognition to function." - } + "empty": "There are no recent face recognition attempts" }, "deleteFaceLibrary": { "title": "Delete Name", @@ -57,7 +53,12 @@ "renameFace": "Rename Face", "deleteFace": "Delete Face", "uploadImage": "Upload Image", - "reprocessFace": "Reprocess Face" + "reprocessFace": "Reprocess Face", + "trainFaces": "Train Faces", + "enableSelection": "Enable Selection", + "disableSelection": "Disable Selection", + "selectImage": "Select Image", + "selectGroup": "Select Group" }, "imageEntry": { "validation": { @@ -83,8 +84,11 @@ "deletedName_other": "{{count}} faces have been successfully deleted.", "renamedFace": "Successfully renamed face to {{name}}", "trainedFace": "Successfully trained face.", + "batchTrainedFaces": "Successfully trained {{count}} faces.", "reclassifiedFace": "Successfully reclassified face.", - "updatedFaceScore": "Successfully updated face score to {{name}} ({{score}})." + "updatedFaceScore": "Successfully updated face score to {{name}} ({{score}}).", + "batchTrainedFaces_one": "Successfully trained {{count}} face.", + "batchTrainedFaces_other": "Successfully trained {{count}} faces." }, "error": { "uploadingImageFailed": "Failed to upload image: {{errorMessage}}", @@ -93,8 +97,14 @@ "deleteNameFailed": "Failed to delete name: {{errorMessage}}", "renameFaceFailed": "Failed to rename face: {{errorMessage}}", "trainFailed": "Failed to train: {{errorMessage}}", + "batchTrainFailed": "Failed to train {{count}} faces.", "reclassifyFailed": "Failed to reclassify face: {{errorMessage}}", - "updateFaceScoreFailed": "Failed to update face score: {{errorMessage}}" + "updateFaceScoreFailed": "Failed to update face score: {{errorMessage}}", + "batchTrainFailed_one": "Failed to train {{count}} face.", + "batchTrainFailed_other": "Failed to train {{count}} faces." + }, + "warning": { + "partialBatchTrained": "Trained {{success}} of {{total}} faces successfully." } } } diff --git a/web/src/components/card/ClassificationCard.tsx b/web/src/components/card/ClassificationCard.tsx index 4fed7e23857..7b4660eb5c3 100644 --- a/web/src/components/card/ClassificationCard.tsx +++ b/web/src/components/card/ClassificationCard.tsx @@ -44,6 +44,7 @@ type ClassificationCardProps = { i18nLibrary: string; showArea?: boolean; count?: number; + topLeftContent?: React.ReactNode; onClick: (data: ClassificationItemData, meta: boolean) => void; children?: React.ReactNode; }; @@ -61,6 +62,7 @@ export const ClassificationCard = forwardRef< i18nLibrary, showArea = true, count, + topLeftContent, onClick, children, }, @@ -143,6 +145,15 @@ export const ClassificationCard = forwardRef< onLoad={() => setImageLoaded(true)} src={`${baseUrl}${data.filepath}`} /> + {topLeftContent && ( +
e.stopPropagation()} + onMouseDown={(e) => e.stopPropagation()} + > + {topLeftContent} +
+ )} {count && (
@@ -199,6 +210,7 @@ type GroupedClassificationCardProps = { i18nLibrary: string; objectType: string; noClassificationLabel?: string; + topLeftContent?: React.ReactNode; onClick: (data: ClassificationItemData | undefined) => void; children?: (data: ClassificationItemData) => React.ReactNode; }; @@ -209,6 +221,7 @@ export function GroupedClassificationCard({ selectedItems, i18nLibrary, noClassificationLabel = "details.none", + topLeftContent, onClick, children, }: GroupedClassificationCardProps) { @@ -314,6 +327,7 @@ export function GroupedClassificationCard({ clickable={true} i18nLibrary={i18nLibrary} count={group.length} + topLeftContent={topLeftContent} onClick={(_, meta) => { if (meta || selectedItems.length > 0) { onClick(undefined); diff --git a/web/src/components/overlay/ClassificationSelectionDialog.tsx b/web/src/components/overlay/ClassificationSelectionDialog.tsx index 470c75ec7fa..7a74966efb0 100644 --- a/web/src/components/overlay/ClassificationSelectionDialog.tsx +++ b/web/src/components/overlay/ClassificationSelectionDialog.tsx @@ -33,9 +33,10 @@ type ClassificationSelectionDialogProps = { className?: string; classes: string[]; modelName: string; - image: string; + image?: string; + images?: string[]; onRefresh?: () => void; - onCategorize?: (category: string) => void; + onCategorize?: (category: string, images?: string[]) => void; excludeCategory?: string; dialogLabel?: string; tooltipLabel?: string; @@ -46,6 +47,7 @@ export default function ClassificationSelectionDialog({ classes, modelName, image, + images, onRefresh, onCategorize, excludeCategory, @@ -57,35 +59,105 @@ export default function ClassificationSelectionDialog({ const onCategorizeImage = useCallback( (category: string) => { + const targetImages = + images?.length && image && !images.includes(image) + ? [image, ...images] + : images?.length + ? images + : image + ? [image] + : []; + + // If custom categorize handler is provided, use it instead if (onCategorize) { - onCategorize(category); + onCategorize(category, targetImages); + return; + } + + if (targetImages.length === 0) { + toast.error(t("toast.error.batchCategorizeFailed", { count: 0 }), { + position: "top-center", + }); return; } - axios - .post(`/classification/${modelName}/dataset/categorize`, { - category, - training_file: image, - }) - .then((resp) => { - if (resp.status == 200) { - toast.success(t("toast.success.categorizedImage"), { + if (targetImages.length === 1) { + // Default behavior: categorize a single image. + axios + .post(`/classification/${modelName}/dataset/categorize`, { + category, + training_file: targetImages[0], + }) + .then((resp) => { + if (resp.status == 200) { + toast.success(t("toast.success.categorizedImage"), { + position: "top-center", + }); + onRefresh?.(); + } + }) + .catch((error) => { + const errorMessage = + error.response?.data?.message || + error.response?.data?.detail || + "Unknown error"; + toast.error(t("toast.error.categorizeFailed", { errorMessage }), { position: "top-center", }); - onRefresh?.(); - } - }) - .catch((error) => { - const errorMessage = - error.response?.data?.message || - error.response?.data?.detail || - "Unknown error"; - toast.error(t("toast.error.categorizeFailed", { errorMessage }), { - position: "top-center", }); - }); + return; + } + + const requests = targetImages.map((filename) => + axios + .post(`/classification/${modelName}/dataset/categorize`, { + category, + training_file: filename, + }) + .then(() => true) + .catch(() => false), + ); + + Promise.allSettled(requests).then((results) => { + const successCount = results.filter( + (result) => result.status === "fulfilled" && result.value, + ).length; + const totalCount = results.length; + + if (successCount === totalCount) { + toast.success( + t("toast.success.batchCategorized", { + count: successCount, + }), + { + position: "top-center", + }, + ); + } else if (successCount > 0) { + toast.warning( + t("toast.warning.partialBatchCategorized", { + success: successCount, + total: totalCount, + }), + { + position: "top-center", + }, + ); + } else { + toast.error( + t("toast.error.batchCategorizeFailed", { + count: totalCount, + }), + { + position: "top-center", + }, + ); + } + + onRefresh?.(); + }); }, - [modelName, image, onRefresh, onCategorize, t], + [modelName, image, images, onRefresh, onCategorize, t], ); const filteredClasses = useMemo( @@ -102,19 +174,6 @@ export default function ClassificationSelectionDialog({ // control const [newClass, setNewClass] = useState(false); - // Non-modal Radix DropdownMenu doesn't propagate wheel events to nested - // scroll containers, so attach a non-passive listener that scrolls manually. - const scrollContainerRef = useCallback((el: HTMLDivElement | null) => { - if (!el || !isDesktop) return; - const handleWheel = (e: WheelEvent) => { - if (el.scrollHeight <= el.clientHeight) return; - e.preventDefault(); - el.scrollTop += e.deltaY; - }; - el.addEventListener("wheel", handleWheel, { passive: false }); - return () => el.removeEventListener("wheel", handleWheel); - }, []); - // components const Selector = isDesktop ? DropdownMenu : Drawer; const SelectorTrigger = isDesktop ? DropdownMenuTrigger : DrawerTrigger; @@ -127,70 +186,68 @@ export default function ClassificationSelectionDialog({ ); - // keep modal false on desktop to prevent dismissable layer pointer events - // issue with dialog auto-close return ( -
+
onCategorizeImage(newCat)} /> - - - - {children} - - - {tooltipLabel ?? t("categorizeImage")} - - - e.preventDefault()} - > - {isMobile && ( - - Details - Details - - )} - - {dialogLabel ?? t("categorizeImageAs")} - -
- {filteredClasses - .sort((a, b) => { - if (a === "none") return 1; - if (b === "none") return -1; - return a.localeCompare(b); - }) - .map((category) => ( - onCategorizeImage(category)} - > - {category === "none" - ? t("details.none") - : category.replaceAll("_", " ")} - - ))} - - setNewClass(true)} + + + + + {children} + + e.preventDefault()} + > + {isMobile && ( + + Details + Details + + )} + + {dialogLabel ?? t("categorizeImageAs")} + +
- {t("createCategory.new")} - -
- - + {filteredClasses + .sort((a, b) => { + if (a === "none") return 1; + if (b === "none") return -1; + return a.localeCompare(b); + }) + .map((category) => ( + onCategorizeImage(category)} + > + {category === "none" + ? t("details.none") + : category.replaceAll("_", " ")} + + ))} + + setNewClass(true)} + > + {t("createCategory.new")} + +
+ + + {tooltipLabel ?? t("categorizeImage")} +
); } diff --git a/web/src/components/overlay/detail/SearchDetailDialog.tsx b/web/src/components/overlay/detail/SearchDetailDialog.tsx index e8e4368ea42..3ff008f7658 100644 --- a/web/src/components/overlay/detail/SearchDetailDialog.tsx +++ b/web/src/components/overlay/detail/SearchDetailDialog.tsx @@ -94,6 +94,11 @@ import { useDetailStream } from "@/context/detail-stream-context"; import { PiSlidersHorizontalBold } from "react-icons/pi"; import { HiSparkles } from "react-icons/hi"; import { useAudioTranscriptionProcessState } from "@/api/ws"; +import FaceSelectionDialog from "@/components/overlay/FaceSelectionDialog"; +import ClassificationSelectionDialog from "@/components/overlay/ClassificationSelectionDialog"; +import { FaceLibraryData } from "@/types/face"; +import AddFaceIcon from "@/components/icons/AddFaceIcon"; +import { TbCategoryPlus } from "react-icons/tb"; const SEARCH_TABS = ["snapshot", "tracking_details"] as const; export type SearchTab = (typeof SEARCH_TABS)[number]; @@ -241,7 +246,7 @@ function AnnotationSettings({ return (
- +
+ {isAdmin && (availableFaceNames.length > 0 || availableClassificationModels.length > 0) && ( +
+
+ {t("details.assignment.title")} +
+
+ {config?.face_recognition?.enabled && availableFaceNames.length > 0 && ( + + + + )} + {availableClassificationModels.length > 0 && + availableClassificationModels.map((modelName) => { + const model = config?.classification?.custom?.[modelName]; + if (!model) return null; + + const displayName = model.name || modelName; + const classes = modelAttributes?.[displayName] ?? []; + if (classes.length === 0) return null; + + return ( + {}} + onCategorize={(category) => + onAssignToClassification(modelName, category) + } + > + + + ); + })} +
+
+ )} + {isAdmin && search.data.type === "object" && config?.plus?.enabled && @@ -1581,7 +1721,7 @@ function ObjectDetailsTab({ {t("button.yes", { ns: "common" })}