diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index c4d8aa7a035..e7800547955 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -10,6 +10,8 @@ on: env: DEFAULT_PYTHON: 3.11 + REGISTRY: ghcr.io + DEV_IMAGE_NAME: ${{ github.repository }}-dev jobs: web_lint: @@ -78,18 +80,18 @@ jobs: uses: actions/checkout@v6 with: persist-credentials: false - - uses: actions/setup-node@v6 + - name: Login to GitHub Container Registry + uses: docker/login-action@v2 with: - node-version: 20.x - - name: Install devcontainer cli - run: npm install --global @devcontainers/cli - - name: Build devcontainer - env: - DOCKER_BUILDKIT: "1" - run: devcontainer build --workspace-folder . - - name: Start devcontainer - run: devcontainer up --workspace-folder . - - name: Run mypy in devcontainer - run: devcontainer exec --workspace-folder . bash -lc "python3 -u -m mypy --config-file frigate/mypy.ini frigate" - - name: Run unit tests in devcontainer - run: devcontainer exec --workspace-folder . bash -lc "python3 -u -m unittest" + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Build devcontainer image and run checks + uses: devcontainers/ci@v0.3 + with: + imageName: ${{ env.REGISTRY }}/${{ env.DEV_IMAGE_NAME }} + cacheFrom: ${{ env.REGISTRY }}/${{ env.DEV_IMAGE_NAME }} + push: always + runCmd: |- + python3 -u -m mypy --config-file frigate/mypy.ini frigate && + python3 -u -m unittest diff --git a/docs/docs/integrations/mqtt.md b/docs/docs/integrations/mqtt.md index 535e1bb4b09..05b0ccecd33 100644 --- a/docs/docs/integrations/mqtt.md +++ b/docs/docs/integrations/mqtt.md @@ -173,7 +173,8 @@ Message published when [object classification](/configuration/custom_classificat "timestamp": 1607123958.748393, "model": "person_classifier", "sub_label": "delivery_person", - "score": 0.87 + "score": 0.87, + "zones": ["front_yard", "driveway"] } ``` @@ -187,10 +188,15 @@ Message published when [object classification](/configuration/custom_classificat "timestamp": 1607123958.748393, "model": "helmet_detector", "attribute": "yes", - "score": 0.92 + "score": 0.92, + "zones": ["front_yard"] } ``` +:::note +The `zones` field is only included if the tracked object is currently in one or more zones. +::: + ### `frigate/reviews` Message published for each changed review item. The first message is published when the `detection` or `alert` is initiated. diff --git a/docs/package-lock.json b/docs/package-lock.json index be16754be3d..98a8664c243 100644 --- a/docs/package-lock.json +++ b/docs/package-lock.json @@ -10423,13 +10423,13 @@ } }, "node_modules/enhanced-resolve": { - "version": "5.18.3", - "resolved": "https://registry.npmjs.org/enhanced-resolve/-/enhanced-resolve-5.18.3.tgz", - "integrity": "sha512-d4lC8xfavMeBjzGr2vECC3fsGXziXZQyJxD868h2M/mBI3PwAuODxAkLkq5HYuvrPYcUtiLzsTo8U3PgX3Ocww==", + "version": "5.19.0", + "resolved": "https://registry.npmjs.org/enhanced-resolve/-/enhanced-resolve-5.19.0.tgz", + "integrity": "sha512-phv3E1Xl4tQOShqSte26C7Fl84EwUdZsyOuSSk9qtAGyyQs2s3jJzComh+Abf4g187lUUAvH+H26omrqia2aGg==", "license": "MIT", "dependencies": { "graceful-fs": "^4.2.4", - "tapable": "^2.2.0" + "tapable": "^2.3.0" }, "engines": { "node": ">=10.13.0" @@ -10475,9 +10475,9 @@ } }, "node_modules/es-module-lexer": { - "version": "1.7.0", - "resolved": "https://registry.npmjs.org/es-module-lexer/-/es-module-lexer-1.7.0.tgz", - "integrity": "sha512-jEQoCwk8hyb2AZziIOLhDqpm5+2ww5uIE6lkO/6jcOCusfk6LhMHpXXfBLXTZ7Ydyt0j4VoUQv6uGNYbdW+kBA==", + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/es-module-lexer/-/es-module-lexer-2.0.0.tgz", + "integrity": "sha512-5POEcUuZybH7IdmGsD8wlf0AI55wMecM9rVBTI/qEAy2c1kTOm3DjFYjrBdI2K3BaJjJYfYFeRtM0t9ssnRuxw==", "license": "MIT" }, "node_modules/es-object-atoms": { @@ -21624,9 +21624,9 @@ } }, "node_modules/terser-webpack-plugin": { - "version": "5.3.15", - "resolved": "https://registry.npmjs.org/terser-webpack-plugin/-/terser-webpack-plugin-5.3.15.tgz", - "integrity": "sha512-PGkOdpRFK+rb1TzVz+msVhw4YMRT9txLF4kRqvJhGhCM324xuR3REBSHALN+l+sAhKUmz0aotnjp5D+P83mLhQ==", + "version": "5.3.16", + "resolved": "https://registry.npmjs.org/terser-webpack-plugin/-/terser-webpack-plugin-5.3.16.tgz", + "integrity": "sha512-h9oBFCWrq78NyWWVcSwZarJkZ01c2AyGrzs1crmHZO3QUg9D61Wu4NPjBy69n7JqylFF5y+CsUZYmYEIZ3mR+Q==", "license": "MIT", "dependencies": { "@jridgewell/trace-mapping": "^0.3.25", @@ -22613,9 +22613,9 @@ } }, "node_modules/watchpack": { - "version": "2.4.4", - "resolved": "https://registry.npmjs.org/watchpack/-/watchpack-2.4.4.tgz", - "integrity": "sha512-c5EGNOiyxxV5qmTtAB7rbiXxi1ooX1pQKMLX/MIabJjRA0SJBQOjKF+KSVfHkr9U1cADPon0mRiVe/riyaiDUA==", + "version": "2.5.1", + "resolved": "https://registry.npmjs.org/watchpack/-/watchpack-2.5.1.tgz", + "integrity": "sha512-Zn5uXdcFNIA1+1Ei5McRd+iRzfhENPCe7LeABkJtNulSxjma+l7ltNx55BWZkRlwRnpOgHqxnjyaDgJnNXnqzg==", "license": "MIT", "dependencies": { "glob-to-regexp": "^0.4.1", @@ -22651,9 +22651,9 @@ "license": "BSD-2-Clause" }, "node_modules/webpack": { - "version": "5.103.0", - "resolved": "https://registry.npmjs.org/webpack/-/webpack-5.103.0.tgz", - "integrity": "sha512-HU1JOuV1OavsZ+mfigY0j8d1TgQgbZ6M+J75zDkpEAwYeXjWSqrGJtgnPblJjd/mAyTNQ7ygw0MiKOn6etz8yw==", + "version": "5.105.0", + "resolved": "https://registry.npmjs.org/webpack/-/webpack-5.105.0.tgz", + "integrity": "sha512-gX/dMkRQc7QOMzgTe6KsYFM7DxeIONQSui1s0n/0xht36HvrgbxtM1xBlgx596NbpHuQU8P7QpKwrZYwUX48nw==", "license": "MIT", "dependencies": { "@types/eslint-scope": "^3.7.7", @@ -22664,10 +22664,10 @@ "@webassemblyjs/wasm-parser": "^1.14.1", "acorn": "^8.15.0", "acorn-import-phases": "^1.0.3", - "browserslist": "^4.26.3", + "browserslist": "^4.28.1", "chrome-trace-event": "^1.0.2", - "enhanced-resolve": "^5.17.3", - "es-module-lexer": "^1.2.1", + "enhanced-resolve": "^5.19.0", + "es-module-lexer": "^2.0.0", "eslint-scope": "5.1.1", "events": "^3.2.0", "glob-to-regexp": "^0.4.1", @@ -22678,8 +22678,8 @@ "neo-async": "^2.6.2", "schema-utils": "^4.3.3", "tapable": "^2.3.0", - "terser-webpack-plugin": "^5.3.11", - "watchpack": "^2.4.4", + "terser-webpack-plugin": "^5.3.16", + "watchpack": "^2.5.1", "webpack-sources": "^3.3.3" }, "bin": { diff --git a/frigate/data_processing/real_time/custom_classification.py b/frigate/data_processing/real_time/custom_classification.py index 19a40fd99d4..c68efe2120d 100644 --- a/frigate/data_processing/real_time/custom_classification.py +++ b/frigate/data_processing/real_time/custom_classification.py @@ -613,19 +613,20 @@ def process_frame(self, obj_data, frame): (object_id, consensus_label, consensus_score), EventMetadataTypeEnum.sub_label, ) + classification_data = { + "type": TrackedObjectUpdateTypesEnum.classification, + "id": object_id, + "camera": camera, + "timestamp": now, + "model": self.model_config.name, + "sub_label": consensus_label, + "score": consensus_score, + } + if obj_data.get("current_zones"): + classification_data["zones"] = obj_data["current_zones"] self.requestor.send_data( "tracked_object_update", - json.dumps( - { - "type": TrackedObjectUpdateTypesEnum.classification, - "id": object_id, - "camera": camera, - "timestamp": now, - "model": self.model_config.name, - "sub_label": consensus_label, - "score": consensus_score, - } - ), + json.dumps(classification_data), ) elif ( self.model_config.object_config.classification_type @@ -640,19 +641,20 @@ def process_frame(self, obj_data, frame): ), EventMetadataTypeEnum.attribute.value, ) + classification_data = { + "type": TrackedObjectUpdateTypesEnum.classification, + "id": object_id, + "camera": camera, + "timestamp": now, + "model": self.model_config.name, + "attribute": consensus_label, + "score": consensus_score, + } + if obj_data.get("current_zones"): + classification_data["zones"] = obj_data["current_zones"] self.requestor.send_data( "tracked_object_update", - json.dumps( - { - "type": TrackedObjectUpdateTypesEnum.classification, - "id": object_id, - "camera": camera, - "timestamp": now, - "model": self.model_config.name, - "attribute": consensus_label, - "score": consensus_score, - } - ), + json.dumps(classification_data), ) def handle_request(self, topic, request_data): diff --git a/frigate/test/test_custom_classification.py b/frigate/test/test_custom_classification.py new file mode 100644 index 00000000000..965896f70ba --- /dev/null +++ b/frigate/test/test_custom_classification.py @@ -0,0 +1,449 @@ +import json +import unittest +from unittest.mock import MagicMock, patch + +MOCK_MODULES = [ + "frigate.data_processing.real_time.custom_classification.cv2", + "frigate.data_processing.real_time.custom_classification.load_labels", + "frigate.data_processing.real_time.custom_classification.write_classification_attempt", + "frigate.data_processing.real_time.custom_classification.suppress_stderr_during", +] +MOCK_CLASSES = [ + "frigate.data_processing.real_time.custom_classification.Interpreter", + "frigate.data_processing.real_time.custom_classification.InferenceSpeed", + "frigate.data_processing.real_time.custom_classification.InterProcessRequestor", + "frigate.data_processing.real_time.custom_classification.EventMetadataPublisher", +] + +WIDTH = 720 +HEIGHT = 1280 + + +class Contains: + def __init__(self, needle): + self.needle = needle + + def __eq__(self, other): + return self.needle in other + + +class TestCustomObjectClassificationZones(unittest.TestCase): + """Test that zone information is correctly added to custom classification MQTT messages""" + + def _build_classification_data( + self, obj_data, classification_type="sub_label", label="person_walking" + ): + """Helper method to build classification data with conditional zones. + + Args: + obj_data: Object data dictionary containing id, camera, and optionally current_zones + classification_type: Either "sub_label" or "attribute" + label: The classification label + + Returns: + Dictionary with classification data, including zones if applicable + """ + classification_data = { + "type": "classification", + "id": obj_data["id"], + "camera": obj_data["camera"], + "timestamp": 1234567890.0, + "model": "test_classifier", + "score": 0.89, + } + + if classification_type == "sub_label": + classification_data["sub_label"] = label + else: + classification_data["attribute"] = label + + if obj_data.get("current_zones"): + classification_data["zones"] = obj_data["current_zones"] + + return classification_data + + def test_sub_label_message_includes_zones_when_present(self): + """Test that zones are included in sub_label classification messages when object is in zones""" + # Create a simple mock requestor + requestor = MagicMock() + + # Create mock obj_data with zones + obj_data = { + "id": "test_object_123", + "camera": "front_door", + "current_zones": ["driveway", "front_yard"], + } + + # Build classification data using helper + classification_data = self._build_classification_data( + obj_data, "sub_label", "person_walking" + ) + + requestor.send_data("tracked_object_update", json.dumps(classification_data)) + + # Verify that send_data was called + requestor.send_data.assert_called_once() + + # Get the actual call arguments + call_args = requestor.send_data.call_args + topic = call_args[0][0] + data_json = call_args[0][1] + + # Verify the topic + self.assertEqual(topic, "tracked_object_update") + + # Parse and verify the data + data = json.loads(data_json) + self.assertEqual(data["type"], "classification") + self.assertEqual(data["id"], "test_object_123") + self.assertEqual(data["camera"], "front_door") + self.assertEqual(data["model"], "test_classifier") + self.assertEqual(data["sub_label"], "person_walking") + self.assertIn("zones", data) + self.assertEqual(data["zones"], ["driveway", "front_yard"]) + + def test_sub_label_message_excludes_zones_when_empty(self): + """Test that zones are not included when object is not in any zones""" + requestor = MagicMock() + + # Create mock obj_data without zones + obj_data = { + "id": "test_object_456", + "camera": "back_door", + "current_zones": [], + } + + # Build classification data using helper + classification_data = self._build_classification_data( + obj_data, "sub_label", "person_running" + ) + classification_data["score"] = 0.87 + + requestor.send_data("tracked_object_update", json.dumps(classification_data)) + + # Get the actual call arguments + call_args = requestor.send_data.call_args + data_json = call_args[0][1] + + # Parse and verify the data + data = json.loads(data_json) + self.assertNotIn("zones", data) + + def test_attribute_message_includes_zones_when_present(self): + """Test that zones are included in attribute classification messages when object is in zones""" + requestor = MagicMock() + + # Create mock obj_data with zones + obj_data = { + "id": "test_object_789", + "camera": "construction_site", + "current_zones": ["site_entrance"], + } + + # Build classification data using helper + classification_data = self._build_classification_data( + obj_data, "attribute", "wearing_helmet" + ) + classification_data["score"] = 0.92 + classification_data["model"] = "helmet_detector" + + requestor.send_data("tracked_object_update", json.dumps(classification_data)) + + # Get the actual call arguments + call_args = requestor.send_data.call_args + data_json = call_args[0][1] + + # Parse and verify the data + data = json.loads(data_json) + self.assertEqual(data["type"], "classification") + self.assertEqual(data["id"], "test_object_789") + self.assertEqual(data["camera"], "construction_site") + self.assertEqual(data["model"], "helmet_detector") + self.assertEqual(data["attribute"], "wearing_helmet") + self.assertIn("zones", data) + self.assertEqual(data["zones"], ["site_entrance"]) + + def test_attribute_message_excludes_zones_when_missing(self): + """Test that zones are not included when current_zones key is missing""" + requestor = MagicMock() + + # Create mock obj_data without current_zones key + obj_data = { + "id": "test_object_999", + "camera": "parking_lot", + } + + # Build classification data using helper + classification_data = self._build_classification_data( + obj_data, "attribute", "sedan" + ) + classification_data["score"] = 0.95 + classification_data["model"] = "vehicle_type" + + requestor.send_data("tracked_object_update", json.dumps(classification_data)) + + # Get the actual call arguments + call_args = requestor.send_data.call_args + data_json = call_args[0][1] + + # Parse and verify the data + data = json.loads(data_json) + self.assertNotIn("zones", data) + + +class TestCustomObjectClassificationIntegration(unittest.TestCase): + """ + TRUE Integration tests that call process_frame() on the actual processor. + These tests exercise the full call stack from process_frame to MQTT output. + + NOTE: These integration tests require the full Frigate Docker environment with + all dependencies (pydantic, psutil, PIL, etc). They demonstrate the proper + integration test pattern but may not run in minimal test environments. + + In the Docker test environment, these tests: + 1. Instantiate the real CustomObjectClassificationProcessor + 2. Call the actual process_frame() method + 3. Verify the full call stack produces correct MQTT messages with zones + """ + + def setUp(self): + """Import the processor after mocking dependencies""" + + def cvtColor(frame, color): + return self.np.zeros((WIDTH, HEIGHT, 3), dtype=self.np.uint8) + + def resize(frame, size): + return self.np.zeros((*size[0:1], 3), dtype=self.np.uint8) + + self.patchers = {} + for mod in MOCK_MODULES: + patcher = patch(mod).start() + self.patchers[mod] = patcher + self.addCleanup(patcher.stop) + + for mod in MOCK_CLASSES: + patcher = patch(mod).start() + self.patchers[mod] = patcher.start() + self.addCleanup(patcher.stop) + patcher.return_value = MagicMock() + + mock_cv2 = self.patchers[ + "frigate.data_processing.real_time.custom_classification.cv2" + ] + + import numpy as np + + self.np = np + mock_cv2.cvtColor.side_effect = cvtColor + mock_cv2.resize.side_effect = resize + + try: + from frigate.data_processing.real_time.custom_classification import ( + CustomObjectClassificationProcessor, + ) + + self.ProcessorClass = CustomObjectClassificationProcessor + except ImportError as e: + # If imports fail, skip these tests (they need full Docker environment) + self.skipTest(f"Requires full Frigate environment: {e}") + + def test_process_frame_with_zones_includes_zones_in_mqtt(self): + """ + Integration test: Actually call process_frame() and verify zones in MQTT. + This tests the FULL call stack. + """ + # Create processor + config = MagicMock() + model_config = MagicMock() + model_config.name = "test_model" + model_config.threshold = 0.7 + model_config.save_attempts = 100 + model_config.object_config.objects = ["person"] + + # Mock classification type with proper comparison support + from frigate.config.classification import ObjectClassificationType + + model_config.object_config.classification_type = ( + ObjectClassificationType.sub_label + ) + + sub_label_publisher = MagicMock() + requestor = MagicMock() + metrics = MagicMock() + + # Instantiate the REAL processor + processor = self.ProcessorClass( + config, model_config, sub_label_publisher, requestor, metrics + ) + + # Prepare obj_data WITH zones + obj_data = { + "id": "test_123", + "camera": "front_door", + "label": "person", + "false_positive": False, + "end_time": None, + "box": [100, 100, 200, 200], + "current_zones": ["driveway", "porch"], # THE KEY FIELD + } + + # Set up for consensus + processor.classification_history[obj_data["id"]] = [ + ("walking", 0.85, 1234567890.0), + ("walking", 0.87, 1234567891.0), + ("walking", 0.89, 1234567892.0), + ] + + # Create frame + frame = self.np.zeros((WIDTH, HEIGHT, 3), dtype=self.np.uint8) + + # Mock TFLite + processor.interpreter = MagicMock() + processor.tensor_input_details = [{"index": 0}] + processor.tensor_output_details = [{"index": 0}] + processor.labelmap = {0: "walking"} + processor.interpreter.get_tensor.return_value = self.np.array([[0.92, 0.08]]) + + # CALL THE ACTUAL METHOD - This exercises the full call stack + processor.process_frame(obj_data, frame) + + # Verify the call stack resulted in MQTT message + self.assertTrue( + requestor.send_data.called, "process_frame must call requestor.send_data" + ) + + # Extract and verify the MQTT message + mqtt_json = requestor.send_data.call_args[0][1] + mqtt_data = json.loads(mqtt_json) + + # THE ACTUAL VERIFICATION: zones from obj_data made it through the stack + self.assertIn("zones", mqtt_data, "MQTT must include zones") + self.assertEqual(mqtt_data["zones"], ["driveway", "porch"]) + self.assertEqual(mqtt_data["sub_label"], "walking") + + def test_process_frame_without_zones_excludes_zones_from_mqtt(self): + """ + Integration test: Call process_frame() with empty zones and verify exclusion. + """ + config = MagicMock() + model_config = MagicMock() + model_config.name = "test_model" + model_config.threshold = 0.7 + model_config.save_attempts = 100 + model_config.object_config.objects = ["person"] + + from frigate.config.classification import ObjectClassificationType + + model_config.object_config.classification_type = ( + ObjectClassificationType.sub_label + ) + + sub_label_publisher = MagicMock() + requestor = MagicMock() + metrics = MagicMock() + + processor = self.ProcessorClass( + config, model_config, sub_label_publisher, requestor, metrics + ) + + # obj_data WITHOUT zones + obj_data = { + "id": "test_456", + "camera": "backyard", + "label": "person", + "false_positive": False, + "end_time": None, + "box": [150, 150, 250, 250], + "current_zones": [], # EMPTY + } + + processor.classification_history[obj_data["id"]] = [ + ("running", 0.85, 1234567890.0), + ("running", 0.87, 1234567891.0), + ("running", 0.89, 1234567892.0), + ] + + frame = self.np.zeros((720, 1280, 3), dtype=self.np.uint8) + + processor.interpreter = MagicMock() + processor.tensor_input_details = [{"index": 0}] + processor.tensor_output_details = [{"index": 0}] + processor.labelmap = {0: "running"} + processor.interpreter.get_tensor.return_value = self.np.array([[0.90, 0.10]]) + + # CALL THE ACTUAL METHOD + processor.process_frame(obj_data, frame) + + # Verify MQTT + self.assertTrue(requestor.send_data.called) + mqtt_json = requestor.send_data.call_args[0][1] + mqtt_data = json.loads(mqtt_json) + + # Verify zones NOT included + self.assertNotIn("zones", mqtt_data, "Empty zones should be excluded") + + def test_process_frame_attribute_type_includes_zones(self): + """ + Integration test: Call process_frame() for attribute type with zones. + """ + config = MagicMock() + model_config = MagicMock() + model_config.name = "test_model" + model_config.threshold = 0.7 + model_config.save_attempts = 100 + model_config.object_config.objects = ["person"] + + from frigate.config.classification import ObjectClassificationType + + model_config.object_config.classification_type = ( + ObjectClassificationType.attribute + ) + + sub_label_publisher = MagicMock() + requestor = MagicMock() + metrics = MagicMock() + + processor = self.ProcessorClass( + config, model_config, sub_label_publisher, requestor, metrics + ) + + obj_data = { + "id": "test_789", + "camera": "garage", + "label": "person", + "false_positive": False, + "end_time": None, + "box": [200, 200, 300, 300], + "current_zones": ["parking_lot"], + } + + processor.classification_history[obj_data["id"]] = [ + ("hat", 0.88, 1234567890.0), + ("hat", 0.90, 1234567891.0), + ("hat", 0.92, 1234567892.0), + ] + + frame = self.np.zeros((720, 1280, 3), dtype=self.np.uint8) + + processor.interpreter = MagicMock() + processor.tensor_input_details = [{"index": 0}] + processor.tensor_output_details = [{"index": 0}] + processor.labelmap = {0: "hat"} + processor.interpreter.get_tensor.return_value = self.np.array([[0.93, 0.07]]) + + # CALL THE ACTUAL METHOD + processor.process_frame(obj_data, frame) + + # Verify MQTT + self.assertTrue(requestor.send_data.called) + mqtt_json = requestor.send_data.call_args[0][1] + mqtt_data = json.loads(mqtt_json) + + # Verify zones included for attribute type + self.assertIn("zones", mqtt_data) + self.assertEqual(mqtt_data["zones"], ["parking_lot"]) + self.assertEqual(mqtt_data["attribute"], "hat") + + +if __name__ == "__main__": + unittest.main()