From 763b7a388633cd0a5d8fd627f6a39093ab3d9a54 Mon Sep 17 00:00:00 2001 From: "Pavel A. Tomskikh" Date: Thu, 16 Jul 2026 14:19:41 +0700 Subject: [PATCH 1/2] Swift cam scene change demo --- samples/scene_change/README.md | 58 ++++++++++ samples/scene_change/docker-compose.l4t.yml | 76 +++++++++++++ samples/scene_change/docker-compose.x86.yml | 46 ++++++++ samples/scene_change/module.yml | 79 ++++++++++++++ samples/scene_change/overlay.py | 50 +++++++++ samples/scene_change/roi_injector.py | 62 +++++++++++ samples/scene_change/scene_change.py | 114 ++++++++++++++++++++ samples/scene_change/set-roi.sh | 19 ++++ 8 files changed, 504 insertions(+) create mode 100644 samples/scene_change/README.md create mode 100644 samples/scene_change/docker-compose.l4t.yml create mode 100644 samples/scene_change/docker-compose.x86.yml create mode 100644 samples/scene_change/module.yml create mode 100644 samples/scene_change/overlay.py create mode 100644 samples/scene_change/roi_injector.py create mode 100644 samples/scene_change/scene_change.py create mode 100755 samples/scene_change/set-roi.sh diff --git a/samples/scene_change/README.md b/samples/scene_change/README.md new file mode 100644 index 000000000..b1fb63415 --- /dev/null +++ b/samples/scene_change/README.md @@ -0,0 +1,58 @@ +# Scene change + +The pipeline computes a ReID embedding vector for a region of interest (ROI) on the frame and compares it with a vector of a frame N seconds ago to detect a scene change (e.g., when a camera moved). The ROI is configurable per-source in Etcd. + +Tested on platforms: + +- Nvidia Ampere + +## Prerequisites + +```bash +git clone https://github.com/insight-platform/Savant.git +cd Savant +git lfs pull +./utils/check-environment-compatible +``` + +**Note**: Ubuntu 22.04 runtime configuration [guide](https://insight-platform.github.io/Savant/develop/getting_started/0_configure_prod_env.html) helps to configure the runtime to run Savant pipelines. + +## Build Engines + +The demo uses models that are compiled into TensorRT engines the first time the demo is run. This takes time. Optionally, you can prepare the engines before running the demo by using the command: + +```bash +# you are expected to be in Savant/ directory + +./scripts/run_module.py --build-engines samples/scene_change/module.yml +``` + +## Run Demo + +```bash +# you are expected to be in Savant/ directory + +# if x86 +docker compose -f samples/scene_change/docker-compose.x86.yml up + +# if Jetson +docker compose -f samples/scene_change/docker-compose.l4t.yml up + +# open 'rtsp://127.0.0.1:554/stream/video' in your player +# or visit 'http://127.0.0.1:888/stream/video/' (LL-HLS) + +# Ctrl+C to stop running the compose bundle +``` + +## ROI configuration + +By default, the pipeline uses ROI from the [module configuration](module.yml). By changing the value of the key `savant/roi/{source_id}` in Etcd you can change ROI of the corresponding source. + +To change a source ROI it is convenient to use the script: + +```bash +# you are expected to be in Savant/ directory +./samples/scene_change/set-roi.sh video "540,100,1000,400" +# to reset to the default ROI +./samples/scene_change/set-roi.sh video +``` diff --git a/samples/scene_change/docker-compose.l4t.yml b/samples/scene_change/docker-compose.l4t.yml new file mode 100644 index 000000000..212ef143f --- /dev/null +++ b/samples/scene_change/docker-compose.l4t.yml @@ -0,0 +1,76 @@ +services: + + video-loop-source: + image: ghcr.io/insight-platform/savant-adapters-gstreamer-l4t:latest + restart: unless-stopped + volumes: + - zmq_sockets:/tmp/zmq-sockets + - /tmp/video-loop-source-downloads:/tmp/video-loop-source-downloads + environment: + - LOCATION=https://eu-central-1.linodeobjects.com/savant-data/demo/scene_change_cam_rotation_yaw.mp4 + - DOWNLOAD_PATH=/tmp/video-loop-source-downloads + - ZMQ_ENDPOINT=pub+connect:ipc:///tmp/zmq-sockets/input-video.ipc + - SOURCE_ID=video + - SYNC_OUTPUT=True + entrypoint: /opt/savant/adapters/gst/sources/video_loop.sh + depends_on: + module: + condition: service_healthy + + module: + privileged: true + image: ghcr.io/insight-platform/savant-deepstream-l4t:latest + restart: unless-stopped + volumes: + - zmq_sockets:/tmp/zmq-sockets + - ../../cache:/cache + - ..:/opt/savant/samples + command: samples/scene_change/module.yml + environment: + - MODEL_PATH=/cache/models/scene_change + - DOWNLOAD_PATH=/cache/downloads/scene_change + - ZMQ_SRC_ENDPOINT=sub+bind:ipc:///tmp/zmq-sockets/input-video.ipc + - ZMQ_SINK_ENDPOINT=pub+bind:ipc:///tmp/zmq-sockets/output-video.ipc + - METRICS_FRAME_PERIOD=1000 + - CODEC=jpeg + depends_on: + etcd: + condition: service_healthy + runtime: nvidia + + always-on-sink: + image: ghcr.io/insight-platform/savant-adapters-deepstream-l4t:latest + restart: unless-stopped + ports: + - "554:554" # RTSP + - "1935:1935" # RTMP + - "888:888" # HLS + - "8889:8889" # WebRTC + volumes: + - zmq_sockets:/tmp/zmq-sockets + - ../assets/stub_imgs:/stub_imgs + environment: + - ZMQ_ENDPOINT=sub+connect:ipc:///tmp/zmq-sockets/output-video.ipc + - SOURCE_ID=video + - FRAMERATE=25/1 + - STUB_FILE_LOCATION=/stub_imgs/smpte100_1280x720.jpeg + - DEV_MODE=True + command: python -m adapters.ds.sinks.always_on_rtsp + + etcd: + container_name: etcd + image: bitnamilegacy/etcd:3.6.4-debian-12-r4 + restart: unless-stopped + environment: + - ALLOW_NONE_AUTHENTICATION=yes + - ETCD_ADVERTISE_CLIENT_URLS=http://etcd:2379 + ports: + - "2379:2379" + healthcheck: + test: [ "CMD", "/opt/bitnami/scripts/etcd/healthcheck.sh" ] + interval: 5s + timeout: 5s + retries: 3 + +volumes: + zmq_sockets: diff --git a/samples/scene_change/docker-compose.x86.yml b/samples/scene_change/docker-compose.x86.yml new file mode 100644 index 000000000..86f61bfb0 --- /dev/null +++ b/samples/scene_change/docker-compose.x86.yml @@ -0,0 +1,46 @@ +services: + + video-loop-source: + image: ghcr.io/insight-platform/savant-adapters-gstreamer:latest + extends: + file: docker-compose.l4t.yml + service: video-loop-source + + module: + privileged: true + image: ghcr.io/insight-platform/savant-deepstream:latest + extends: + file: docker-compose.l4t.yml + service: module + runtime: runc + environment: + - CODEC=h264 + deploy: + resources: + reservations: + devices: + - driver: nvidia + count: 1 + capabilities: [ gpu ] + + always-on-sink: + privileged: true + image: ghcr.io/insight-platform/savant-adapters-deepstream:latest + extends: + file: docker-compose.l4t.yml + service: always-on-sink + deploy: + resources: + reservations: + devices: + - driver: nvidia + count: 1 + capabilities: [ gpu ] + + etcd: + extends: + file: docker-compose.l4t.yml + service: etcd + +volumes: + zmq_sockets: diff --git a/samples/scene_change/module.yml b/samples/scene_change/module.yml new file mode 100644 index 000000000..583ad0dd7 --- /dev/null +++ b/samples/scene_change/module.yml @@ -0,0 +1,79 @@ +# module name, required +name: scene_change + +# base module parameters +parameters: + # pipeline processing frame parameters + frame: + width: 1280 + height: 720 + output_frame: + codec: ${oc.env:CODEC, 'h264'} + # PyFunc drawing the ROI and the scene-change status on the frame. + draw_func: + module: samples.scene_change.overlay + class_name: Overlay + # Etcd storage for per-source ROI + etcd: + # Etcd hosts to connect to + hosts: [etcd:2379] + # Path in Etcd to watch changes + watch_path: savant + + # Name of the ReID (attribute) model used to fingerprint the ROI. + reid_model_name: scene_reid + # Default region of interest; format: "left,top,right,bottom". + roi_default: "100,100,1180,620" + + batch_size: 1 + +# pipeline definition +pipeline: + # source definition is skipped, zeromq source is used by default to connect with source adapters + + # define pipeline's main elements + elements: + # Inject the "roi" object; its coordinates are read live from Etcd. + - element: pyfunc + module: samples.scene_change.roi_injector + class_name: RoiInjector + + # ReID model computing the embedding vector of the ROI. + - element: nvinfer@attribute_model + # Model's name in the pipeline, mandatory + name: ${parameters.reid_model_name} + # model definition + model: + remote: + url: s3://savant-data/models/scene_reid/scene_reid.zip + checksum_url: s3://savant-data/models/scene_reid/scene_reid.md5 + parameters: + endpoint: https://eu-central-1.linodeobjects.com + model_file: scene_reid.onnx + batch_size: ${parameters.batch_size} + + input: + object: roi_injector.roi + shape: [3, 224, 224] + scale_factor: 0.01735207357279195 + offsets: [123.675, 116.28, 103.53] + maintain_aspect_ratio: true + output: + layer_names: [output] + converter: + module: savant.converter.vector_attribute + class_name: TensorToVectorConverter + attributes: + - name: reid + + # Detect scene changes based on ReID vectors. + - element: pyfunc + module: samples.scene_change.scene_change + class_name: SceneChangeDetector + kwargs: + # Cosine-distance threshold above which a scene change is reported. + dist_threshold: 0.1 + # Age (seconds) of the baseline fingerprint the current frame is compared against. + reference_delay: 5.0 + + # sink definition is skipped, zeromq sink is used by default to connect with sink adapters diff --git a/samples/scene_change/overlay.py b/samples/scene_change/overlay.py new file mode 100644 index 000000000..0ee1b59e1 --- /dev/null +++ b/samples/scene_change/overlay.py @@ -0,0 +1,50 @@ +"""Overlay drawing the ROI and the scene-change status on the frame.""" + +from samples.scene_change.roi_injector import LABEL as ROI_LABEL +from samples.scene_change.scene_change import ATTR_CHANGED, ATTR_DISTANCE +from samples.scene_change.scene_change import ELEMENT_NAME as SCENE_CHANGE_ELEMENT_NAME +from savant.deepstream.drawfunc import NvDsDrawFunc +from savant.deepstream.meta.frame import NvDsFrameMeta +from savant.utils.artist import Artist, Position + +COLOR_SCENE_STABLE = (64, 255, 32, 255) +COLOR_SCENE_CHANGED = (255, 64, 64, 255) +COLOR_NO_DATA = (255, 192, 0, 255) + + +class Overlay(NvDsDrawFunc): + """Draws the ROI box, scene-change status, and distance.""" + + def draw_on_frame(self, frame_meta: NvDsFrameMeta, artist: Artist): + for obj_meta in frame_meta.objects: + if obj_meta.label != ROI_LABEL: + continue + + changed_attr = obj_meta.get_attr_meta( + SCENE_CHANGE_ELEMENT_NAME, ATTR_CHANGED + ) + dist_attr = obj_meta.get_attr_meta(SCENE_CHANGE_ELEMENT_NAME, ATTR_DISTANCE) + changed = bool(changed_attr.value) if changed_attr is not None else False + distance = dist_attr.value if dist_attr is not None else None + + if distance is None: + text = 'No data' + color = COLOR_NO_DATA + else: + if changed: + status = 'Scene changed' + color = COLOR_SCENE_CHANGED + else: + status = 'Stable' + color = COLOR_SCENE_STABLE + text = f'{status} | dist={distance:.3f}' + + artist.add_bbox(obj_meta.bbox, border_width=2, border_color=color) + artist.add_text( + text, + (int(obj_meta.bbox.left), int(obj_meta.bbox.top)), + padding=(2, 2, 2, 2), + font_color=color, + font_scale=0.8, + anchor_point_type=Position.LEFT_TOP, + ) diff --git a/samples/scene_change/roi_injector.py b/samples/scene_change/roi_injector.py new file mode 100644 index 000000000..646bbe95e --- /dev/null +++ b/samples/scene_change/roi_injector.py @@ -0,0 +1,62 @@ +"""Adds a ROI object to a frame.""" + +from savant_rs.primitives.geometry import BBox +from savant_rs.utils import eval_expr + +from savant.deepstream.meta.frame import NvDsFrameMeta +from savant.deepstream.pyfunc import NvDsPyFuncPlugin +from savant.gstreamer import Gst # noqa: F401 +from savant.meta.object import ObjectMeta +from savant.parameter_storage import param_storage + +ELEMENT_NAME = 'roi_injector' +LABEL = 'roi' +ROI_CACHE_TTL = 5 + + +class RoiInjector(NvDsPyFuncPlugin): + """PyFunc implementing roi injector.""" + + def __init__(self, **kwargs): + super().__init__(**kwargs) + self.default_roi = self._parse_default_roi() + self._roi_cache = {} + + def _parse_default_roi(self): + default_roi_str = str(param_storage()['roi_default']) + left, top, right, bottom = (float(v) for v in default_roi_str.split(',')) + return BBox.ltrb(left, top, right, bottom) + + def _parse_roi(self, raw: str, source_id: str) -> BBox: + """Parse a ROI "left,top,right,bottom".""" + + if not raw: + return self.default_roi + try: + left, top, right, bottom = (float(v) for v in raw.split(',')) + return BBox.ltrb(left, top, right, bottom) + except Exception as e: + self.logger.warning( + 'Failed to parse ROI %r for source %s: %s.', raw, source_id, e + ) + return self.default_roi + + def _read_roi(self, source_id: str) -> BBox: + """Read the current ROI for the source from Etcd.""" + val, is_cached = eval_expr( + f'etcd("roi/{source_id}", "")', + ttl=ROI_CACHE_TTL, + no_gil=True, + ) + if not is_cached or source_id not in self._roi_cache: + self._roi_cache[source_id] = self._parse_roi(val, source_id) + return self._roi_cache[source_id] + + def process_frame(self, buffer: Gst.Buffer, frame_meta: NvDsFrameMeta): + """Callback on each frame in a Deepstream pipeline batch.""" + obj_meta = ObjectMeta( + element_name=ELEMENT_NAME, + label=LABEL, + bbox=self._read_roi(frame_meta.source_id), + ) + frame_meta.add_obj_meta(object_meta=obj_meta) diff --git a/samples/scene_change/scene_change.py b/samples/scene_change/scene_change.py new file mode 100644 index 000000000..682b51fee --- /dev/null +++ b/samples/scene_change/scene_change.py @@ -0,0 +1,114 @@ +"""Scene change detector. + +Reads the ReID "fingerprint" (embedding vector) computed for the ROI object and +compares it with a reference frame using cosine distance. The reference frame +is a frame ``reference_delay`` seconds ago. +""" + +import collections + +import numpy as np + +from samples.scene_change.roi_injector import LABEL as ROI_LABEL +from savant.deepstream.meta.frame import NvDsFrameMeta +from savant.deepstream.pyfunc import NvDsPyFuncPlugin +from savant.gstreamer import Gst +from savant.parameter_storage import param_storage + +# Element/attribute names written by this pyfunc (read by the overlay). +ELEMENT_NAME = 'scene_change' +ATTR_DISTANCE = 'distance' +ATTR_CHANGED = 'changed' + + +def cosine_distance(a: np.ndarray, b: np.ndarray) -> float: + """Cosine distance between two normalized vectors.""" + return 1.0 - float(np.dot(a, b)) + + +class SceneChangeDetector(NvDsPyFuncPlugin): + """Detects scene changes by comparing ROI fingerprints over time.""" + + def __init__( + self, + dist_threshold: float = 0.2, + reference_delay: float = 5.0, + **kwargs, + ): + super().__init__(**kwargs) + self.dist_threshold = dist_threshold + if reference_delay <= 0: + raise ValueError(f'reference_delay must be > 0, got {reference_delay!r}.') + self.reference_delay = reference_delay + self.model_name = param_storage()['reid_model_name'] + self.history = collections.defaultdict(collections.deque) + # To rate-limit logs + self._changed_state = {} + + def process_frame(self, buffer: Gst.Buffer, frame_meta: NvDsFrameMeta): + """Callback on each frame in a Deepstream pipeline batch.""" + roi_obj = None + for obj_meta in frame_meta.objects: + if obj_meta.label == ROI_LABEL: + roi_obj = obj_meta + break + if roi_obj is None: + return + attr = roi_obj.get_attr_meta(self.model_name, 'reid') + if attr is None: + return + + current = np.asarray(attr.value, dtype=np.float32) + norm = np.linalg.norm(current) + if norm == 0: + return + + current = current / norm + source_id = frame_meta.source_id + result = self._compare_delayed(source_id, frame_meta, current) + if result is None: + return + + distance, changed = result + roi_obj.add_attr_meta(ELEMENT_NAME, ATTR_DISTANCE, distance) + roi_obj.add_attr_meta(ELEMENT_NAME, ATTR_CHANGED, bool(changed)) + + # Log only on the transition into a scene change, not every changed frame. + if changed and not self._changed_state.get(source_id, False): + self.logger.info( + 'Scene change detected for source %s (distance %.3f > %.3f).', + source_id, + distance, + self.dist_threshold, + ) + self._changed_state[source_id] = changed + + def _compare_delayed(self, source_id, frame_meta, current): + """Compare the current frame against the reference.""" + + pts = frame_meta.pts + num, den = frame_meta.time_base + if pts is None or den == 0: + return None + + t_sec = pts * num / den + target = t_sec - self.reference_delay + buf = self.history[source_id] + # Drop old entries + while len(buf) >= 2 and buf[1][0] <= target: + buf.popleft() + buf.append((t_sec, current)) + + reference = buf[0][1] if buf and buf[0][0] <= target else None + if reference is None: + # Not enough history yet. + return None + + distance = cosine_distance(current, reference) + changed = distance > self.dist_threshold + + return distance, changed + + def on_source_eos(self, source_id: str): + self.history.pop(source_id, None) + self._changed_state.pop(source_id, None) diff --git a/samples/scene_change/set-roi.sh b/samples/scene_change/set-roi.sh new file mode 100755 index 000000000..e758bb22a --- /dev/null +++ b/samples/scene_change/set-roi.sh @@ -0,0 +1,19 @@ +#!/bin/bash +# Change the region of interest (ROI) for a source at runtime. +# Usage: set-roi.sh SOURCE_ID [ROI] +# ROI format: "left,top,right,bottom" +# If ROI is omitted, the key is removed (the source falls back to the default ROI). + +if [ -z "$1" ]; then + echo "Usage: set-roi.sh SOURCE_ID [ROI]" >&2 + exit 1 +fi + +SOURCE_ID=$1 +ROI=$2 + +if [ -z "${ROI}" ]; then + docker exec -it etcd etcdctl del savant/roi/"${SOURCE_ID}" +else + docker exec -it etcd etcdctl put savant/roi/"${SOURCE_ID}" "${ROI}" +fi From 1b7f1f1efeec5e63941ac865a828d4cf65cf7af9 Mon Sep 17 00:00:00 2001 From: "Pavel A. Tomskikh" Date: Thu, 16 Jul 2026 15:13:05 +0700 Subject: [PATCH 2/2] Add ROI validation --- samples/scene_change/roi_injector.py | 67 ++++++++++++++++++---------- 1 file changed, 43 insertions(+), 24 deletions(-) diff --git a/samples/scene_change/roi_injector.py b/samples/scene_change/roi_injector.py index 646bbe95e..02a119c1e 100644 --- a/samples/scene_change/roi_injector.py +++ b/samples/scene_change/roi_injector.py @@ -1,5 +1,7 @@ """Adds a ROI object to a frame.""" +from typing import Tuple + from savant_rs.primitives.geometry import BBox from savant_rs.utils import eval_expr @@ -11,7 +13,8 @@ ELEMENT_NAME = 'roi_injector' LABEL = 'roi' -ROI_CACHE_TTL = 5 +ROI_CACHE_TTL = 1000 +MIN_ROI_SIZE = 32 class RoiInjector(NvDsPyFuncPlugin): @@ -19,41 +22,57 @@ class RoiInjector(NvDsPyFuncPlugin): def __init__(self, **kwargs): super().__init__(**kwargs) - self.default_roi = self._parse_default_roi() - self._roi_cache = {} - - def _parse_default_roi(self): - default_roi_str = str(param_storage()['roi_default']) - left, top, right, bottom = (float(v) for v in default_roi_str.split(',')) - return BBox.ltrb(left, top, right, bottom) - - def _parse_roi(self, raw: str, source_id: str) -> BBox: - """Parse a ROI "left,top,right,bottom".""" - - if not raw: - return self.default_roi - try: - left, top, right, bottom = (float(v) for v in raw.split(',')) - return BBox.ltrb(left, top, right, bottom) - except Exception as e: - self.logger.warning( - 'Failed to parse ROI %r for source %s: %s.', raw, source_id, e + self.frame_width = param_storage()['frame']['width'] + self.frame_height = param_storage()['frame']['height'] + self.default_roi = BBox.ltrb( + *self._parse_ltrb(str(param_storage()['roi_default'])) + ) + self.per_source_roi = {} + + def _parse_ltrb(self, raw: str) -> Tuple[int, int, int, int]: + """Parse the ROI string as (left, top, right, bottom).""" + + left, top, right, bottom = (int(v) for v in raw.split(',')) + if not ( + 0 <= left < right <= self.frame_width + and 0 <= top < bottom <= self.frame_height + ): + raise ValueError( + f'ROI {raw} is out of frame bounds ' + f'({self.frame_width}x{self.frame_height}).' + ) + if right - left < MIN_ROI_SIZE or bottom - top < MIN_ROI_SIZE: + raise ValueError( + f'ROI {raw} size must be at least {MIN_ROI_SIZE} in each dimension.' ) - return self.default_roi + + return left, top, right, bottom def _read_roi(self, source_id: str) -> BBox: """Read the current ROI for the source from Etcd.""" + val, is_cached = eval_expr( f'etcd("roi/{source_id}", "")', ttl=ROI_CACHE_TTL, no_gil=True, ) - if not is_cached or source_id not in self._roi_cache: - self._roi_cache[source_id] = self._parse_roi(val, source_id) - return self._roi_cache[source_id] + if not is_cached: + if val: + try: + self.per_source_roi[source_id] = BBox.ltrb(*self._parse_ltrb(val)) + except Exception as e: + self.logger.warning( + 'Failed to parse ROI %r for source %s: %s.', val, source_id, e + ) + self.per_source_roi.pop(source_id, None) + else: + self.per_source_roi.pop(source_id, None) + + return self.per_source_roi.get(source_id, self.default_roi) def process_frame(self, buffer: Gst.Buffer, frame_meta: NvDsFrameMeta): """Callback on each frame in a Deepstream pipeline batch.""" + obj_meta = ObjectMeta( element_name=ELEMENT_NAME, label=LABEL,