Swift cam scene change demo

samples/scene_change/README.md

@@ -0,0 +1,58 @@
+# Scene change
+
+The pipeline computes a ReID embedding vector for a region of interest (ROI) on the frame and compares it with a vector of a frame N seconds ago to detect a scene change (e.g., when a camera moved). The ROI is configurable per-source in Etcd.
+
+Tested on platforms:
+
+- Nvidia Ampere
+
+## Prerequisites
+
+```bash
+git clone https://github.com/insight-platform/Savant.git
+cd Savant
+git lfs pull
+./utils/check-environment-compatible
+```
+
+**Note**: Ubuntu 22.04 runtime configuration [guide](https://insight-platform.github.io/Savant/develop/getting_started/0_configure_prod_env.html) helps to configure the runtime to run Savant pipelines.
+
+## Build Engines
+
+The demo uses models that are compiled into TensorRT engines the first time the demo is run. This takes time. Optionally, you can prepare the engines before running the demo by using the command:
+
+```bash
+# you are expected to be in Savant/ directory
+
+./scripts/run_module.py --build-engines samples/scene_change/module.yml
+```
+
+## Run Demo
+
+```bash
+# you are expected to be in Savant/ directory
+
+# if x86
+docker compose -f samples/scene_change/docker-compose.x86.yml up
+
+# if Jetson
+docker compose -f samples/scene_change/docker-compose.l4t.yml up
+
+# open 'rtsp://127.0.0.1:554/stream/video' in your player
+# or visit 'http://127.0.0.1:888/stream/video/' (LL-HLS)
+
+# Ctrl+C to stop running the compose bundle
+```
+
+## ROI configuration
+
+By default, the pipeline uses ROI from the [module configuration](module.yml). By changing the value of the key `savant/roi/{source_id}` in Etcd you can change ROI of the corresponding source.
+
+To change a source ROI it is convenient to use the script:
+
+```bash
+# you are expected to be in Savant/ directory
+./samples/scene_change/set-roi.sh video "540,100,1000,400"
+# to reset to the default ROI
+./samples/scene_change/set-roi.sh video
+```

samples/scene_change/docker-compose.l4t.yml

@@ -0,0 +1,76 @@
+services:
+
+  video-loop-source:
+    image: ghcr.io/insight-platform/savant-adapters-gstreamer-l4t:latest
+    restart: unless-stopped
+    volumes:
+      - zmq_sockets:/tmp/zmq-sockets
+      - /tmp/video-loop-source-downloads:/tmp/video-loop-source-downloads
+    environment:
+      - LOCATION=https://eu-central-1.linodeobjects.com/savant-data/demo/scene_change_cam_rotation_yaw.mp4
+      - DOWNLOAD_PATH=/tmp/video-loop-source-downloads
+      - ZMQ_ENDPOINT=pub+connect:ipc:///tmp/zmq-sockets/input-video.ipc
+      - SOURCE_ID=video
+      - SYNC_OUTPUT=True
+    entrypoint: /opt/savant/adapters/gst/sources/video_loop.sh
+    depends_on:
+      module:
+        condition: service_healthy
+
+  module:
+    privileged: true
+    image: ghcr.io/insight-platform/savant-deepstream-l4t:latest
+    restart: unless-stopped
+    volumes:
+      - zmq_sockets:/tmp/zmq-sockets
+      - ../../cache:/cache
+      - ..:/opt/savant/samples
+    command: samples/scene_change/module.yml
+    environment:
+      - MODEL_PATH=/cache/models/scene_change
+      - DOWNLOAD_PATH=/cache/downloads/scene_change
+      - ZMQ_SRC_ENDPOINT=sub+bind:ipc:///tmp/zmq-sockets/input-video.ipc
+      - ZMQ_SINK_ENDPOINT=pub+bind:ipc:///tmp/zmq-sockets/output-video.ipc
+      - METRICS_FRAME_PERIOD=1000
+      - CODEC=jpeg
+    depends_on:
+      etcd:
+        condition: service_healthy
+    runtime: nvidia
+
+  always-on-sink:
+    image: ghcr.io/insight-platform/savant-adapters-deepstream-l4t:latest
+    restart: unless-stopped
+    ports:
+      - "554:554"    # RTSP
+      - "1935:1935"  # RTMP
+      - "888:888"    # HLS
+      - "8889:8889"  # WebRTC
+    volumes:
+      - zmq_sockets:/tmp/zmq-sockets
+      - ../assets/stub_imgs:/stub_imgs
+    environment:
+      - ZMQ_ENDPOINT=sub+connect:ipc:///tmp/zmq-sockets/output-video.ipc
+      - SOURCE_ID=video
+      - FRAMERATE=25/1
+      - STUB_FILE_LOCATION=/stub_imgs/smpte100_1280x720.jpeg
+      - DEV_MODE=True
+    command: python -m adapters.ds.sinks.always_on_rtsp
+
+  etcd:
+    container_name: etcd
+    image: bitnamilegacy/etcd:3.6.4-debian-12-r4
+    restart: unless-stopped
+    environment:
+      - ALLOW_NONE_AUTHENTICATION=yes
+      - ETCD_ADVERTISE_CLIENT_URLS=http://etcd:2379
+    ports:
+      - "2379:2379"
+    healthcheck:
+      test: [ "CMD", "/opt/bitnami/scripts/etcd/healthcheck.sh" ]
+      interval: 5s
+      timeout: 5s
+      retries: 3
+
+volumes:
+  zmq_sockets:

samples/scene_change/docker-compose.x86.yml

@@ -0,0 +1,46 @@
+services:
+
+  video-loop-source:
+    image: ghcr.io/insight-platform/savant-adapters-gstreamer:latest
+    extends:
+      file: docker-compose.l4t.yml
+      service: video-loop-source
+
+  module:
+    privileged: true
+    image: ghcr.io/insight-platform/savant-deepstream:latest
+    extends:
+      file: docker-compose.l4t.yml
+      service: module
+    runtime: runc
+    environment:
+      - CODEC=h264
+    deploy:
+      resources:
+        reservations:
+          devices:
+            - driver: nvidia
+              count: 1
+              capabilities: [ gpu ]
+
+  always-on-sink:
+    privileged: true
+    image: ghcr.io/insight-platform/savant-adapters-deepstream:latest
+    extends:
+      file: docker-compose.l4t.yml
+      service: always-on-sink
+    deploy:
+      resources:
+        reservations:
+          devices:
+            - driver: nvidia
+              count: 1
+              capabilities: [ gpu ]
+
+  etcd:
+    extends:
+      file: docker-compose.l4t.yml
+      service: etcd
+
+volumes:
+  zmq_sockets:

samples/scene_change/module.yml

@@ -0,0 +1,79 @@
+# module name, required
+name: scene_change
+
+# base module parameters
+parameters:
+  # pipeline processing frame parameters
+  frame:
+    width: 1280
+    height: 720
+  output_frame:
+    codec: ${oc.env:CODEC, 'h264'}
+  # PyFunc drawing the ROI and the scene-change status on the frame.
+  draw_func:
+    module: samples.scene_change.overlay
+    class_name: Overlay
+  # Etcd storage for per-source ROI
+  etcd:
+    # Etcd hosts to connect to
+    hosts: [etcd:2379]
+    # Path in Etcd to watch changes
+    watch_path: savant
+
+  # Name of the ReID (attribute) model used to fingerprint the ROI.
+  reid_model_name: scene_reid
+  # Default region of interest; format: "left,top,right,bottom".
+  roi_default: "100,100,1180,620"
+
+  batch_size: 1
+
+# pipeline definition
+pipeline:
+  # source definition is skipped, zeromq source is used by default to connect with source adapters
+
+  # define pipeline's main elements
+  elements:
+    # Inject the "roi" object; its coordinates are read live from Etcd.
+    - element: pyfunc
+      module: samples.scene_change.roi_injector
+      class_name: RoiInjector
+
+    # ReID model computing the embedding vector of the ROI.
+    - element: nvinfer@attribute_model
+      # Model's name in the pipeline, mandatory
+      name: ${parameters.reid_model_name}
+      # model definition
+      model:
+        remote:
+          url: s3://savant-data/models/scene_reid/scene_reid.zip
+          checksum_url: s3://savant-data/models/scene_reid/scene_reid.md5
+          parameters:
+            endpoint: https://eu-central-1.linodeobjects.com
+        model_file: scene_reid.onnx
+        batch_size: ${parameters.batch_size}
+
+        input:
+          object: roi_injector.roi
+          shape: [3, 224, 224]
+          scale_factor: 0.01735207357279195
+          offsets: [123.675, 116.28, 103.53]
+          maintain_aspect_ratio: true
+        output:
+          layer_names: [output]
+          converter:
+            module: savant.converter.vector_attribute
+            class_name: TensorToVectorConverter
+          attributes:
+            - name: reid
+
+    # Detect scene changes based on ReID vectors.
+    - element: pyfunc
+      module: samples.scene_change.scene_change
+      class_name: SceneChangeDetector
+      kwargs:
+        # Cosine-distance threshold above which a scene change is reported.
+        dist_threshold: 0.1
+        # Age (seconds) of the baseline fingerprint the current frame is compared against.
+        reference_delay: 5.0
+
+  # sink definition is skipped, zeromq sink is used by default to connect with sink adapters

samples/scene_change/overlay.py

@@ -0,0 +1,50 @@
+"""Overlay drawing the ROI and the scene-change status on the frame."""
+
+from samples.scene_change.roi_injector import LABEL as ROI_LABEL
+from samples.scene_change.scene_change import ATTR_CHANGED, ATTR_DISTANCE
+from samples.scene_change.scene_change import ELEMENT_NAME as SCENE_CHANGE_ELEMENT_NAME
+from savant.deepstream.drawfunc import NvDsDrawFunc
+from savant.deepstream.meta.frame import NvDsFrameMeta
+from savant.utils.artist import Artist, Position
+
+COLOR_SCENE_STABLE = (64, 255, 32, 255)
+COLOR_SCENE_CHANGED = (255, 64, 64, 255)
+COLOR_NO_DATA = (255, 192, 0, 255)
+
+
+class Overlay(NvDsDrawFunc):
+    """Draws the ROI box, scene-change status, and distance."""
+
+    def draw_on_frame(self, frame_meta: NvDsFrameMeta, artist: Artist):
+        for obj_meta in frame_meta.objects:
+            if obj_meta.label != ROI_LABEL:
+                continue
+
+            changed_attr = obj_meta.get_attr_meta(
+                SCENE_CHANGE_ELEMENT_NAME, ATTR_CHANGED
+            )
+            dist_attr = obj_meta.get_attr_meta(SCENE_CHANGE_ELEMENT_NAME, ATTR_DISTANCE)
+            changed = bool(changed_attr.value) if changed_attr is not None else False
+            distance = dist_attr.value if dist_attr is not None else None
+
+            if distance is None:
+                text = 'No data'
+                color = COLOR_NO_DATA
+            else:
+                if changed:
+                    status = 'Scene changed'
+                    color = COLOR_SCENE_CHANGED
+                else:
+                    status = 'Stable'
+                    color = COLOR_SCENE_STABLE
+                text = f'{status} | dist={distance:.3f}'
+
+            artist.add_bbox(obj_meta.bbox, border_width=2, border_color=color)
+            artist.add_text(
+                text,
+                (int(obj_meta.bbox.left), int(obj_meta.bbox.top)),
+                padding=(2, 2, 2, 2),
+                font_color=color,
+                font_scale=0.8,
+                anchor_point_type=Position.LEFT_TOP,
+            )

samples/scene_change/roi_injector.py

@@ -0,0 +1,81 @@
+"""Adds a ROI object to a frame."""
+
+from typing import Tuple
+
+from savant_rs.primitives.geometry import BBox
+from savant_rs.utils import eval_expr
+
+from savant.deepstream.meta.frame import NvDsFrameMeta
+from savant.deepstream.pyfunc import NvDsPyFuncPlugin
+from savant.gstreamer import Gst  # noqa: F401
+from savant.meta.object import ObjectMeta
+from savant.parameter_storage import param_storage
+
+ELEMENT_NAME = 'roi_injector'
+LABEL = 'roi'
+ROI_CACHE_TTL = 1000
+MIN_ROI_SIZE = 32
+
+
+class RoiInjector(NvDsPyFuncPlugin):
+    """PyFunc implementing roi injector."""
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+        self.frame_width = param_storage()['frame']['width']
+        self.frame_height = param_storage()['frame']['height']
+        self.default_roi = BBox.ltrb(
+            *self._parse_ltrb(str(param_storage()['roi_default']))
+        )
+        self.per_source_roi = {}
+
+    def _parse_ltrb(self, raw: str) -> Tuple[int, int, int, int]:
+        """Parse the ROI string as (left, top, right, bottom)."""
+
+        left, top, right, bottom = (int(v) for v in raw.split(','))
+        if not (
+            0 <= left < right <= self.frame_width
+            and 0 <= top < bottom <= self.frame_height
+        ):
+            raise ValueError(
+                f'ROI {raw} is out of frame bounds '
+                f'({self.frame_width}x{self.frame_height}).'
+            )
+        if right - left < MIN_ROI_SIZE or bottom - top < MIN_ROI_SIZE:
+            raise ValueError(
+                f'ROI {raw} size must be at least {MIN_ROI_SIZE} in each dimension.'
+            )
+
+        return left, top, right, bottom
+
+    def _read_roi(self, source_id: str) -> BBox:
+        """Read the current ROI for the source from Etcd."""
+
+        val, is_cached = eval_expr(
+            f'etcd("roi/{source_id}", "")',
+            ttl=ROI_CACHE_TTL,
+            no_gil=True,
+        )
+        if not is_cached:
+            if val:
+                try:
+                    self.per_source_roi[source_id] = BBox.ltrb(*self._parse_ltrb(val))
+                except Exception as e:
+                    self.logger.warning(
+                        'Failed to parse ROI %r for source %s: %s.', val, source_id, e
+                    )
+                    self.per_source_roi.pop(source_id, None)
+            else:
+                self.per_source_roi.pop(source_id, None)
+
+        return self.per_source_roi.get(source_id, self.default_roi)
+
+    def process_frame(self, buffer: Gst.Buffer, frame_meta: NvDsFrameMeta):
+        """Callback on each frame in a Deepstream pipeline batch."""
+
+        obj_meta = ObjectMeta(
+            element_name=ELEMENT_NAME,
+            label=LABEL,
+            bbox=self._read_roi(frame_meta.source_id),
+        )
+        frame_meta.add_obj_meta(object_meta=obj_meta)