feat: enabled device camera input via browser for tree_demo

README.md

@@ -75,6 +75,10 @@ NanoOWL runs real-time on Jetson Orin Nano.
 <a id="setup"></a>
 ## 🛠️ Setup
 
+| Repository/Tag | Date | Arch | Size |
+| :-- | :--: | :--: | :--: |
+| &nbsp;&nbsp;[`roylvzn/owlvit:latest`](https://hub.docker.com/repository/docker/roylvzn/owlvit/general) | `2025-12-25` | `arm64` | `9.66GB` |
+
 1. Install the dependencies
 
     1. Install PyTorch

examples/tree_demo/index.html

@@ -50,6 +50,10 @@
             width: 640px;
         }
 
+        #videoElement {
+            display: none;
+        }
+
     </style>
 
     <script type="text/javascript">
@@ -62,7 +66,10 @@
         }
 
         var ws = undefined
-
+        var streaming = false
+
+        const canvas = document.createElement("canvas");
+        const ctx = canvas.getContext("2d");
         console.log(location.host);
 
         ws = new WebSocket("ws://" + location.host + "/ws");
@@ -97,6 +104,47 @@
                 console.log("Received message.");
                 camera_image.src = reader.result;
             }
+        };
+
+        async function startCamera() {
+            if (!ws || ws.readyState !== WebSocket.OPEN) {
+                console.warn("WebSocket not ready");
+                return;
+            }
+            ws.send("use_browser");
+            const video = document.getElementById("videoElement");
+
+            const stream = await navigator.mediaDevices.getUserMedia({
+                video: true,
+                audio: false
+            });
+
+            video.srcObject = stream;
+            await new Promise(resolve => {
+                video.onloadedmetadata = () => resolve();
+            });
+
+            await video.play();
+
+            canvas.width = video.videoWidth;
+            canvas.height = video.videoHeight;
+
+            console.log("Browser camera started:", canvas.width, canvas.height);
+
+            streaming = true;
+            sendFrameLoop(video);
+        }
+
+        function sendFrameLoop(video) {
+            if (!streaming || ws.readyState !== WebSocket.OPEN) return;
+
+            ctx.drawImage(video, 0, 0, canvas.width, canvas.height);
+
+            canvas.toBlob(function (blob) {
+                if (blob) ws.send(blob);
+            }, "image/jpeg", 0.7);
+
+            setTimeout(() => sendFrameLoop(video), 33);
         }
 
     </script>
@@ -106,7 +154,9 @@
         <h1>NanoOWL</h1>
         <img id="camera_image" src="" alt="Camera Image"/>
         <br/>
+        <button id="startButton" onclick="startCamera()">Use Browser</button>
         <input id="prompt_input" type="text" placeholder="[a face [an eye, a nose]]"/>
     </div>
+    <video id="videoElement" autoplay playsinline muted style="display:none;"></video>
 </body>
-</html>
+</html>

examples/tree_demo/tree_demo.py

@@ -19,6 +19,7 @@
 from aiohttp import web, WSCloseCode
 import logging
 import weakref
+import re
 import cv2
 import time
 import PIL.Image
@@ -38,13 +39,14 @@
     parser.add_argument("--image_quality", type=int, default=50)
     parser.add_argument("--port", type=int, default=7860)
     parser.add_argument("--host", type=str, default="0.0.0.0")
-    parser.add_argument("--camera", type=int, default=0)
+    parser.add_argument("--camera", type=int, default=0, help="0=USB/CSI (/dev/video0), 1=Browser (Websocket frames)")
     parser.add_argument("--resolution", type=str, default="640x480", help="Camera resolution as WIDTHxHEIGHT")
     args = parser.parse_args()
     width, height = map(int, args.resolution.split("x"))
 
-    CAMERA_DEVICE = args.camera
+    CAMERA_MODE = int(args.camera)
     IMAGE_QUALITY = args.image_quality
+    USE_BROWSER = False
 
     predictor = TreePredictor(
         owl_predictor=OwlPredictor(
@@ -53,7 +55,8 @@
     )
 
     prompt_data = None
-
+    latest_frame = None
+
     def get_colors(count: int):
         cmap = plt.cm.get_cmap("rainbow", count)
         colors = []
@@ -76,7 +79,7 @@ async def handle_index_get(request: web.Request):
 
     async def websocket_handler(request):
 
-        global prompt_data
+        global prompt_data, latest_frame, USE_BROWSER
 
         ws = web.WebSocketResponse()
 
@@ -88,8 +91,19 @@ async def websocket_handler(request):
 
         try:
             async for msg in ws:
-                logging.info(f"Received message from websocket.")
-                if "prompt" in msg.data:
+                # logging.info(f"Received data from websocket.")
+                if msg.type == web.WSMsgType.BINARY:
+                    latest_frame = cv2.imdecode(
+                        np.frombuffer(msg.data, np.uint8),
+                        cv2.IMREAD_COLOR
+                    )
+                elif msg.type == web.WSMsgType.TEXT:
+                    if msg.data == "use_browser":
+                        if not USE_BROWSER:
+                            USE_BROWSER = True
+                            logging.info("User switched to browser camera")
+
+                    elif msg.data.startswith("prompt:"):
                     header, prompt = msg.data.split(":")
                     logging.info("Received prompt: " + prompt)
                     try:
@@ -120,20 +134,29 @@ async def detection_loop(app: web.Application):
 
         loop = asyncio.get_running_loop()
 
-        logging.info("Opening camera.")
+        logging.info(f"Opening camera: {CAMERA_DEVICE}")
 
-        camera = cv2.VideoCapture(CAMERA_DEVICE)
-        camera.set(cv2.CAP_PROP_FRAME_WIDTH, width)
-        camera.set(cv2.CAP_PROP_FRAME_HEIGHT, height)
+        if CAMERA_DEVICE == 0:
+            logging.info(f"Opening V4L2 camera index: {CAMERA_DEVICE}")
+            camera = cv2.VideoCapture(CAMERA_DEVICE)
+            camera.set(cv2.CAP_PROP_FRAME_WIDTH, width)
+            camera.set(cv2.CAP_PROP_FRAME_HEIGHT, height)
+        else:
+            camera = None
 
         logging.info("Loading predictor.")
 
         def _read_and_encode_image():
 
-            re, image = camera.read()
-
-            if not re:
-                return re, None
+            if USE_BROWSER:
+                image = latest_frame
+                if image is None:
+                    return False, None
+                re = True
+            else:
+                re, image = camera.read()
+                if not re:
+                    return False, None
 
             image_pil = cv2_to_pil(image)
 
@@ -162,12 +185,14 @@ def _read_and_encode_image():
             re, image = await loop.run_in_executor(None, _read_and_encode_image)
 
             if not re:
-                break
+                await asyncio.sleep(0.01)
+                continue
 
             for ws in app["websockets"]:
                 await ws.send_bytes(image)
 
-        camera.release()
+        if camera is not None:
+            camera.release()
 
 
     async def run_detection_loop(app):

nanoowl/owl_predictor.py

@@ -156,6 +156,7 @@ def __init__(self,
         self.device = device
         self.model = OwlViTForObjectDetection.from_pretrained(model_name).to(self.device).eval()
         self.processor = OwlViTProcessor.from_pretrained(model_name)
+        self.image_encoder_engine = None
         self.patch_size = _owl_get_patch_size(model_name)
         self.num_patches_per_side = self.image_size // self.patch_size
         self.box_bias = _owl_compute_box_bias(self.num_patches_per_side).to(self.device)