object_detector.js

const ort = require("onnxruntime-web");
const express = require('express');
const multer = require("multer");
const sharp = require("sharp");
const fs = require("fs");

/**
 * Main function that setups and starts a
 * web server on port 8080
 */
function main() {
    const app = express();
    const upload = multer();

    /**
     * The site root handler. Returns content of index.html file.
     */
    app.get("/", (req,res) => {
        res.end(fs.readFileSync("index.html", "utf8"))
    })

    /**
     * The handler of /detect endpoint that receives uploaded
     * image file, passes it through YOLOv8 object detection network and returns
     * an array of bounding boxes in format [[x1,y1,x2,y2,object_type,probability],..] as a JSON
     */
    app.post('/detect', upload.single('image_file'), async function (req, res) {
        console.log("\n=== New Detection Request ===");
        console.log(`File received: ${req.file.originalname}, size: ${req.file.size} bytes`);
        
        const boxes = await detect_objects_on_image(req.file.buffer);
        
        console.log(`\n=== FINAL DETECTION RESULTS ===`);
        console.log(`Total objects detected: ${boxes.length}`);
        
        if (boxes.length === 0) {
            console.log("No objects detected (confidence < 0.5 or filtered by NMS)");
        } else {
            boxes.forEach((box, idx) => {
                console.log(`\nObject ${idx + 1}:`);
                console.log(`  - Label: ${box[4]}`);
                console.log(`  - Confidence: ${(box[5] * 100).toFixed(2)}%`);
                console.log(`  - Bounding Box: [x1: ${box[0].toFixed(2)}, y1: ${box[1].toFixed(2)}, x2: ${box[2].toFixed(2)}, y2: ${box[3].toFixed(2)}]`);
            });
        }
        console.log("=== End of Results ===\n");
        
        res.json(boxes);
    });

    app.listen(8080, () => {
        console.log(`Server is listening on port 8080`);
        console.log(`URL: http://localhost:8080`);
    });
}

/**
 * Function receives an image, passes it through YOLOv8 neural network
 * and returns an array of detected objects and their bounding boxes
 * @param buf Input image body
 * @returns Array of bounding boxes in format [[x1,y1,x2,y2,object_type,probability],..]
 */
async function detect_objects_on_image(buf) {
    console.log("Preparing input image...");
    const [input,img_width,img_height] = await prepare_input(buf);
    console.log(`Image prepared: original size ${img_width}x${img_height}`);
    
    console.log("Running model inference...");
    const output = await run_model(input);
    console.log("Model inference completed");
    
    console.log("Processing output (filtering and NMS)...");
    const results = process_output(output,img_width,img_height);
    
    return results;
}

/**
 * Function used to convert input image to tensor,
 * required as an input to YOLOv8 object detection
 * network.
 * @param buf Content of uploaded file
 * @returns Array of pixels
 */
async function prepare_input(buf) {
    const img = sharp(buf);
    const md = await img.metadata();
    const [img_width,img_height] = [md.width, md.height];
    const pixels = await img.removeAlpha()
        .resize({width:640,height:640,fit:'fill'})
        .raw()
        .toBuffer();
    const red = [], green = [], blue = [];
    for (let index=0; index<pixels.length; index+=3) {
        red.push(pixels[index]/255.0);
        green.push(pixels[index+1]/255.0);
        blue.push(pixels[index+2]/255.0);
    }
    const input = [...red, ...green, ...blue];
    return [input, img_width, img_height];
}

/**
 * Function used to pass provided input tensor to YOLOv8 neural network and return result
 * @param input Input pixels array
 * @returns Raw output of neural network as a flat array of numbers
 */
async function run_model(input) {
    console.log("Loading ONNX model...");
    const model = await ort.InferenceSession.create("yolov8m.onnx");
    console.log("Model loaded successfully");
    
    input = new ort.Tensor(Float32Array.from(input),[1, 3, 640, 640]);
    const outputs = await model.run({images:input});
    return outputs["output0"].data;
}

/**
 * Function used to convert RAW output from YOLOv8 to an array of detected objects.
 * Each object contain the bounding box of this object, the type of object and the probability
 * @param output Raw output of YOLOv8 network
 * @param img_width Width of original image
 * @param img_height Height of original image
 * @returns Array of detected objects in a format [[x1,y1,x2,y2,object_type,probability],..]
 */
function process_output(output, img_width, img_height) {
    let boxes = [];
    let rawDetections = []; // Store raw detections before NMS for logging
    
    for (let index=0;index<8400;index++) {
        const [class_id,prob] = [...Array(80).keys()]
            .map(col => [col, output[8400*(col+4)+index]])
            .reduce((accum, item) => item[1]>accum[1] ? item : accum,[0,0]);
        
        if (prob >= 0.5) {
            const label = yolo_classes[class_id];
            const xc = output[index];
            const yc = output[8400+index];
            const w = output[2*8400+index];
            const h = output[3*8400+index];
            const x1 = (xc-w/2)/640*img_width;
            const y1 = (yc-h/2)/640*img_height;
            const x2 = (xc+w/2)/640*img_width;
            const y2 = (yc+h/2)/640*img_height;
            
            rawDetections.push({label, prob, x1, y1, x2, y2});
            boxes.push([x1,y1,x2,y2,label,prob]);
        }
    }
    
    // Log raw detections before NMS
    console.log(`\n--- Raw detections (before NMS): ${rawDetections.length} objects with confidence >= 0.5 ---`);
    if (rawDetections.length > 0) {
        rawDetections.forEach((det, i) => {
            console.log(`  ${i+1}. ${det.label} (${(det.prob * 100).toFixed(2)}%)`);
        });
    } else {
        console.log("  No detections above threshold");
    }
    
    // Sort by confidence
    boxes = boxes.sort((box1,box2) => box2[5]-box1[5]);
    
    // Apply NMS
    const result = [];
    while (boxes.length > 0) {
        result.push(boxes[0]);
        boxes = boxes.filter(box => iou(boxes[0], box) < 0.7);
    }
    
    console.log(`\n--- After NMS (IoU threshold 0.7): ${result.length} objects retained ---`);
    if (result.length > 0) {
        result.forEach((det, i) => {
            console.log(`  ${i+1}. ${det[4]} (${(det[5] * 100).toFixed(2)}%)`);
        });
    }
    
    return result;
}

/**
 * Function calculates "Intersection-over-union" coefficient for specified two boxes
 * https://pyimagesearch.com/2016/11/07/intersection-over-union-iou-for-object-detection/.
 * @param box1 First box in format: [x1,y1,x2,y2,object_class,probability]
 * @param box2 Second box in format: [x1,y1,x2,y2,object_class,probability]
 * @returns Intersection over union ratio as a float number
 */
function iou(box1,box2) {
    return intersection(box1,box2)/union(box1,box2);
}

/**
 * Function calculates union area of two boxes.
 *     :param box1: First box in format [x1,y1,x2,y2,object_class,probability]
 *     :param box2: Second box in format [x1,y1,x2,y2,object_class,probability]
 *     :return: Area of the boxes union as a float number
 * @param box1 First box in format [x1,y1,x2,y2,object_class,probability]
 * @param box2 Second box in format [x1,y1,x2,y2,object_class,probability]
 * @returns Area of the boxes union as a float number
 */
function union(box1,box2) {
    const [box1_x1,box1_y1,box1_x2,box1_y2] = box1;
    const [box2_x1,box2_y1,box2_x2,box2_y2] = box2;
    const box1_area = (box1_x2-box1_x1)*(box1_y2-box1_y1)
    const box2_area = (box2_x2-box2_x1)*(box2_y2-box2_y1)
    return box1_area + box2_area - intersection(box1,box2)
}

/**
 * Function calculates intersection area of two boxes
 * @param box1 First box in format [x1,y1,x2,y2,object_class,probability]
 * @param box2 Second box in format [x1,y1,x2,y2,object_class,probability]
 * @returns Area of intersection of the boxes as a float number
 */
function intersection(box1,box2) {
    const [box1_x1,box1_y1,box1_x2,box1_y2] = box1;
    const [box2_x1,box2_y1,box2_x2,box2_y2] = box2;
    const x1 = Math.max(box1_x1,box2_x1);
    const y1 = Math.max(box1_y1,box2_y1);
    const x2 = Math.min(box1_x2,box2_x2);
    const y2 = Math.min(box1_y2,box2_y2);
    return (x2-x1)*(y2-y1)
}

/**
 * Array of YOLOv8 class labels
 */
const yolo_classes = [
    'person', 'bicycle', 'car', 'motorcycle', 'airplane', 'bus', 'train', 'truck', 'boat',
    'traffic light', 'fire hydrant', 'stop sign', 'parking meter', 'bench', 'bird', 'cat', 'dog', 'horse',
    'sheep', 'cow', 'elephant', 'bear', 'zebra', 'giraffe', 'backpack', 'umbrella', 'handbag', 'tie', 'suitcase',
    'frisbee', 'skis', 'snowboard', 'sports ball', 'kite', 'baseball bat', 'baseball glove', 'skateboard',
    'surfboard', 'tennis racket', 'bottle', 'wine glass', 'cup', 'fork', 'knife', 'spoon', 'bowl', 'banana', 'apple',
    'sandwich', 'orange', 'broccoli', 'carrot', 'hot dog', 'pizza', 'donut', 'cake', 'chair', 'couch', 'potted plant',
    'bed', 'dining table', 'toilet', 'tv', 'laptop', 'mouse', 'remote', 'keyboard', 'cell phone', 'microwave', 'oven',
    'toaster', 'sink', 'refrigerator', 'book', 'clock', 'vase', 'scissors', 'teddy bear', 'hair drier', 'toothbrush'
];

main()