Object Detection — YOLO and Feature Pyramids

Classification is multiple choice: "Is this a cat, dog, or bird?" Object detection is open-ended search: "Find every animal in this photo, draw a box around each one, and tell me what kind it is." The search problem is fundamentally harder — the model must check every possible location and scale simultaneously.

YOLO's insight: instead of searching sequentially, divide the image into a grid and have every grid cell predict boxes simultaneously. One forward pass, all detections at once. Fast enough for real-time video at 30+ frames per second.

import numpy as np
import torch

# ── IoU from scratch — two formats ───────────────────────────────────
def iou_corner_format(box1, box2):
    """
    box: [x_min, y_min, x_max, y_max]
    Returns IoU scalar.
    """
    # Intersection
    x1 = max(box1[0], box2[0])
    y1 = max(box1[1], box2[1])
    x2 = min(box1[2], box2[2])
    y2 = min(box1[3], box2[3])

    inter_w = max(0, x2 - x1)
    inter_h = max(0, y2 - y1)
    intersection = inter_w * inter_h

    # Union
    area1 = (box1[2] - box1[0]) * (box1[3] - box1[1])
    area2 = (box2[2] - box2[0]) * (box2[3] - box2[1])
    union = area1 + area2 - intersection

    return intersection / (union + 1e-8)

def iou_batch(boxes1: torch.Tensor, boxes2: torch.Tensor) -> torch.Tensor:
    """
    Vectorised IoU for (N, 4) and (M, 4) tensors.
    Returns (N, M) IoU matrix — every pair.
    Used in NMS and matching.
    """
    area1 = (boxes1[:, 2] - boxes1[:, 0]) * (boxes1[:, 3] - boxes1[:, 1])
    area2 = (boxes2[:, 2] - boxes2[:, 0]) * (boxes2[:, 3] - boxes2[:, 1])

    # Broadcast to compute all pairs
    lt = torch.max(boxes1[:, None, :2], boxes2[None, :, :2])  # (N, M, 2)
    rb = torch.min(boxes1[:, None, 2:], boxes2[None, :, 2:])  # (N, M, 2)

    wh   = (rb - lt).clamp(min=0)           # (N, M, 2)
    inter = wh[:, :, 0] * wh[:, :, 1]      # (N, M)
    union = area1[:, None] + area2[None, :] - inter

    return inter / (union + 1e-8)

# ── Test IoU ──────────────────────────────────────────────────────────
cases = [
    ([0, 0, 100, 100], [0, 0, 100, 100],  1.0,   "Perfect overlap"),
    ([0, 0, 100, 100], [50, 50, 150, 150], None,  "50% overlap"),
    ([0, 0, 100, 100], [200, 200, 300, 300], 0.0, "No overlap"),
    ([0, 0, 100, 100], [70, 30, 170, 120],  None, "Partial overlap"),
]

print("IoU calculations:")
for box1, box2, expected, desc in cases:
    iou = iou_corner_format(box1, box2)
    exp_str = f"(expected {expected})" if expected is not None else ""
    print(f"  {desc:<20}: IoU = {iou:.4f}  {exp_str}")

# ── Vectorised batch IoU ──────────────────────────────────────────────
pred_boxes = torch.tensor([[10, 20, 80, 90],
                             [50, 50, 150, 150],
                             [200, 200, 300, 280]], dtype=torch.float)
gt_boxes   = torch.tensor([[15, 25, 75, 85],
                             [60, 60, 140, 140]], dtype=torch.float)

iou_matrix = iou_batch(pred_boxes, gt_boxes)
print(f"
Batch IoU matrix (3 preds × 2 GT boxes):")
print(f"Shape: {iou_matrix.shape}")
print(iou_matrix.round(decimals=3))

import numpy as np
import torch

def nms(boxes: np.ndarray, scores: np.ndarray,
        iou_threshold: float = 0.45) -> list[int]:
    """
    Non-Maximum Suppression from scratch.
    boxes:  (N, 4) array of [x_min, y_min, x_max, y_max]
    scores: (N,) confidence scores
    Returns: list of kept box indices
    """
    x1, y1, x2, y2 = boxes[:,0], boxes[:,1], boxes[:,2], boxes[:,3]
    areas = (x2 - x1) * (y2 - y1)

    # Sort by score descending
    order = scores.argsort()[::-1]
    keep  = []

    while order.size > 0:
        i = order[0]
        keep.append(i)

        # Compute IoU of box i with all remaining boxes
        xx1 = np.maximum(x1[i], x1[order[1:]])
        yy1 = np.maximum(y1[i], y1[order[1:]])
        xx2 = np.minimum(x2[i], x2[order[1:]])
        yy2 = np.minimum(y2[i], y2[order[1:]])

        w    = np.maximum(0, xx2 - xx1)
        h    = np.maximum(0, yy2 - yy1)
        inter = w * h
        union = areas[i] + areas[order[1:]] - inter
        iou   = inter / (union + 1e-8)

        # Keep boxes with IoU below threshold
        remaining = np.where(iou <= iou_threshold)[0]
        order     = order[remaining + 1]

    return keep

# ── Simulate detection output before NMS ─────────────────────────────
# Three clusters — same car detected multiple times
np.random.seed(42)
boxes = np.array([
    [100, 100, 200, 180],  # car 1 — main box
    [102, 98,  198, 182],  # car 1 — duplicate (high overlap)
    [105, 103, 205, 185],  # car 1 — another duplicate
    [300, 200, 420, 310],  # car 2 — main box
    [298, 202, 418, 308],  # car 2 — duplicate
    [10,  10,  50,  50],   # pedestrian — isolated
], dtype=float)

scores = np.array([0.95, 0.87, 0.72, 0.91, 0.83, 0.78])

print(f"Before NMS: {len(boxes)} boxes")
for i, (box, score) in enumerate(zip(boxes, scores)):
    print(f"  Box {i}: score={score:.2f}  {box.astype(int).tolist()}")

kept = nms(boxes, scores, iou_threshold=0.45)
print(f"
After NMS (threshold=0.45): {len(kept)} boxes")
for i in kept:
    print(f"  Box {i}: score={scores[i]:.2f}  {boxes[i].astype(int).tolist()}")

# ── PyTorch built-in NMS ──────────────────────────────────────────────
from torchvision.ops import nms as torch_nms

boxes_t  = torch.FloatTensor(boxes)
scores_t = torch.FloatTensor(scores)
kept_t   = torch_nms(boxes_t, scores_t, iou_threshold=0.45)
print(f"
torchvision.ops.nms result: {kept_t.tolist()}  ← same as manual")

YOLO version history — what each version improved

# pip install ultralytics

from ultralytics import YOLO
import numpy as np
from PIL import Image
import torch

# ── Load pretrained YOLOv8 model ──────────────────────────────────────
# Model sizes: nano(n), small(s), medium(m), large(l), extra(x)
# Larger = more accurate, slower
model = YOLO('yolov8n.pt')   # nano — fastest, least accurate
# model = YOLO('yolov8s.pt') # small — good balance
# model = YOLO('yolov8m.pt') # medium — production default

print(f"Model: {model.model.__class__.__name__}")
print(f"Parameters: {sum(p.numel() for p in model.model.parameters()):,}")
print(f"Classes: {len(model.names)} (COCO 80-class)")
print(f"Sample classes: {list(model.names.values())[:10]}")

# ── Inference on an image ─────────────────────────────────────────────
# Create a test image (in production: pass real image path)
img_array = np.random.randint(50, 200, (640, 640, 3), dtype=np.uint8)
img_pil   = Image.fromarray(img_array)

results = model.predict(
    source=img_pil,
    conf=0.25,        # confidence threshold — discard boxes below this
    iou=0.45,         # NMS IoU threshold
    verbose=False,
)

result = results[0]
print(f"
Detection results:")
print(f"  Input size:   {result.orig_shape}")
print(f"  Boxes found:  {len(result.boxes)}")

if len(result.boxes) > 0:
    boxes  = result.boxes.xyxy.numpy()    # (N, 4) corner format
    scores = result.boxes.conf.numpy()    # (N,) confidence
    classes = result.boxes.cls.numpy()   # (N,) class indices

    for i, (box, score, cls) in enumerate(zip(boxes, scores, classes)):
        cls_name = model.names[int(cls)]
        print(f"  [{i+1}] {cls_name:<15} conf={score:.3f}  "
              f"box=[{box[0]:.0f},{box[1]:.0f},{box[2]:.0f},{box[3]:.0f}]")

# ── Fine-tuning on custom data ────────────────────────────────────────
print("""
# Fine-tuning YOLOv8 on your own dataset:

# 1. Prepare data in YOLO format:
#    dataset/
#      images/train/  image1.jpg  image2.jpg  ...
#      images/val/    image1.jpg  ...
#      labels/train/  image1.txt  image2.txt  ...
#      labels/val/    image1.txt  ...
#
# Each label file (one per image):
#    class_id  x_center  y_center  width  height
#    (all values normalised 0-1 relative to image size)
#    0  0.45  0.62  0.30  0.25
#    1  0.72  0.31  0.15  0.40

# 2. Dataset YAML file (dataset.yaml):
#    path: /path/to/dataset
#    train: images/train
#    val:   images/val
#    names:
#      0: defect
#      1: ok_product

# 3. Fine-tune:
model = YOLO('yolov8s.pt')
results = model.train(
    data='dataset.yaml',
    epochs=50,
    imgsz=640,
    batch=16,
    lr0=0.01,        # initial learning rate
    lrf=0.01,        # final lr = lr0 × lrf
    patience=10,     # early stopping patience
    save=True,
    project='meesho-defect-detection',
)

# 4. Evaluate on validation set:
metrics = model.val()
print(f"mAP50: {metrics.box.map50:.4f}")
print(f"mAP50-95: {metrics.box.map:.4f}")

# 5. Export for deployment:
model.export(format='onnx')    # ONNX for CPU deployment
model.export(format='tflite')  # TFLite for mobile
model.export(format='engine')  # TensorRT for NVIDIA GPU
""")

# ── Batch inference for video/stream ─────────────────────────────────
def process_batch(image_paths: list, model, conf: float = 0.25):
    """Process multiple images in one batch — faster than one at a time."""
    results = model.predict(image_paths, conf=conf, verbose=False)
    detections = []
    for result in results:
        frame_dets = []
        if len(result.boxes) > 0:
            for box, score, cls in zip(
                result.boxes.xyxy.numpy(),
                result.boxes.conf.numpy(),
                result.boxes.cls.numpy(),
            ):
                frame_dets.append({
                    'class':  model.names[int(cls)],
                    'score':  float(score),
                    'box':    box.tolist(),
                })
        detections.append(frame_dets)
    return detections

print("YOLOv8 batch inference: pass list of image paths for throughput")

import numpy as np

def compute_ap(recalls: np.ndarray, precisions: np.ndarray) -> float:
    """
    Compute Average Precision (AP) using the 11-point interpolation.
    Used in Pascal VOC evaluation.
    """
    ap = 0.0
    for threshold in np.arange(0, 1.1, 0.1):
        # Precision at recall >= threshold
        prec_at_thresh = precisions[recalls >= threshold]
        ap += prec_at_thresh.max() if prec_at_thresh.size > 0 else 0.0
    return ap / 11.0

def evaluate_detections(pred_boxes, pred_scores, pred_classes,
                         gt_boxes, gt_classes,
                         iou_threshold=0.5, num_classes=3):
    """
    Compute mAP for a set of predictions vs ground truth.
    All boxes in corner format [x1, y1, x2, y2].
    """
    aps = []

    for cls in range(num_classes):
        # Filter to current class
        pred_mask = pred_classes == cls
        gt_mask   = gt_classes   == cls

        p_boxes  = pred_boxes[pred_mask]
        p_scores = pred_scores[pred_mask]
        g_boxes  = gt_boxes[gt_mask]

        if len(g_boxes) == 0:
            continue
        if len(p_boxes) == 0:
            aps.append(0.0)
            continue

        # Sort predictions by score descending
        order  = p_scores.argsort()[::-1]
        p_boxes  = p_boxes[order]
        p_scores = p_scores[order]

        tp = np.zeros(len(p_boxes))
        fp = np.zeros(len(p_boxes))
        matched_gt = set()

        for i, pb in enumerate(p_boxes):
            if len(g_boxes) == 0:
                fp[i] = 1; continue

            # Compute IoU with all GT boxes
            ious = np.array([iou_corner(pb, gb) for gb in g_boxes])
            best_iou_idx = ious.argmax()
            best_iou     = ious[best_iou_idx]

            if best_iou >= iou_threshold and best_iou_idx not in matched_gt:
                tp[i] = 1
                matched_gt.add(best_iou_idx)
            else:
                fp[i] = 1

        cum_tp  = np.cumsum(tp)
        cum_fp  = np.cumsum(fp)
        recalls    = cum_tp / len(g_boxes)
        precisions = cum_tp / (cum_tp + cum_fp + 1e-8)

        aps.append(compute_ap(recalls, precisions))

    return np.mean(aps) if aps else 0.0

def iou_corner(b1, b2):
    x1 = max(b1[0], b2[0]); y1 = max(b1[1], b2[1])
    x2 = min(b1[2], b2[2]); y2 = min(b1[3], b2[3])
    inter = max(0, x2-x1) * max(0, y2-y1)
    a1 = (b1[2]-b1[0])*(b1[3]-b1[1])
    a2 = (b2[2]-b2[0])*(b2[3]-b2[1])
    return inter / (a1 + a2 - inter + 1e-8)

# ── Simulate evaluation ───────────────────────────────────────────────
np.random.seed(42)
n_pred, n_gt = 20, 10

pred_boxes   = np.random.randint(0, 400, (n_pred, 4)).astype(float)
pred_boxes[:, 2:] += pred_boxes[:, :2] + 50   # ensure x2>x1
pred_scores  = np.random.uniform(0.3, 0.99, n_pred)
pred_classes = np.random.randint(0, 3, n_pred)

gt_boxes   = np.random.randint(0, 400, (n_gt, 4)).astype(float)
gt_boxes[:, 2:] += gt_boxes[:, :2] + 50
gt_classes = np.random.randint(0, 3, n_gt)

map50 = evaluate_detections(
    pred_boxes, pred_scores, pred_classes,
    gt_boxes, gt_classes, iou_threshold=0.5,
)
print(f"mAP@0.5: {map50:.4f}")

# mAP@0.5:0.95 (COCO standard)
thresholds = np.arange(0.5, 1.0, 0.05)
map_coco = np.mean([
    evaluate_detections(pred_boxes, pred_scores, pred_classes,
                         gt_boxes, gt_classes, iou_threshold=t)
    for t in thresholds
])
print(f"mAP@0.5:0.95: {map_coco:.4f}  ← COCO standard (harder)")

# A simplified version of the decision most detection teams actually make:
# given a latency budget and an accuracy floor, which model size fits?

model_options = [
    # name,         params_M, latency_ms_edge, mAP50_95, deploy_target
    ('yolov8n', 3.2,   8,  37.3, 'edge / embedded'),
    ('yolov8s', 11.2, 15,  44.9, 'edge GPU / Jetson'),
    ('yolov8m', 25.9, 30,  50.2, 'cloud GPU, moderate latency'),
    ('yolov8l', 43.7, 48,  52.9, 'cloud GPU, batch or offline'),
    ('yolov8x', 68.2, 70,  53.9, 'offline / max accuracy'),
]

def pick_model(latency_budget_ms: float, min_map: float):
    """
    Return every model that fits within both constraints, cheapest first.
    This is the actual shape of the decision — not "which model is best"
    but "which is the cheapest model that clears the bar we need."
    """
    candidates = [
        m for m in model_options
        if m[2] <= latency_budget_ms and m[3] >= min_map
    ]
    return sorted(candidates, key=lambda m: m[1])   # cheapest (fewest params) first

print("Retail shelf monitor — edge hardware, latency budget 20ms, mAP floor 35:")
for m in pick_model(latency_budget_ms=20, min_map=35.0):
    print(f"  {m[0]:<10} {m[1]:>6.1f}M params  {m[2]:>4}ms  mAP50-95={m[3]}  ({m[4]})")

print("
Quality inspection — belt speed allows 100ms, mAP floor 50:")
for m in pick_model(latency_budget_ms=100, min_map=50.0):
    print(f"  {m[0]:<10} {m[1]:>6.1f}M params  {m[2]:>4}ms  mAP50-95={m[3]}  ({m[4]})")

# Note: the "right" model is rarely the most accurate one available —
# it is the cheapest model that still clears the accuracy and latency
# bar the actual use case sets.

Classification always outputs exactly one label for a fixed-size input. Detection has to solve a problem classification never faces: the number of objects in an image is unknown and variable — zero, one, or forty — and each one sits at an unknown location and scale. That is why detection needed its own architectural ideas (a grid of simultaneous predictors, anchor boxes, NMS to clean up duplicates) rather than just classification with extra output neurons. Sliding a classifier over every possible window and scale would technically work, but it is combinatorially too slow — YOLO's single-pass grid exists specifically to avoid that search.

The IoU threshold is an evaluation choice, not a property of the model — it decides how strictly a predicted box must overlap the ground truth to count as correct. Raising it from 0.5 to 0.9 does not make a model better; it makes the bar for "correct" harder to clear, so the same model reports a lower mAP. This is exactly why COCO's mAP@0.5:0.95 exists — it is deliberately more demanding than Pascal VOC's mAP@0.5 — and why comparing two models' mAP numbers is meaningless unless they were evaluated at the same threshold(s).

"Faster" was never the only axis. Two-stage detectors first propose candidate regions then classify each one — more compute, but historically better localisation on small or densely packed objects because the region proposal step gets a second dedicated look at each candidate. One-stage detectors predict everything in a single forward pass, trading some of that per-region refinement for speed. Modern one-stage models (YOLOv8 and later) have closed most of the accuracy gap through multi-scale detection and better training tricks, but the underlying tradeoff — a dedicated proposal-refinement stage versus a single unified pass — is still real and still shows up on small-object-heavy datasets.

A lower threshold suppresses more aggressively, but "more aggressive" is not free — NMS cannot tell the difference between two boxes on the same object and two boxes on two different objects that happen to be standing close together. Push the threshold too low (say, 0.2) in a crowd scene or a shelf of tightly packed products, and legitimate neighbouring objects get merged into one detection. The right threshold is a tuned tradeoff against your specific data's object density, not a knob to minimise.

Anchor boxes are starting references, not a fixed catalog the model must pick from verbatim. At each grid location the model predicts *offsets* — how much to shift and resize a given anchor to match the actual object — so a tall anchor can still stretch to cover a wide object if the regression pushes it there. Anchors exist to give the regression a sane starting point (matched to typical object shapes in the training data via clustering) so training converges faster, not to hard-constrain what shapes are detectable. Anchor-free detectors like YOLOv8 drop this mechanism entirely and predict box dimensions directly, which is a separate design choice, not proof that anchors were ever a hard constraint.

The image is divided into an S×S grid and passed through the backbone in one forward pass. Each grid cell predicts a fixed number of bounding boxes (as offsets from anchor boxes or, in anchor-free versions, directly), a confidence score for each box, and class probabilities. The cell "responsible" for an object is the one containing that object's centre. This produces hundreds of raw candidate boxes, most low-confidence or duplicates. Boxes below a confidence threshold (e.g. 0.25) are discarded, then Non-Maximum Suppression removes duplicate boxes on the same object, keeping only the highest-confidence box per cluster. What remains is the final detection list — boxes, classes, and scores.

IoU is intersection area divided by union area between a predicted box and the ground truth box — it quantifies how well two boxes overlap, from 0 (no overlap) to 1 (perfect match). It matters for evaluation because it is the criterion that decides whether a prediction counts as a True Positive: IoU ≥ threshold means correct, below it means the prediction is a False Positive even if the class label was right. The threshold you pick changes the reported score without changing the model — Pascal VOC's mAP@0.5 is lenient, COCO's mAP@0.5:0.95 averages across ten thresholds from 0.5 to 0.95 and is much harder to score well on, which is why COCO numbers are always lower than VOC-style numbers for the same model.

A detector typically produces many overlapping candidate boxes around the same real object — without cleanup you would report the same car ten times. NMS removes the duplicates: sort all boxes by confidence score, take the highest-scoring box and add it to the final output, then remove every remaining box whose IoU with that box exceeds a threshold (commonly 0.45). Repeat with whatever boxes are left until none remain. The IoU threshold controls aggressiveness — too low and it merges genuinely distinct nearby objects, too high and duplicate boxes survive.

Two-stage detectors (Faster R-CNN) first generate candidate regions likely to contain an object, then run a second network to classify and refine each region — more compute per image but historically stronger localisation, especially on small or crowded objects, because each candidate gets dedicated attention. One-stage detectors (YOLO, SSD) skip the proposal step and predict boxes and classes directly from the full image in one pass — much faster, which is why YOLO is the standard choice for real-time video at 30+ fps. I'd pick two-stage when accuracy on small/dense objects matters more than latency (offline batch analysis of satellite imagery, for example) and one-stage for anything real-time (live camera feeds, robotics, mobile deployment).

First suspect a distribution mismatch between validation and production data — training images are often cleaner (studio lighting, no motion blur) than what the camera actually captures in the field, so I'd pull real production frames and manually inspect detections on them. Second, check whether the confidence threshold used to compute mAP (often very low, like 0.001, to build the full precision-recall curve) matches the threshold actually used at inference time (often 0.5) — a high mAP built at a permissive threshold does not guarantee good precision at the stricter threshold you deploy with. Third, verify the NMS and confidence settings are identical between evaluation and the production inference code — a silent mismatch there is a common and easy-to-miss bug.