Image Fundamentals — Pixels, Channels and Tensors

Think of a crossword puzzle grid. Each cell has a letter. The grid has rows, columns, and one layer of content. An image is like three crossword grids stacked on top of each other — one grid for red intensity, one for green, one for blue. Each cell has a number 0–255 instead of a letter. Read all three grids together and you reconstruct the colour.

PyTorch uses channels-first format: (channels, height, width). Pillow and OpenCV use channels-last: (height, width, channels). Mixing these two formats silently produces garbage predictions — this is the single most common computer vision bug.

import numpy as np
from PIL import Image
import torch

# ── Load an image and inspect its representation ──────────────────────
# Create a synthetic image (in practice: Image.open('product.jpg'))
img_array = np.random.randint(0, 256, (224, 224, 3), dtype=np.uint8)
img_pil   = Image.fromarray(img_array)

print("PIL Image:")
print(f"  Mode:   {img_pil.mode}         ← RGB, RGBA, L (greyscale), etc.")
print(f"  Size:   {img_pil.size}     ← (width, height) — note: width first!")
print(f"  Format: {img_pil.format}       ← None for in-memory, 'JPEG'/'PNG' for files")

# ── PIL → NumPy ───────────────────────────────────────────────────────
img_np = np.array(img_pil)
print(f"
NumPy array (PIL default — channels last):")
print(f"  Shape:  {img_np.shape}    ← (height, width, channels)")
print(f"  Dtype:  {img_np.dtype}          ← uint8, values 0–255")
print(f"  Min:    {img_np.min()}  Max: {img_np.max()}")

# Access individual channels
red_channel   = img_np[:, :, 0]   # shape (224, 224)
green_channel = img_np[:, :, 1]
blue_channel  = img_np[:, :, 2]
print(f"  Red channel mean:   {red_channel.mean():.1f}")
print(f"  Green channel mean: {green_channel.mean():.1f}")
print(f"  Blue channel mean:  {blue_channel.mean():.1f}")

# ── NumPy → PyTorch tensor ─────────────────────────────────────────────
# PyTorch expects (channels, height, width) — must transpose
img_torch = torch.from_numpy(img_np).permute(2, 0, 1)  # HWC → CHW
print(f"
PyTorch tensor (channels first):")
print(f"  Shape:  {tuple(img_torch.shape)}    ← (channels, height, width)")
print(f"  Dtype:  {img_torch.dtype}       ← torch.uint8, values 0–255")

# ── Convert to float and normalise to [0, 1] ──────────────────────────
img_float = img_torch.float() / 255.0
print(f"
After /255 normalisation:")
print(f"  Dtype:  {img_float.dtype}")
print(f"  Min:    {img_float.min():.4f}  Max: {img_float.max():.4f}")

# ── torchvision ToTensor — does both steps at once ────────────────────
import torchvision.transforms as T
to_tensor  = T.ToTensor()   # PIL (HWC uint8) → torch (CHW float32 0–1)
img_tensor = to_tensor(img_pil)
print(f"
torchvision.ToTensor output:")
print(f"  Shape:  {tuple(img_tensor.shape)}")
print(f"  Dtype:  {img_tensor.dtype}")
print(f"  Range:  [{img_tensor.min():.3f}, {img_tensor.max():.3f}]")

# ── Common image stats: how many bytes? ──────────────────────────────
h, w, c = 224, 224, 3
print(f"
Memory for 224×224 RGB:")
print(f"  uint8  (0-255):    {h*w*c:,} bytes  = {h*w*c/1024:.1f} KB")
print(f"  float32 (0-1):     {h*w*c*4:,} bytes = {h*w*c*4/1024:.1f} KB")
print(f"  Batch of 32 float: {h*w*c*4*32/1024/1024:.1f} MB")

import numpy as np
from PIL import Image
import torch
import torchvision.transforms as T

# ── Colour space conversions ──────────────────────────────────────────
img_np = np.random.randint(0, 256, (64, 64, 3), dtype=np.uint8)
img_pil = Image.fromarray(img_np)

# ── Greyscale — single channel ────────────────────────────────────────
img_grey  = img_pil.convert('L')   # L = luminance
grey_np   = np.array(img_grey)
print(f"RGB:       {img_np.shape}   range: {img_np.min()}–{img_np.max()}")
print(f"Greyscale: {grey_np.shape}     range: {grey_np.min()}–{grey_np.max()}")
# Greyscale formula: 0.299×R + 0.587×G + 0.114×B (human luminance perception)

# ── PyTorch greyscale ─────────────────────────────────────────────────
grey_transform = T.Grayscale(num_output_channels=1)
grey_tensor    = grey_transform(T.ToTensor()(img_pil))
print(f"PyTorch greyscale: {tuple(grey_tensor.shape)}")

grey_to_3ch    = T.Grayscale(num_output_channels=3)  # keep 3 channels (some models need it)
grey3_tensor   = grey_to_3ch(T.ToTensor()(img_pil))
print(f"Grey as 3-channel: {tuple(grey3_tensor.shape)}")

# ── OpenCV colour conversions (if available) ──────────────────────────
try:
    import cv2

    img_bgr = cv2.cvtColor(img_np, cv2.COLOR_RGB2BGR)  # PIL is RGB, OpenCV is BGR!
    img_hsv = cv2.cvtColor(img_bgr, cv2.COLOR_BGR2HSV)
    img_lab = cv2.cvtColor(img_bgr, cv2.COLOR_BGR2LAB)

    print(f"
OpenCV colour spaces (all 64×64):")
    print(f"  BGR:  {img_bgr.shape}  H={img_bgr[...,0].mean():.0f} S={img_bgr[...,1].mean():.0f} V={img_bgr[...,2].mean():.0f}")
    print(f"  HSV:  {img_hsv.shape}  H range: {img_hsv[...,0].min()}–{img_hsv[...,0].max()}")
    print(f"  LAB:  {img_lab.shape}")

    # Critical: OpenCV uses BGR, PIL uses RGB
    # Always convert when mixing PIL and OpenCV
    print(f"
⚠ OpenCV loads as BGR. Always convert:")
    print(f"   cv2.imread() → BGR")
    print(f"   cv2.cvtColor(img, cv2.COLOR_BGR2RGB) → RGB (for PIL/PyTorch)")

except ImportError:
    print("
OpenCV not installed: pip install opencv-python")

# ── When to use which colour space ───────────────────────────────────
print("
Colour space selection guide:")
guide = [
    ('RGB',       'Default for all deep learning models'),
    ('Greyscale', 'Documents, X-rays, fingerprints — when colour carries no info'),
    ('HSV',       'Rule-based colour detection (find red objects in any lighting)'),
    ('LAB',       'Medical imaging, colour correction, perceptual distance metrics'),
    ('YCrCb',     'Video compression, face detection under varied lighting'),
]
for space, use in guide:
    print(f"  {space:<12}: {use}")

import torch
import torchvision.transforms as T
import numpy as np
from PIL import Image

# ── Create test image with known aspect ratio ─────────────────────────
# Simulate a portrait-mode product photo (tall image)
img = Image.fromarray(np.random.randint(0, 256, (400, 200, 3), dtype=np.uint8))
print(f"Original: {img.size} (W×H)  →  array shape: {np.array(img).shape}")

# ── Strategy 1: Direct resize (distorts) ─────────────────────────────
t1 = T.Resize((224, 224))
out1 = t1(img)
print(f"
Direct resize 224×224: {np.array(out1).shape}  ← squashed!")

# ── Strategy 2: Resize shorter side + centre crop (standard for eval) ─
t2 = T.Compose([T.Resize(256), T.CenterCrop(224)])
out2 = t2(img)
print(f"Resize+CenterCrop:     {np.array(out2).shape}  ← no distortion, crops edges")

# ── Strategy 3: RandomResizedCrop (standard for training) ─────────────
t3 = T.RandomResizedCrop(
    size=224,
    scale=(0.08, 1.0),    # crop between 8% and 100% of image area
    ratio=(0.75, 1.33),   # aspect ratio range
)
out3 = t3(img)
print(f"RandomResizedCrop:     {np.array(out3).shape}  ← random region, different each call")

# ── Strategy 4: Resize + pad to square ───────────────────────────────
def resize_and_pad(img: Image.Image, target: int = 224,
                    fill: int = 0) -> Image.Image:
    """Resize longest side to target, pad shorter side to make square."""
    w, h   = img.size
    ratio  = target / max(w, h)
    new_w  = int(w * ratio)
    new_h  = int(h * ratio)
    img_r  = img.resize((new_w, new_h), Image.BILINEAR)

    # Pad to square
    pad_w  = target - new_w
    pad_h  = target - new_h
    left   = pad_w // 2
    top    = pad_h // 2
    result = Image.new('RGB', (target, target), (fill, fill, fill))
    result.paste(img_r, (left, top))
    return result

out4 = resize_and_pad(img, 224)
print(f"Resize+Pad:            {np.array(out4).shape}  ← full image preserved")

# ── Memory and batch implications ─────────────────────────────────────
batch_size = 32
for h, w in [(224, 224), (384, 384), (512, 512)]:
    mb = h * w * 3 * 4 * batch_size / 1024 / 1024
    print(f"
Batch={batch_size}, size={h}×{w}: {mb:.1f} MB in float32")

import torch
import torchvision.transforms as T
import numpy as np
from PIL import Image

# ── ImageNet normalisation — standard for pretrained models ──────────
IMAGENET_MEAN = [0.485, 0.456, 0.406]
IMAGENET_STD  = [0.229, 0.224, 0.225]

normalize = T.Normalize(mean=IMAGENET_MEAN, std=IMAGENET_STD)

img_pil    = Image.fromarray(np.random.randint(0, 256, (224, 224, 3), dtype=np.uint8))
img_tensor = T.ToTensor()(img_pil)   # 0–1 float

print("Before normalisation:")
print(f"  Mean per channel: {img_tensor.mean(dim=[1,2]).tolist()}")
print(f"  Std  per channel: {img_tensor.std(dim=[1,2]).tolist()}")
print(f"  Range: [{img_tensor.min():.3f}, {img_tensor.max():.3f}]")

img_norm = normalize(img_tensor)
print(f"
After ImageNet normalisation:")
print(f"  Mean per channel: {[round(v,3) for v in img_norm.mean(dim=[1,2]).tolist()]}")
print(f"  Std  per channel: {[round(v,3) for v in img_norm.std(dim=[1,2]).tolist()]}")
print(f"  Range: [{img_norm.min():.3f}, {img_norm.max():.3f}]  ← can go negative!")

# ── Standard training pipeline ────────────────────────────────────────
train_transform = T.Compose([
    T.RandomResizedCrop(224),
    T.RandomHorizontalFlip(),
    T.ColorJitter(brightness=0.2, contrast=0.2, saturation=0.2),
    T.ToTensor(),
    T.Normalize(mean=IMAGENET_MEAN, std=IMAGENET_STD),
])

val_transform = T.Compose([
    T.Resize(256),
    T.CenterCrop(224),
    T.ToTensor(),
    T.Normalize(mean=IMAGENET_MEAN, std=IMAGENET_STD),
])

train_tensor = train_transform(img_pil)
val_tensor   = val_transform(img_pil)
print(f"
Training transform output:   {tuple(train_tensor.shape)}")
print(f"Validation transform output: {tuple(val_tensor.shape)}")

# ── Computing custom normalisation stats for your dataset ─────────────
def compute_dataset_stats(image_list: list) -> tuple:
    """
    Compute mean and std over a list of PIL images.
    In production: iterate over your full training DataLoader.
    """
    all_pixels = []
    for img in image_list:
        t = T.ToTensor()(img)   # (3, H, W) float 0–1
        all_pixels.append(t.view(3, -1))  # (3, H*W)

    all_pixels = torch.cat(all_pixels, dim=1)   # (3, N_pixels)
    mean = all_pixels.mean(dim=1).tolist()
    std  = all_pixels.std(dim=1).tolist()
    return mean, std

# Simulate 10 product images
fake_images = [
    Image.fromarray(np.random.randint(50, 200, (64, 64, 3), dtype=np.uint8))
    for _ in range(10)
]
custom_mean, custom_std = compute_dataset_stats(fake_images)
print(f"
Custom dataset stats:")
print(f"  Mean: {[round(v, 4) for v in custom_mean]}")
print(f"  Std:  {[round(v, 4) for v in custom_std]}")
print(f"  (Compare to ImageNet mean: {IMAGENET_MEAN})")
print(f"  If very different → use custom stats, not ImageNet")

import torch
import torchvision.transforms as T
import torchvision.transforms.functional as TF
from torch.utils.data import Dataset, DataLoader
from PIL import Image
import numpy as np
import os

# ── Complete Dataset class with preprocessing ─────────────────────────
class ProductImageDataset(Dataset):
    """
    Shopify product image dataset.
    Directory structure:
      data/
        kurta/    image1.jpg  image2.jpg  ...
        saree/    image1.jpg  ...
        jeans/    ...
    """
    CATEGORIES = ['kurta', 'saree', 'jeans', 'sneakers', 'watch', 'handbag']
    IMAGENET_MEAN = [0.485, 0.456, 0.406]
    IMAGENET_STD  = [0.229, 0.224, 0.225]

    def __init__(self, root_dir: str, split: str = 'train',
                  image_size: int = 224):
        self.split      = split
        self.image_size = image_size

        # Build file list (in production: scan directory)
        # Here: generate synthetic data
        self.samples = []
        for label, category in enumerate(self.CATEGORIES):
            for i in range(20):
                self.samples.append((f"{root_dir}/{category}/{i}.jpg", label))

        # Define transforms
        if split == 'train':
            self.transform = T.Compose([
                T.RandomResizedCrop(image_size, scale=(0.7, 1.0)),
                T.RandomHorizontalFlip(p=0.5),
                T.ColorJitter(brightness=0.3, contrast=0.3,
                               saturation=0.2, hue=0.1),
                T.RandomGrayscale(p=0.05),    # rare greyscale augmentation
                T.ToTensor(),
                T.Normalize(mean=self.IMAGENET_MEAN, std=self.IMAGENET_STD),
            ])
        else:
            self.transform = T.Compose([
                T.Resize(int(image_size * 256 / 224)),   # resize to 256 if target 224
                T.CenterCrop(image_size),
                T.ToTensor(),
                T.Normalize(mean=self.IMAGENET_MEAN, std=self.IMAGENET_STD),
            ])

    def __len__(self):
        return len(self.samples)

    def __getitem__(self, idx: int):
        path, label = self.samples[idx]

        # In production: img = Image.open(path).convert('RGB')
        # Simulate with random image
        img = Image.fromarray(
            np.random.randint(0, 256, (300, 200, 3), dtype=np.uint8)
        )
        img = self.transform(img)
        return img, label

# ── Test the pipeline ─────────────────────────────────────────────────
train_ds = ProductImageDataset('/data/meesho', split='train')
val_ds   = ProductImageDataset('/data/meesho', split='val')

train_loader = DataLoader(
    train_ds, batch_size=32,
    shuffle=True,
    num_workers=0,         # 4 in production for parallel loading
    pin_memory=False,      # True if using GPU
    drop_last=True,
)
val_loader = DataLoader(
    val_ds, batch_size=64,
    shuffle=False,
    num_workers=0,
)

batch_imgs, batch_labels = next(iter(train_loader))
print(f"Batch shape:  {tuple(batch_imgs.shape)}")
print(f"Labels shape: {tuple(batch_labels.shape)}")
print(f"Value range:  [{batch_imgs.min():.3f}, {batch_imgs.max():.3f}]")
print(f"Dtype:        {batch_imgs.dtype}")

# ── Denormalise for visualisation ─────────────────────────────────────
def denormalise(tensor: torch.Tensor,
                mean = [0.485, 0.456, 0.406],
                std  = [0.229, 0.224, 0.225]) -> np.ndarray:
    """Convert normalised tensor back to uint8 for display."""
    m = torch.tensor(mean).view(3, 1, 1)
    s = torch.tensor(std).view(3, 1, 1)
    img = tensor * s + m          # undo normalisation
    img = img.clamp(0, 1)         # clip to [0, 1]
    img = (img * 255).byte()      # float32 → uint8
    return img.permute(1, 2, 0).numpy()  # CHW → HWC for display

sample = batch_imgs[0]
sample_display = denormalise(sample)
print(f"
Denormalised for display: {sample_display.shape}  dtype={sample_display.dtype}")

The fix is not "be more careful" — it is a concrete, automatable test: run the exact same input image through the training-side preprocessing function and the serving-side preprocessing function, and assert the resulting tensors match within a tight numerical tolerance. This is normally added as a check in CI for the serving code, run against a small fixed set of real sample images every time either pipeline changes, precisely because a rotation, a channel swap, or a wrong normalisation constant all pass every existing test that only checks tensor shape and dtype.

import numpy as np
from PIL import Image, ImageOps
import torchvision.transforms as T

IMAGENET_MEAN = [0.485, 0.456, 0.406]
IMAGENET_STD  = [0.229, 0.224, 0.225]

def training_preprocess(path: str):
    """
    The pipeline the ML team actually trains with.
    PIL respects EXIF orientation automatically via exif_transpose.
    """
    img = Image.open(path)
    img = ImageOps.exif_transpose(img)     # apply the camera's rotation tag
    img = img.convert('RGB')
    transform = T.Compose([
        T.Resize(256), T.CenterCrop(224), T.ToTensor(),
        T.Normalize(mean=IMAGENET_MEAN, std=IMAGENET_STD),
    ])
    return transform(img)

def serving_preprocess_buggy(path: str):
    """
    A simplified stand-in for a re-implemented serving-side decoder that
    forgets the EXIF step — this is the realistic failure mode, not a
    contrived one. Everything else about it looks correct.
    """
    img = Image.open(path)               # no exif_transpose call
    img = img.convert('RGB')
    transform = T.Compose([
        T.Resize(256), T.CenterCrop(224), T.ToTensor(),
        T.Normalize(mean=IMAGENET_MEAN, std=IMAGENET_STD),
    ])
    return transform(img)

def preprocessing_parity_check(path: str, tolerance: float = 1e-3) -> bool:
    """
    Run both pipelines on the same real image, compare the output tensors.
    This is the test that belongs in CI — not a visual inspection, an
    automated numerical assertion that runs on every deploy.
    """
    train_tensor  = training_preprocess(path)
    serve_tensor  = serving_preprocess_buggy(path)

    max_diff = (train_tensor - serve_tensor).abs().max().item()
    matches  = max_diff < tolerance

    print(f"Max pixel-value difference: {max_diff:.4f}")
    print(f"Parity check: {'PASS' if matches else 'FAIL — pipelines disagree'}")
    return matches

# In CI: run this against a small fixed set of real photos, including at
# least one taken with the phone held sideways or upside down — the exact
# case that a shape-and-dtype-only test will never catch.
# preprocessing_parity_check('sample_images/sideways_phone_photo.jpg')

RGB is simply the encoding a camera sensor and most display hardware happen to use — it is not a more fundamental representation of the underlying scene than HSV or LAB. Each colour space packages the same information differently for a different purpose: HSV separates hue from brightness so you can find "red objects" regardless of lighting, LAB separates luminance from colour and is perceptually uniform for medical imaging and colour correction, greyscale discards colour entirely because tasks like document OCR or X-ray analysis do not need it. Choosing RGB by default is correct for most deep learning models because that is what they were trained on — not because RGB is a more truthful representation of the image.

Shape is only one of three things a model expects to match: shape, dtype, and value distribution. A (3, 224, 224) float32 tensor with values in [0, 1] has the correct shape but the wrong distribution for any ImageNet-pretrained model, which expects roughly zero mean and unit variance per channel. As this module's errors section shows, skipping normalisation produces a model that predicts the same class for every input — not a crash, which would be easy to catch, but silent, confident, wrong output. Resize gets the geometry right; normalisation gets the statistics right, and a pretrained model needs both.

Both orderings describe the identical pixel data — the danger is exactly that they are interchangeable in shape but not in meaning. A (3, 224, 224) tensor and a (224, 224, 3) array can both pass a "is this a valid input shape" check while representing completely different things: three 224×224 channel planes versus 224 rows of 224 pixels with 3 values each. Feed one where the other is expected and most code does not error — it silently reinterprets 3 rows of a 224-wide image as 3 colour channels, or vice versa, producing garbled input that still has a valid tensor shape. This is why the module calls channel-order mismatches the single most common computer vision bug: it fails silently rather than loudly.

A pretrained model's first layer only checks tensor shape — it has no way to verify that your resize strategy, normalisation statistics, or train/validation transform split match what the model was actually trained on. This module's errors section covers a real case: applying RandomResizedCrop to a validation set instead of a deterministic Resize plus CenterCrop still produces a correctly-shaped tensor, but validation accuracy becomes inconsistent between runs because the input distribution silently changed. Correct shape is necessary but nowhere near sufficient — the preprocessing pipeline has to reproduce the exact statistics the model expects, not just its input dimensions.

The 2D-plus-channels structure is not cosmetic — it is the specific property that lets convolutional layers work at all. A convolution slides the same small filter across every spatial location, detecting a feature like an edge or a curve regardless of where it appears in the image; this only makes sense because neighbouring entries in the (height, width) grid are actually neighbouring pixels in the real image. Flatten that same image into a 1D vector of 150,528 numbers and a convolutional filter has nothing meaningful to slide across — position information collapses, and two pixels that were adjacent in the image can end up far apart in the vector. The array shape covered in this module is not bookkeeping; it is the substrate that gives a CNN's translation invariance its meaning.

PyTorch's convolution kernels are implemented to expect the channel dimension first for memory-layout and performance reasons on GPU; PIL and OpenCV inherited channels-last from how image file formats and most C image libraries store pixel data, row by row with interleaved channel values. Neither is more "correct" — they are different conventions for the same information. Mixing them is dangerous specifically because a transposed shape like (224, 224, 3) can be reinterpreted as (224 channels, 224 height, 3 width) by code that only validates tensor rank, not axis meaning — it doesn't crash, it silently treats 224 rows as 224 channels. The fix is always an explicit .permute(2, 0, 1) or .transpose when converting between the two, and checking shape assumptions by name, not just by count.

The pretrained weights, especially in the first few layers, were learned assuming inputs with roughly zero mean and unit standard deviation per channel — that is what T.Normalize with the ImageNet statistics produces. An image left at [0, 1] has a mean around 0.5 and a much smaller spread, so it lands in a different region of the input space than the one the first convolutional filters were tuned to respond to. The activations that follow are not necessarily huge or NaN — often they are just systematically off — and because the shift compounds through every subsequent layer, the final predictions collapse toward whichever class the model defaults to, which is why the practical symptom is "the model predicts the same class for every image" rather than an obvious crash.

A fully connected layer treats every pixel as an independent input feature: it learns a separate weight connecting each pixel to each output neuron, with no assumption that pixel (10, 10) and pixel (10, 11) are spatially related. A convolutional layer instead learns one small filter — say 3×3 — and reuses those same few weights at every position in the image, which is only meaningful because the image is stored as a spatial grid where adjacent array entries really are adjacent pixels. That weight sharing is what gives CNNs translation invariance: a filter that learns to detect an edge detects that edge wherever it appears, using orders of magnitude fewer parameters than a fully connected layer would need to learn the same pattern separately at every location.

Normalisation statistics define the input distribution the model's weights were optimised against; changing them between training and inference — or between training and validation — shifts every activation downstream in a way the model was never trained to compensate for, even though nothing about the code raises an error. Use ImageNet's mean and std whenever you are fine-tuning an ImageNet-pretrained model, since that is the distribution its weights already expect. Compute your own statistics over your training set when the domain is different enough that ImageNet's numbers no longer describe your data well — medical scans, satellite imagery, or product photos shot under unusual lighting are the common cases where recomputing mean and std measurably helps.

First, pick the colour space — RGB by default, unless the task is colour-irrelevant (documents, X-rays) where greyscale saves memory and compute, or colour-based detection where HSV is more robust to lighting. Second, pick a resize strategy matched to the use case — deterministic Resize plus CenterCrop for validation and inference, so metrics are reproducible, versus RandomResizedCrop for training, so the model sees the object at varying scales and positions. Third, always convert to RGB explicitly on load to handle RGBA PNGs and greyscale images consistently. Fourth, normalise with statistics matched to the model — ImageNet stats for a pretrained backbone, custom-computed stats for a sufficiently different domain. And finally, keep the training and validation transform pipelines separate, differing only in which steps are random, so a train/validation mismatch never becomes the reason accuracy numbers cannot be trusted.