BERT and the Encoder-Only Family

Reading comprehension in school: you read the full passage, then answer questions about it. You read forwards and backwards, checking context in both directions. A student who only reads left to right and never re-reads misses nuance. BERT is the student who reads the full passage before answering. GPT is the student writing an essay — they cannot read what they have not written yet.

This is why BERT dominates understanding tasks (classification, NER, Q&A) while GPT dominates generation tasks (completion, summarisation, chat). Same Transformer architecture, different direction of attention, completely different use cases.

Note: Later research (RoBERTa, 2019) showed NSP does not help and may hurt. RoBERTa removed it entirely and achieved better results training with MLM only on more data for longer.

# ── Demonstrate MLM manually — what BERT is trained to do ───────────
# In production BERT is pretrained for you — this shows the concept

from transformers import AutoTokenizer, AutoModelForMaskedLM
import torch

# Load BERT tokeniser and MLM model
tokenizer = AutoTokenizer.from_pretrained('bert-base-uncased')
model     = AutoModelForMaskedLM.from_pretrained('bert-base-uncased')
model.eval()

# ── Task: fill in the [MASK] ──────────────────────────────────────────
sentences = [
    "Razorpay [MASK] my payment successfully.",
    "The [MASK] was declined due to insufficient funds.",
    "Please [MASK] your transaction after some time.",
]

print("BERT Masked Language Modelling predictions:")
for sentence in sentences:
    inputs = tokenizer(sentence, return_tensors='pt')
    mask_idx = (inputs['input_ids'] == tokenizer.mask_token_id).nonzero(as_tuple=True)[1]

    with torch.no_grad():
        logits = model(**inputs).logits

    # Top 5 predictions for the masked token
    mask_logits = logits[0, mask_idx, :]
    top5 = torch.topk(mask_logits, 5, dim=-1)
    top5_tokens = tokenizer.convert_ids_to_tokens(top5.indices[0])
    top5_probs  = torch.softmax(top5.values[0], dim=0)

    print(f"
  '{sentence}'")
    for tok, prob in zip(top5_tokens, top5_probs):
        bar = '█' * int(prob.item() * 30)
        print(f"    {tok:<15} {bar:<30} {prob.item():.3f}")

from transformers import AutoTokenizer
import torch

tokenizer = AutoTokenizer.from_pretrained('bert-base-uncased')

# ── BERT tokenisation — what actually goes into the model ─────────────
text_a = "Razorpay payment was declined"
text_b = "Please retry after some time"

# Single sentence
single = tokenizer(text_a, return_tensors='pt')
print("Single sentence encoding:")
print(f"  input_ids:      {single['input_ids'][0].tolist()}")
print(f"  token_type_ids: {single['token_type_ids'][0].tolist()}  ← all 0 (sentence A)")
print(f"  attention_mask: {single['attention_mask'][0].tolist()}")
print(f"  tokens:         {tokenizer.convert_ids_to_tokens(single['input_ids'][0])}")

# Two sentences (for NSP, Q&A, sentence pair tasks)
pair = tokenizer(text_a, text_b, return_tensors='pt')
print(f"
Sentence pair encoding:")
print(f"  tokens:         {tokenizer.convert_ids_to_tokens(pair['input_ids'][0])}")
print(f"  token_type_ids: {pair['token_type_ids'][0].tolist()}")
print(f"  ← 0s = sentence A, 1s = sentence B")

# ── Batch encoding with padding ───────────────────────────────────────
batch_texts = [
    "payment declined",
    "transaction successful confirmation received",
    "upi failed",
]
batch = tokenizer(batch_texts, padding=True, truncation=True,
                  max_length=32, return_tensors='pt')
print(f"
Batch encoding (padded to same length):")
print(f"  input_ids shape:      {batch['input_ids'].shape}")
print(f"  attention_mask shape: {batch['attention_mask'].shape}")
print(f"  Padding token ID:     {tokenizer.pad_token_id}")
print(f"  [CLS] token ID:       {tokenizer.cls_token_id}")
print(f"  [SEP] token ID:       {tokenizer.sep_token_id}")
print(f"  [MASK] token ID:      {tokenizer.mask_token_id}")
print(f"
Vocab size: {tokenizer.vocab_size:,}  (WordPiece)")

from transformers import (
    AutoTokenizer, AutoModelForSequenceClassification,
    AutoModelForTokenClassification,
    TrainingArguments, Trainer, DataCollatorWithPadding,
)
from datasets import Dataset
import evaluate
import numpy as np
import torch

# ── Task: Swiggy complaint classification ────────────────────────────
# Categories: delivery_late, wrong_order, missing_items, quality_issue

LABELS   = ['delivery_late', 'wrong_order', 'missing_items', 'quality_issue']
label2id = {l: i for i, l in enumerate(LABELS)}
id2label = {i: l for i, l in enumerate(LABELS)}

complaints = [
    ("Order took 2 hours to arrive, completely cold", 0),
    ("Received biryani instead of ordered pizza", 1),
    ("Only 2 of 4 items delivered, rest missing", 2),
    ("Food was stale and tasted terrible", 3),
    ("Delivery was delayed by 45 minutes", 0),
    ("Wrong restaurant items delivered entirely", 1),
    ("Half my order was not delivered", 2),
    ("Found hair in the food very unhygienic", 3),
] * 50

tokenizer  = AutoTokenizer.from_pretrained('distilbert-base-uncased')
model      = AutoModelForSequenceClassification.from_pretrained(
    'distilbert-base-uncased',
    num_labels=4,
    id2label=id2label,
    label2id=label2id,
)

def tokenise(examples):
    return tokenizer(examples['text'], max_length=128,
                     truncation=True, padding=False)

texts  = [c[0] for c in complaints]
labels = [c[1] for c in complaints]

raw   = Dataset.from_dict({'text': texts, 'label': labels})
raw   = raw.train_test_split(test_size=0.2, seed=42)
tok   = raw.map(tokenise, batched=True, remove_columns=['text'])

collator = DataCollatorWithPadding(tokenizer=tokenizer)
acc_metric = evaluate.load('accuracy')

def compute_metrics(eval_pred):
    preds  = np.argmax(eval_pred.predictions, axis=1)
    return acc_metric.compute(predictions=preds, references=eval_pred.label_ids)

args = TrainingArguments(
    output_dir='./swiggy-complaints',
    num_train_epochs=3,
    per_device_train_batch_size=16,
    per_device_eval_batch_size=32,
    learning_rate=2e-5,
    warmup_ratio=0.1,
    weight_decay=0.01,
    evaluation_strategy='epoch',
    save_strategy='epoch',
    load_best_model_at_end=True,
    metric_for_best_model='accuracy',
    seed=42,
    logging_steps=20,
)

trainer = Trainer(
    model=model, args=args,
    train_dataset=tok['train'],
    eval_dataset=tok['test'],
    tokenizer=tokenizer,
    data_collator=collator,
    compute_metrics=compute_metrics,
)

print("Fine-tuning DistilBERT on Swiggy complaints:")
trainer.train()
results = trainer.evaluate()
print(f"
Accuracy: {results['eval_accuracy']:.4f}")

# ── Inference on new complaints ───────────────────────────────────────
model.eval()
new_complaints = [
    "Waited 90 minutes for delivery",
    "Got someone else's order completely",
    "Three items from my cart were not in the bag",
]
inputs = tokenizer(new_complaints, padding=True, truncation=True,
                   max_length=128, return_tensors='pt')
with torch.no_grad():
    logits = model(**inputs).logits
    probs  = torch.softmax(logits, dim=1)

print("
Predictions on new complaints:")
for complaint, prob in zip(new_complaints, probs):
    pred = LABELS[prob.argmax()]
    conf = prob.max().item()
    print(f"  '{complaint[:40]}' → {pred} ({conf:.3f})")

from transformers import AutoTokenizer, AutoModelForSequenceClassification
import torch, time

# ── Benchmark: accuracy vs speed tradeoff across BERT family ──────────
models_to_compare = [
    ('distilbert-base-uncased', 'DistilBERT'),
    ('bert-base-uncased',       'BERT-base'),
    ('roberta-base',            'RoBERTa-base'),
]

texts = ["Razorpay payment declined please help retry"] * 64

print(f"{'Model':<20} {'Params':>8} {'Latency (ms)':>14} {'Throughput'}")
print("─" * 60)

for model_id, name in models_to_compare:
    tok   = AutoTokenizer.from_pretrained(model_id)
    model = AutoModelForSequenceClassification.from_pretrained(
        model_id, num_labels=3
    )
    model.eval()

    params = sum(p.numel() for p in model.parameters()) / 1e6

    inputs = tok(texts, return_tensors='pt', padding=True,
                 truncation=True, max_length=64)

    # Warmup
    with torch.no_grad(): _ = model(**inputs)

    # Time
    start = time.time()
    for _ in range(5):
        with torch.no_grad(): _ = model(**inputs)
    ms_per_batch = (time.time() - start) / 5 * 1000

    throughput = len(texts) / (ms_per_batch / 1000)
    print(f"  {name:<18}  {params:>6.0f}M  {ms_per_batch:>12.1f}ms  "
          f"{throughput:>6.0f} samples/s")

from transformers import (
    AutoTokenizer, AutoModelForTokenClassification, pipeline,
)
import torch

# ── NER with pretrained BERT ──────────────────────────────────────────
# Use a pretrained NER model — fine-tuned BERT on CoNLL-2003
tokenizer = AutoTokenizer.from_pretrained('dbmdz/bert-large-cased-finetuned-conll03-english')
model     = AutoModelForTokenClassification.from_pretrained(
    'dbmdz/bert-large-cased-finetuned-conll03-english'
)

# Pipeline handles aggregation (merging subword tokens into words)
ner_pipeline = pipeline(
    'ner',
    model=model,
    tokenizer=tokenizer,
    aggregation_strategy='simple',  # merge subword tokens
)

# ── Extract entities from Razorpay dispute texts ──────────────────────
dispute_texts = [
    "I made a payment of Rs 2500 to Swiggy on 15th March 2026.",
    "Flipkart charged me twice for an order from Mumbai warehouse.",
    "Amazon India debited Rs 4999 from my HDFC account yesterday.",
]

print("Named Entity Recognition on payment disputes:")
for text in dispute_texts:
    entities = ner_pipeline(text)
    print(f"
  Text: '{text}'")
    for ent in entities:
        print(f"    {ent['entity_group']:<6} '{ent['word']}'  "
              f"score={ent['score']:.3f}")

# ── Fine-tuning BERT for custom NER ───────────────────────────────────
# Custom entities: MERCHANT, AMOUNT, DATE, ACCOUNT
print("""
# Custom NER training pattern:
# 1. Annotate data in BIO format:
#    tokens: ["Paid", "Rs", "500", "to", "Swiggy"]
#    labels: ["O",  "B-AMOUNT", "I-AMOUNT", "O", "B-MERCHANT"]
#
# 2. Load model:
#    model = AutoModelForTokenClassification.from_pretrained(
#        'distilbert-base-uncased',
#        num_labels=len(label_list),  # B-MERCHANT, I-MERCHANT, B-AMOUNT, ... O
#    )
#
# 3. Key difference from sequence classification:
#    - Labels are per-token not per-sequence
#    - Subword tokens get label -100 (ignored in loss)
#    - Only the first subword of each word gets the real label
#
# 4. Tokenisation with word_ids alignment:
def tokenise_and_align(examples, tokenizer, label2id):
    tokenized = tokenizer(examples['tokens'], is_split_into_words=True,
                          truncation=True, max_length=128)
    all_labels = []
    for i, labels in enumerate(examples['ner_tags']):
        word_ids   = tokenized.word_ids(batch_index=i)
        prev_word  = None
        label_ids  = []
        for word_id in word_ids:
            if word_id is None:
                label_ids.append(-100)   # special tokens ignored
            elif word_id != prev_word:
                label_ids.append(label2id[labels[word_id]])
            else:
                label_ids.append(-100)   # subword tokens ignored
            prev_word = word_id
        all_labels.append(label_ids)
    tokenized['labels'] = all_labels
    return tokenized
""")