LLMs — Pretraining, RLHF, and Scaling Laws

Pretraining is like a person reading every book, article, and website ever written. They become extraordinarily knowledgeable about language and the world. But they have no manners, no values, and no sense of what is helpful vs harmful — they just know what typically follows what in text. Alignment is giving them social training, teaching them to be genuinely helpful, and giving them the judgment to refuse harmful requests.

The insight from OpenAI's InstructGPT paper (2022): a 1.3B parameter model fine-tuned with RLHF was preferred by human raters over a raw 175B GPT-3. Alignment is more important than raw scale for user-facing applications.

import torch
import torch.nn as nn
import torch.nn.functional as F
import numpy as np

# ── Minimal GPT-style decoder-only Transformer ────────────────────────
class CausalSelfAttention(nn.Module):
    """Multi-head self-attention with causal (left-to-right) masking."""
    def __init__(self, d_model: int, n_heads: int, max_seq: int = 512):
        super().__init__()
        assert d_model % n_heads == 0
        self.n_heads = n_heads
        self.d_head  = d_model // n_heads
        self.qkv     = nn.Linear(d_model, 3 * d_model)
        self.proj    = nn.Linear(d_model, d_model)

        # Causal mask — lower triangular, registered as buffer (not parameter)
        mask = torch.tril(torch.ones(max_seq, max_seq))
        self.register_buffer('mask', mask.view(1, 1, max_seq, max_seq))

    def forward(self, x):
        B, T, C = x.shape
        qkv = self.qkv(x).chunk(3, dim=-1)
        q, k, v = [t.view(B, T, self.n_heads, self.d_head).transpose(1, 2)
                    for t in qkv]                     # (B, H, T, d_head)

        scale   = self.d_head ** -0.5
        scores  = (q @ k.transpose(-2, -1)) * scale  # (B, H, T, T)
        # Apply causal mask — positions cannot attend to future positions
        scores  = scores.masked_fill(self.mask[:, :, :T, :T] == 0, float('-inf'))
        attn    = F.softmax(scores, dim=-1)
        out     = (attn @ v).transpose(1, 2).reshape(B, T, C)
        return self.proj(out)

class TransformerBlock(nn.Module):
    def __init__(self, d_model: int, n_heads: int, ffn_mult: int = 4):
        super().__init__()
        self.attn    = CausalSelfAttention(d_model, n_heads)
        self.ffn     = nn.Sequential(
            nn.Linear(d_model, d_model * ffn_mult),
            nn.GELU(),
            nn.Linear(d_model * ffn_mult, d_model),
        )
        self.ln1 = nn.LayerNorm(d_model)
        self.ln2 = nn.LayerNorm(d_model)

    def forward(self, x):
        x = x + self.attn(self.ln1(x))   # pre-norm (GPT-2 style)
        x = x + self.ffn(self.ln2(x))
        return x

class MiniGPT(nn.Module):
    def __init__(self, vocab_size: int, d_model: int, n_layers: int,
                  n_heads: int, max_seq: int = 512):
        super().__init__()
        self.token_emb = nn.Embedding(vocab_size, d_model)
        self.pos_emb   = nn.Embedding(max_seq, d_model)
        self.blocks    = nn.ModuleList([
            TransformerBlock(d_model, n_heads) for _ in range(n_layers)
        ])
        self.ln_final  = nn.LayerNorm(d_model)
        self.lm_head   = nn.Linear(d_model, vocab_size, bias=False)

        # Weight tying: input and output embeddings share weights (saves params)
        self.lm_head.weight = self.token_emb.weight

        # Initialise weights
        self.apply(self._init_weights)

    def _init_weights(self, module):
        if isinstance(module, nn.Linear):
            nn.init.normal_(module.weight, std=0.02)
            if module.bias is not None:
                nn.init.zeros_(module.bias)
        elif isinstance(module, nn.Embedding):
            nn.init.normal_(module.weight, std=0.02)

    def forward(self, tokens: torch.Tensor, targets: torch.Tensor = None):
        B, T = tokens.shape
        pos  = torch.arange(T, device=tokens.device)

        x    = self.token_emb(tokens) + self.pos_emb(pos)
        for block in self.blocks:
            x = block(x)
        x    = self.ln_final(x)
        logits = self.lm_head(x)   # (B, T, vocab_size)

        loss = None
        if targets is not None:
            # Shift: predict token i+1 from token i
            loss = F.cross_entropy(
                logits[:, :-1].reshape(-1, logits.size(-1)),
                targets[:, 1:].reshape(-1),
                ignore_index=-1,
            )
        return logits, loss

    @torch.no_grad()
    def generate(self, prompt: torch.Tensor, max_new: int = 50,
                  temperature: float = 0.8, top_k: int = 40) -> torch.Tensor:
        """Autoregressive generation — sample one token at a time."""
        for _ in range(max_new):
            # Crop to max context window
            ctx    = prompt[:, -512:]
            logits, _ = self(ctx)
            logits = logits[:, -1, :] / temperature   # last token only

            # Top-k sampling
            if top_k:
                v, _ = torch.topk(logits, top_k)
                logits[logits < v[:, -1:]] = -float('inf')

            probs  = F.softmax(logits, dim=-1)
            next_t = torch.multinomial(probs, num_samples=1)
            prompt = torch.cat([prompt, next_t], dim=1)
        return prompt

# ── Shape and parameter check ─────────────────────────────────────────
VOCAB  = 50257   # GPT-2 vocabulary size
model  = MiniGPT(vocab_size=VOCAB, d_model=256, n_layers=4,
                  n_heads=8, max_seq=512)

tokens  = torch.randint(0, VOCAB, (2, 128))
targets = tokens.clone()
logits, loss = model(tokens, targets)

params = sum(p.numel() for p in model.parameters())
print(f"MiniGPT (4 layers, d=256):")
print(f"  Parameters: {params:,}")
print(f"  Logits:     {tuple(logits.shape)}")
print(f"  Loss:       {loss.item():.4f}  (random init ≈ log({VOCAB}) = {np.log(VOCAB):.2f})")
print(f"
Real LLM parameter counts:")
for name, p in [('GPT-2',125e6),('GPT-3',175e9),('LLaMA-3-8B',8e9),
                 ('LLaMA-3-70B',70e9),('GPT-4 (est)',~1e12)]:
    print(f"  {name:<20}: {p/1e9:.1f}B parameters")

import numpy as np

# ── Chinchilla scaling law prediction ────────────────────────────────
def chinchilla_optimal(compute_flops: float) -> dict:
    """
    Predict optimal model size and token count for a compute budget.
    From Hoffmann et al. 2022 (Chinchilla paper).
    C ≈ 6 × N × D  (one forward + backward ≈ 6× forward FLOPs)
    Optimal: N_opt = (C / 6 / 20)^0.5, D_opt = 20 × N_opt
    """
    # Derived from Chinchilla power law coefficients
    N_opt = (compute_flops / 6 / 20) ** 0.5
    D_opt = 20 * N_opt
    return {'params': N_opt, 'tokens': D_opt, 'compute': compute_flops}

print("Chinchilla optimal allocations:")
print(f"{'Compute (FLOPs)':>18} {'Optimal params':>16} {'Optimal tokens':>16}")
print("─" * 52)
for log_c in [20, 21, 22, 23, 24]:
    c = 10 ** log_c
    r = chinchilla_optimal(c)
    def fmt(n):
        if n >= 1e12: return f"{n/1e12:.1f}T"
        if n >= 1e9:  return f"{n/1e9:.1f}B"
        if n >= 1e6:  return f"{n/1e6:.0f}M"
        return f"{n:.0f}"
    print(f"  1e{log_c}              {fmt(r['params']):>16} {fmt(r['tokens']):>16}")

# ── Loss prediction from scaling laws ────────────────────────────────
def predict_loss(N: float, D: float) -> float:
    """
    Chinchilla loss prediction:
    L(N, D) = E + A/N^α + B/D^β
    E=1.69 (irreducible loss), A=406.4, B=410.7, α=0.34, β=0.28
    """
    E, A, B, alpha, beta = 1.69, 406.4, 410.7, 0.34, 0.28
    return E + A / N**alpha + B / D**beta

print(f"
Loss predictions for different training regimes:")
configs = [
    ('GPT-2 (125M, 10B tokens)',       125e6, 10e9),
    ('GPT-3 (175B, 300B tokens)',       175e9, 300e9),
    ('Chinchilla (70B, 1.4T tokens)',   70e9,  1.4e12),
    ('LLaMA-3-8B (8B, 15T tokens)',     8e9,   15e12),
    ('LLaMA-3-70B (70B, 15T tokens)',   70e9,  15e12),
]
print(f"  {'Config':<40} {'Loss':>8}  {'Perplexity':>12}")
print("  " + "─" * 62)
for name, N, D in configs:
    loss = predict_loss(N, D)
    ppl  = np.exp(loss)
    print(f"  {name:<40} {loss:>8.3f}  {ppl:>12.1f}")

import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim

# ── Stage 2: SFT — supervised fine-tuning ────────────────────────────
# The key: compute loss only on the response tokens, not the prompt tokens.
# The model already knows how to process prompts from pretraining.
# We only want to train it to produce the target response.

def sft_loss(model, prompt_tokens, response_tokens, pad_id=-1):
    """
    SFT loss: cross-entropy on response tokens only.
    prompt_tokens:   (B, T_prompt)  — the instruction/context
    response_tokens: (B, T_response) — the target response
    """
    # Concatenate prompt + response
    full_seq = torch.cat([prompt_tokens, response_tokens], dim=1)

    logits, _ = model(full_seq)   # (B, T_full, vocab)

    # Only compute loss on response positions
    T_p = prompt_tokens.size(1)
    T_r = response_tokens.size(1)

    # logits at position T_p-1 predicts first response token
    response_logits  = logits[:, T_p-1 : T_p+T_r-1, :]   # (B, T_r, vocab)
    response_targets = response_tokens                      # (B, T_r)

    # Flatten and compute cross-entropy
    loss = F.cross_entropy(
        response_logits.reshape(-1, response_logits.size(-1)),
        response_targets.reshape(-1),
        ignore_index=pad_id,
    )
    return loss

# ── Stage 3a: Reward Model ────────────────────────────────────────────
class RewardModel(nn.Module):
    """
    Takes a (prompt, response) pair and outputs a scalar reward score.
    Trained on human preference comparisons: which response is better?
    Architecture: LLM backbone + linear head predicting scalar reward.
    """
    def __init__(self, d_model: int = 256, vocab_size: int = 50257):
        super().__init__()
        # In practice: initialise from SFT model weights
        self.transformer = nn.TransformerEncoder(
            nn.TransformerEncoderLayer(d_model, 4, batch_first=True),
            num_layers=2,
        )
        self.token_emb = nn.Embedding(vocab_size, d_model)
        self.reward_head = nn.Linear(d_model, 1)   # scalar reward

    def forward(self, tokens):
        x = self.token_emb(tokens)
        x = self.transformer(x)
        # Use last token's representation as the reward signal
        reward = self.reward_head(x[:, -1, :])
        return reward.squeeze(-1)   # (B,)

def reward_model_loss(reward_model, prompt_tokens,
                       chosen_tokens, rejected_tokens):
    """
    Bradley-Terry preference loss:
    Maximise P(chosen > rejected) = sigmoid(r_chosen - r_rejected)
    """
    chosen_full   = torch.cat([prompt_tokens, chosen_tokens], dim=1)
    rejected_full = torch.cat([prompt_tokens, rejected_tokens], dim=1)

    r_chosen   = reward_model(chosen_full)    # scalar
    r_rejected = reward_model(rejected_full)  # scalar

    # Loss = -log sigmoid(r_chosen - r_rejected)
    # Equivalent to: log(1 + exp(r_rejected - r_chosen))
    loss = -F.logsigmoid(r_chosen - r_rejected).mean()
    return loss, r_chosen.mean().item(), r_rejected.mean().item()

# ── Demonstrate reward model training ─────────────────────────────────
rm = RewardModel(d_model=64, vocab_size=1000)
opt = optim.Adam(rm.parameters(), lr=1e-4)

print("Reward model training (preference learning):")
print(f"{'Step':>6} {'Loss':>8} {'r_chosen':>10} {'r_rejected':>12} {'margin':>8}")
print("─" * 48)

for step in range(1, 6):
    # Simulate prompt + chosen/rejected response tokens
    B = 8
    prompt   = torch.randint(0, 1000, (B, 20))
    chosen   = torch.randint(0, 1000, (B, 30))   # "better" response
    rejected = torch.randint(0, 1000, (B, 30))   # "worse" response

    opt.zero_grad()
    loss, rc, rr = reward_model_loss(rm, prompt, chosen, rejected)
    loss.backward()
    opt.step()
    print(f"  {step:>4}  {loss.item():>8.4f}  {rc:>10.4f}  {rr:>12.4f}  {rc-rr:>8.4f}")

import torch
import torch.nn as nn
import torch.nn.functional as F

def compute_log_probs(model, tokens, labels):
    """
    Compute sum of log probabilities for a sequence of tokens.
    Used to compute log π(y|x) in DPO.
    tokens: (B, T_prompt + T_response)
    labels: (B, T_prompt + T_response) — -100 for prompt tokens (ignored)
    """
    logits, _ = model(tokens)                    # (B, T, vocab)
    # Shift: logit at position i predicts token i+1
    shift_logits = logits[:, :-1, :]             # (B, T-1, vocab)
    shift_labels = labels[:, 1:]                 # (B, T-1)

    # Cross-entropy per token (not reduced)
    log_probs = -F.cross_entropy(
        shift_logits.reshape(-1, shift_logits.size(-1)),
        shift_labels.reshape(-1),
        ignore_index=-100,
        reduction='none',
    ).view(logits.size(0), -1)   # (B, T-1)

    # Sum over response tokens only (prompt tokens have ignore_index=-100)
    return log_probs.sum(dim=-1)  # (B,) — total log prob of response

def dpo_loss(policy_model, reference_model,
              prompt_tokens, chosen_tokens, rejected_tokens,
              beta: float = 0.1):
    """
    DPO loss: direct preference optimisation without RL.
    policy_model:    the model being trained (π)
    reference_model: frozen SFT model (π_ref)
    beta:            KL regularisation strength
    """
    B = prompt_tokens.size(0)

    def build_input(response):
        tokens = torch.cat([prompt_tokens, response], dim=1)
        # Labels: -100 for prompt positions (ignored in loss)
        labels = tokens.clone()
        labels[:, :prompt_tokens.size(1)] = -100
        return tokens, labels

    chosen_input,   chosen_labels   = build_input(chosen_tokens)
    rejected_input, rejected_labels = build_input(rejected_tokens)

    # Compute log probs under policy (trainable)
    policy_chosen_logp   = compute_log_probs(policy_model,
                                               chosen_input,   chosen_labels)
    policy_rejected_logp = compute_log_probs(policy_model,
                                               rejected_input, rejected_labels)

    # Compute log probs under reference (frozen)
    with torch.no_grad():
        ref_chosen_logp   = compute_log_probs(reference_model,
                                               chosen_input,   chosen_labels)
        ref_rejected_logp = compute_log_probs(reference_model,
                                               rejected_input, rejected_labels)

    # DPO implicit reward ratio
    chosen_rewards   = beta * (policy_chosen_logp   - ref_chosen_logp)
    rejected_rewards = beta * (policy_rejected_logp - ref_rejected_logp)

    # DPO loss: -log sigmoid(r_w - r_l)
    loss = -F.logsigmoid(chosen_rewards - rejected_rewards).mean()

    # Metrics for monitoring
    accuracy = (chosen_rewards > rejected_rewards).float().mean()
    margin   = (chosen_rewards - rejected_rewards).mean()

    return loss, accuracy.item(), margin.item()

# ── DPO vs RLHF comparison ────────────────────────────────────────────
print("DPO vs RLHF:")
comparison = [
    ('Models needed',       'LLM + Reward Model + Reference', 'LLM + Reference (frozen)'),
    ('Training algorithm',  'PPO (complex, unstable)',         'Simple BCE loss'),
    ('Data format',         'Scalar rewards per response',     'Preference pairs (A>B)'),
    ('Compute overhead',    '3-4× SFT compute',                '~1.5× SFT compute'),
    ('Engineering effort',  'Very high (PPO tuning)',          'Low (standard training)'),
    ('Alignment quality',   'Slightly better (in theory)',     'Comparable in practice'),
    ('Who uses it',         'OpenAI (GPT-4), Anthropic early', 'Meta (LLaMA), Mistral, Phi'),
]
print(f"  {'Aspect':<25} {'RLHF':>35} {'DPO':>35}")
print("  " + "─" * 95)
for aspect, rlhf, dpo in comparison:
    print(f"  {aspect:<25} {rlhf:>35} {dpo:>35}")

import torch
import torch.nn.functional as F
import numpy as np

# ── Sampling strategies — controlling generation quality vs diversity ──

def greedy_decode(logits):
    """Always pick the highest probability token. Deterministic, repetitive."""
    return logits.argmax(dim=-1)

def temperature_sample(logits, temperature=1.0):
    """
    Divide logits by temperature before softmax.
    temperature < 1: sharper distribution, more conservative/repetitive
    temperature = 1: sample from true model distribution
    temperature > 1: flatter distribution, more random/creative
    """
    scaled = logits / temperature
    probs  = F.softmax(scaled, dim=-1)
    return torch.multinomial(probs, num_samples=1).squeeze(-1)

def top_k_sample(logits, k=50, temperature=1.0):
    """
    Only consider top-k most likely tokens.
    Prevents sampling from the long tail of unlikely tokens.
    k=50 is standard. k=1 is greedy. k=vocab_size is full sampling.
    """
    logits = logits / temperature
    top_k_vals, top_k_idx = torch.topk(logits, k, dim=-1)
    # Mask everything outside top-k
    filtered = torch.full_like(logits, float('-inf'))
    filtered.scatter_(-1, top_k_idx, top_k_vals)
    probs = F.softmax(filtered, dim=-1)
    return torch.multinomial(probs, num_samples=1).squeeze(-1)

def top_p_sample(logits, p=0.9, temperature=1.0):
    """
    Nucleus sampling: keep the smallest set of tokens whose
    cumulative probability exceeds p.
    Adaptive — focuses on fewer tokens for confident predictions.
    p=0.9 is standard. p=1.0 is full sampling.
    """
    logits = logits / temperature
    probs  = F.softmax(logits, dim=-1)
    sorted_probs, sorted_idx = torch.sort(probs, descending=True)
    cumprobs = torch.cumsum(sorted_probs, dim=-1)

    # Remove tokens once cumulative prob exceeds p
    mask = cumprobs - sorted_probs > p
    sorted_probs[mask] = 0.0
    sorted_probs /= sorted_probs.sum(dim=-1, keepdim=True)  # renormalise

    sampled_idx = torch.multinomial(sorted_probs, num_samples=1)
    return sorted_idx.gather(-1, sampled_idx).squeeze(-1)

# ── Demonstrate sampling strategy effects ────────────────────────────
torch.manual_seed(42)
VOCAB_SIZE = 100

# Simulate logits with a peaked distribution
logits = torch.randn(VOCAB_SIZE)
logits[5]  = 5.0   # dominant token
logits[12] = 3.0   # second choice
logits[23] = 2.5   # third choice

print("Sampling strategy comparison (10 samples each):")
strategies = {
    'Greedy':              lambda l: greedy_decode(l.unsqueeze(0)).item(),
    'Temperature=0.5':     lambda l: temperature_sample(l.unsqueeze(0), 0.5).item(),
    'Temperature=1.0':     lambda l: temperature_sample(l.unsqueeze(0), 1.0).item(),
    'Temperature=1.5':     lambda l: temperature_sample(l.unsqueeze(0), 1.5).item(),
    'Top-k (k=10)':        lambda l: top_k_sample(l.unsqueeze(0), k=10).item(),
    'Top-p (p=0.9)':       lambda l: top_p_sample(l.unsqueeze(0), p=0.9).item(),
}
for name, fn in strategies.items():
    samples = [fn(logits.clone()) for _ in range(10)]
    unique  = len(set(samples))
    print(f"  {name:<22}: samples={samples[:6]}  unique={unique}")

# ── Quantisation — running large models on limited hardware ───────────
print("
Quantisation options for LLM deployment:")
quant_methods = [
    ('fp32 (no quantisation)', '4 bytes/param', '280GB for 70B model',  'Full accuracy',         'Research only'),
    ('fp16 / bf16',            '2 bytes/param', '140GB for 70B model',  '~same accuracy',        'Standard GPU training'),
    ('int8 (bitsandbytes)',    '1 byte/param',  '70GB for 70B model',   '<0.5% degradation',     '2× memory reduction'),
    ('int4 (GPTQ/AWQ)',        '0.5 bytes/param','35GB for 70B model',  '1-2% degradation',      'Consumer GPU inference'),
    ('int4 (llama.cpp)',       '0.5 bytes/param','35GB for 70B model',  '1-2% degradation',      'CPU inference possible'),
    ('2-bit (AQLM)',           '0.25 bytes/param','17.5GB for 70B',     '3-5% degradation',      'Extreme compression'),
]
print(f"  {'Method':<28} {'Size':>14} {'70B VRAM':>16} {'Quality':>18} {'Use case'}")
print("  " + "─" * 100)
for m, sz, vram, qual, use in quant_methods:
    print(f"  {m:<28} {sz:>14} {vram:>16} {qual:>18}  {use}")

Slack message from the alignment lead: "New checkpoint scores worse on the refusal benchmark than last week's — it is refusing benign coding questions it used to answer fine. Reward model score went up though. Investigate before we ship." This is a textbook reward hacking symptom: the reward model learned a shortcut (maybe associating caution-flavoured language with higher preference scores) that raised its own metric while making the actual product worse. The fix is rarely "retrain from scratch" — it usually means pulling the specific failing transcripts, adding targeted preference examples that penalise the over-refusal, adjusting the KL penalty against the reference model so the policy cannot drift as far, and re-running the held-out benchmark suite before touching the reward model weights again.

RLHF and DPO are alignment steps, not capability steps. They reshape which of the base model's already-latent behaviours get surfaced and reinforced — being concise, refusing harmful requests, following instruction formatting — but they do not teach the model new facts or new reasoning ability that pretraining did not already provide. This is exactly why InstructGPT's famous result was so striking: a small, heavily aligned 1.3B model being preferred over a raw 175B base model is not evidence that alignment created new capability out of nothing, it is evidence that the raw capability was already present in the base model and alignment simply made it reliably accessible and pleasant to interact with.

The Chinchilla scaling laws describe an optimal token count assuming reasonably high-quality, deduplicated data — they say nothing about dumping more raw, duplicated, or low-quality text into the mix. In practice, heavily duplicated web content wastes compute reinforcing the same patterns repeatedly, and low-quality or toxic content measurably degrades downstream behaviour. A large fraction of the real engineering effort — and a large fraction of what actually differentiates one lab's pretraining run from another's at the same parameter count — goes into deduplication, quality filtering, and data mixing, not into simply acquiring more raw tokens.

The reward model is trained on nothing but human comparisons of which response a rater preferred — it has learned to predict human preference, not truth. Human raters carry systematic biases: a tendency to prefer longer, more detailed-looking answers even when a shorter one is equally correct, or to prefer a confident, agreeable tone even when pushing back would be more honest. A policy optimised hard against this reward model will happily learn to exploit exactly those biases — becoming verbose or sycophantic — because from the reward model's point of view that behaviour genuinely does score higher. This gap between "scores well on the reward model" and "is actually a better response" is the entire reason reward hacking exists.

They differ in almost every dimension that matters. Pretraining is self-supervised — next-token prediction over unlabeled internet-scale text needs no human annotation at all — and consumes roughly 99% of total training compute over trillions of tokens. SFT and RLHF or DPO instead train on tiny, expensive, carefully curated human-produced or human-labelled datasets, often tens of thousands to a few hundred thousand examples, using entirely different objectives: cross-entropy imitation for SFT, and a preference- or reward-based objective for RLHF and DPO. Treating alignment as "more of the same training, just with better data" misses that the whole point of the later stages is to optimise for a fundamentally different signal — human preference — using a completely different data pipeline and, in RLHF's case, a completely different training algorithm.

Alignment training reshapes a statistical distribution over outputs — it does not install a hard, unbreakable rule. Adversarial prompting and jailbreaks routinely find inputs that push the model outside the distribution the alignment training covered well, surfacing the underlying pretrained behaviour it was supposed to suppress. Even more relevant in practice: further fine-tuning a model on unrelated downstream data, including seemingly harmless task-specific fine-tuning, can measurably erode safety behaviour that RLHF or DPO installed — a documented phenomenon sometimes called safety regression. This is why serious deployments pair alignment training with ongoing red-teaming, input and output monitoring, and re-evaluation after any further fine-tuning, rather than treating one alignment pass as a permanent guarantee.

Pretraining first: a decoder-only Transformer is trained with next-token prediction on trillions of tokens of largely unlabelled web, book, and code text, consuming the large majority of total compute and producing a model that completes text fluently but follows no instructions. Supervised fine-tuning comes next: the pretrained model is fine-tuned on tens of thousands of curated prompt-and-ideal-response pairs, with loss computed only on the response tokens, teaching it to behave like an assistant rather than a text completer. Then alignment: human annotators compare pairs of the SFT model's responses, and either a reward model is trained on those comparisons and the policy is optimised against it with PPO, or the comparisons are used directly in a DPO loss. The result goes through safety red-teaming, and often several more rounds of targeted preference data collection, before release.

PPO-based RLHF requires training and maintaining three models at once — the policy being optimised, a separately trained reward model, and a frozen reference policy — and running proximal policy optimisation, a reinforcement learning algorithm that is notoriously sensitive to hyperparameters and prone to instability. DPO sidesteps the reward model and the RL loop entirely: it derives a closed-form loss showing that the optimal RLHF policy can be reached directly from preference pairs using a simple binary cross-entropy-style objective against the reference model. The industry shift toward DPO, visible in most open-source model releases, is mostly about engineering cost — DPO reaches comparable alignment quality with a fraction of the moving parts and a much shorter, more stable training loop.

Pretraining optimises for exactly one thing: predicting the next token consistent with internet-scale text. A base model asked a question will complete it in whatever style matches its training distribution — which might be helpful, might be evasive, might continue with more questions instead of answering, and will happily continue harmful content if that is what similar text in its training data looked like. Knowing a huge amount about the world is orthogonal to reliably being helpful, honest, and refusing harmful requests on demand — alignment is the stage that specifically optimises for that behavioural target, using human preference data the next-token objective never saw during pretraining.

Reward hacking happens when the policy finds a shortcut that raises its score from the reward model without genuinely improving response quality — becoming needlessly verbose because raters tend to prefer longer answers, or becoming sycophantic because agreeing with the user scores well even when the user is wrong. I would catch this by never trusting the reward model score in isolation: track a held-out suite of task benchmarks and a refusal-calibration benchmark across training, and watch for the reward score climbing while those independent metrics stay flat or get worse. Manually reading a sample of high-reward transcripts each training run is also worth the time — reward hacking patterns are usually obvious to a human within a handful of examples even when they are invisible in the aggregate reward number.

I would not think of it as splitting the same budget, because the two stages consume wildly different amounts of compute for wildly different returns — pretraining is roughly 99% of total compute and alignment is closer to 1%, yet the InstructGPT result showed a heavily aligned model an order of magnitude smaller beating a raw base model on human preference. That asymmetry means alignment investment is almost always underfunded relative to its impact on user-perceived quality, so I would treat pretraining scale as the given from the overall budget and separately make sure the alignment stage has enough dedicated preference-data collection and red-teaming cycles, rather than treating it as an afterthought squeezed into whatever time is left after the pretraining run finishes.