Encoding Categorical Features

What this module covers:

import pandas as pd
import numpy as np
from sklearn.model_selection import train_test_split

np.random.seed(42)
n = 8_000

restaurants = ['Pizza Hut', 'Biryani Blues', "McDonald's", "Haldiram's",
               'Dominos', 'KFC', 'Subway', 'Burger King']
cities      = ['Seattle', 'New York', 'Denver', 'Austin', 'Boston', 'Chicago']
time_slots  = ['breakfast', 'lunch', 'evening', 'dinner']
ratings_cat = ['poor', 'average', 'good', 'excellent']   # ordinal

distance   = np.abs(np.random.normal(4.0, 2.0, n)).clip(0.5, 15).round(2)
traffic    = np.random.randint(1, 11, n).astype(float)
prep       = np.abs(np.random.normal(15, 5, n)).clip(5, 35).round(1)
value      = np.abs(np.random.normal(350, 150, n)).clip(50, 1200).round(0)
delivery   = (8.6 + 7.3*distance + 0.8*prep + 1.5*traffic
              + np.random.normal(0, 4, n)).clip(10, 120).round(1)

# Ordinal rating: bucketised from delivery time (worse delivery = worse rating)
rating_num = pd.cut(delivery, bins=[0, 25, 40, 55, 200],
                    labels=[3, 2, 1, 0]).astype(int)   # 3=excellent, 0=poor
rating_cat = pd.Categorical.from_codes(
    rating_num, categories=['poor', 'average', 'good', 'excellent']
)

df = pd.DataFrame({
    'restaurant':   np.random.choice(restaurants, n),
    'city':         np.random.choice(cities, n),
    'time_slot':    np.random.choice(time_slots, n),
    'rating_cat':   rating_cat,          # ordinal: poor < average < good < excellent
    'distance_km':  distance,
    'traffic_score': traffic,
    'restaurant_prep': prep,
    'order_value':  value,
    'delivery_time': delivery,
    'is_late':      (delivery > 45).astype(int),
})

X = df.drop(columns=['delivery_time', 'is_late'])
y = df['delivery_time']

X_train, X_test, y_train, y_test = train_test_split(
    X, y, test_size=0.2, random_state=42
)

print(f"Train: {len(X_train):,}  Test: {len(X_test):,}")
print(f"
Categorical columns:")
for col in ['restaurant', 'city', 'time_slot', 'rating_cat']:
    print(f"  {col:<20}: {X_train[col].nunique()} unique values → {sorted(X_train[col].unique())[:5]}")

import numpy as np
import pandas as pd
from sklearn.linear_model import Ridge
from sklearn.preprocessing import StandardScaler, LabelEncoder, OneHotEncoder
from sklearn.metrics import mean_absolute_error

# Demonstrate the damage naive encoding does to a linear model
num_cols = ['distance_km', 'traffic_score', 'restaurant_prep', 'order_value']

# ── Approach 1: naive integer encoding for restaurant ─────────────────
le = LabelEncoder()
X_train_naive = X_train[num_cols].copy()
X_test_naive  = X_test[num_cols].copy()
X_train_naive['restaurant_int'] = le.fit_transform(X_train['restaurant'])
X_test_naive['restaurant_int']  = le.transform(X_test['restaurant'])

sc1 = StandardScaler()
model_naive = Ridge(alpha=1.0)
model_naive.fit(sc1.fit_transform(X_train_naive), y_train)
mae_naive = mean_absolute_error(y_test, model_naive.predict(sc1.transform(X_test_naive)))

# ── Approach 2: one-hot encoding for restaurant ────────────────────────
ohe = OneHotEncoder(sparse_output=False, handle_unknown='ignore', drop='first')
rest_ohe_train = ohe.fit_transform(X_train[['restaurant']])
rest_ohe_test  = ohe.transform(X_test[['restaurant']])

X_train_ohe = np.hstack([X_train[num_cols].values, rest_ohe_train])
X_test_ohe  = np.hstack([X_test[num_cols].values, rest_ohe_test])

sc2 = StandardScaler()
model_ohe = Ridge(alpha=1.0)
model_ohe.fit(sc2.fit_transform(X_train_ohe), y_train)
mae_ohe = mean_absolute_error(y_test, model_ohe.predict(sc2.transform(X_test_ohe)))

print(f"MAE with naive int encoding: {mae_naive:.4f} min")
print(f"MAE with one-hot encoding:   {mae_ohe:.4f} min")
print(f"Improvement from OHE:        {(mae_naive - mae_ohe):.4f} min")
# One-hot is always better for nominal categoricals in linear models

import numpy as np
import pandas as pd
from sklearn.preprocessing import OrdinalEncoder

# ── OrdinalEncoder — specify the order explicitly ──────────────────────
# ALWAYS provide categories= explicitly so the order is guaranteed
# Never rely on alphabetical order — 'good' comes before 'poor' alphabetically

ord_enc = OrdinalEncoder(
    categories=[['poor', 'average', 'good', 'excellent']],
    handle_unknown='use_encoded_value',
    unknown_value=-1,    # unseen categories at test time → -1
)

rating_train = ord_enc.fit_transform(X_train[['rating_cat']])
rating_test  = ord_enc.transform(X_test[['rating_cat']])

print("Ordinal encoding — rating_cat:")
for cat, code in zip(ord_enc.categories_[0], range(4)):
    print(f"  {cat:<12} → {code}")

# Verify: poor=0, average=1, good=2, excellent=3
sample = pd.DataFrame({'rating_cat': ['poor', 'good', 'excellent', 'average']})
print(f"
Sample encoding: {ord_enc.transform(sample).flatten()}")
# [0. 2. 3. 1.] — correct order ✓

# ── When ordinal encoding is wrong ────────────────────────────────────
# time_slot: 'breakfast', 'lunch', 'evening', 'dinner'
# These have a time-of-day order but the ML relationship to delivery
# time is NOT linear — dinner might be worse than evening.
# For time_slot: use one-hot OR cyclical encoding (from Module 17).
# Only use ordinal for features where the ordering is the SIGNAL.

# ── Ordinal encoding for tree models ─────────────────────────────────
# Tree models can use ordinal encoding even for nominal features
# because trees find the optimal threshold themselves.
# XGBoost and LightGBM work well with label-encoded categoricals.
# This is one case where integer encoding is acceptable.

from sklearn.ensemble import RandomForestRegressor
from sklearn.metrics import mean_absolute_error

num_cat_cols = ['distance_km', 'traffic_score', 'restaurant_prep', 'order_value']

# For Random Forest: just label-encode everything, trees handle it fine
from sklearn.preprocessing import LabelEncoder
X_tr_rf = X_train[num_cat_cols].copy()
X_te_rf = X_test[num_cat_cols].copy()
for col in ['restaurant', 'city', 'time_slot', 'rating_cat']:
    le = LabelEncoder()
    X_tr_rf[col] = le.fit_transform(X_train[col])
    X_te_rf[col] = le.transform(X_test[col])

rf = RandomForestRegressor(n_estimators=100, random_state=42, n_jobs=-1)
rf.fit(X_tr_rf, y_train)
mae_rf = mean_absolute_error(y_test, rf.predict(X_te_rf))
print(f"
Random Forest with label encoding: MAE = {mae_rf:.4f} min")
print("(Trees are fine with integer-encoded categoricals)")

import numpy as np
import pandas as pd
from sklearn.preprocessing import OneHotEncoder
from sklearn.pipeline import Pipeline
from sklearn.preprocessing import StandardScaler
from sklearn.linear_model import Ridge
from sklearn.metrics import mean_absolute_error

# ── OneHotEncoder — the production way ───────────────────────────────
ohe = OneHotEncoder(
    sparse_output=False,      # return dense array not sparse matrix
    handle_unknown='ignore',  # unknown categories at test → all zeros (not an error)
    drop='first',             # drop one column per feature — avoids dummy trap
    dtype=np.float32,         # float32 saves memory vs float64
)

cat_cols = ['restaurant', 'city', 'time_slot']
ohe.fit(X_train[cat_cols])

train_ohe = ohe.transform(X_train[cat_cols])
test_ohe  = ohe.transform(X_test[cat_cols])

print(f"Columns created: {train_ohe.shape[1]}")
print(f"Feature names:")
for name in ohe.get_feature_names_out(cat_cols):
    print(f"  {name}")

# ── What handle_unknown='ignore' does at inference ─────────────────────
# Restaurant not seen during training → all zeros for that restaurant's columns
# The model falls back to "none of the known restaurants" behaviour (baseline)
new_order = pd.DataFrame({'restaurant': ['New Brand Burger'],
                          'city':       ['Seattle'],
                          'time_slot':  ['lunch']})
encoded_new = ohe.transform(new_order)
print(f"
Unknown restaurant encoded as: {encoded_new[0, :8]}")  # all zeros

# ── pd.get_dummies — quick exploration, not for production ────────────
# Use for notebooks / exploration because it is one line
# Do NOT use in pipelines — test set may have different columns
dummies = pd.get_dummies(X_train['restaurant'], prefix='rest', drop_first=True)
print(f"
pd.get_dummies output columns ({len(dummies.columns)}):")
print(dummies.columns.tolist())

# ── Full pipeline with OHE ─────────────────────────────────────────────
from sklearn.compose import ColumnTransformer
from sklearn.impute import SimpleImputer

NUM_COLS = ['distance_km', 'traffic_score', 'restaurant_prep', 'order_value']
CAT_COLS = ['restaurant', 'city', 'time_slot']

preprocessor = ColumnTransformer([
    ('num', Pipeline([
        ('imp',   SimpleImputer(strategy='median')),
        ('scale', StandardScaler()),
    ]), NUM_COLS),
    ('cat', OneHotEncoder(
        sparse_output=False, handle_unknown='ignore', drop='first'
    ), CAT_COLS),
])

pipe = Pipeline([
    ('prep',  preprocessor),
    ('model', Ridge(alpha=1.0)),
])
pipe.fit(X_train, y_train)
mae = mean_absolute_error(y_test, pipe.predict(X_test))
print(f"
Pipeline with OHE — MAE: {mae:.4f} min")

import numpy as np
import pandas as pd
from sklearn.base import BaseEstimator, TransformerMixin

# ── Manual frequency encoding ─────────────────────────────────────────
def frequency_encode(X_train_col: pd.Series,
                     X_apply_col: pd.Series) -> pd.Series:
    """
    Compute frequency map on training data, apply to any dataset.
    Unknown categories get the minimum training frequency (not 0).
    """
    freq_map  = X_train_col.value_counts(normalize=True)
    min_freq  = freq_map.min()   # fallback for unseen categories
    return X_apply_col.map(freq_map).fillna(min_freq)

X_train_freq = X_train.copy()
X_test_freq  = X_test.copy()

for col in ['restaurant', 'city', 'time_slot']:
    X_train_freq[f'{col}_freq'] = frequency_encode(X_train[col], X_train[col])
    X_test_freq[f'{col}_freq']  = frequency_encode(X_train[col], X_test[col])

# Inspect what frequencies were assigned
print("Restaurant frequency encoding (training set):")
freq_table = (
    X_train[['restaurant']]
    .assign(freq=X_train_freq['restaurant_freq'])
    .drop_duplicates()
    .sort_values('freq', ascending=False)
)
for _, row in freq_table.iterrows():
    bar = '█' * int(row['freq'] * 200)
    print(f"  {row['restaurant']:<20}: {bar} {row['freq']:.4f}")

# ── sklearn-compatible FrequencyEncoder ───────────────────────────────
class FrequencyEncoder(BaseEstimator, TransformerMixin):
    """
    Frequency encoder compatible with sklearn Pipeline.
    Fits frequency maps on training data, applies to any dataset.
    """
    def __init__(self, min_frequency: float = 1e-4):
        self.min_frequency = min_frequency
        self.freq_maps_: dict = {}

    def fit(self, X: pd.DataFrame, y=None):
        X = pd.DataFrame(X)
        for col in X.columns:
            self.freq_maps_[col] = X[col].value_counts(normalize=True).to_dict()
        return self

    def transform(self, X: pd.DataFrame) -> np.ndarray:
        X = pd.DataFrame(X)
        result = np.zeros((len(X), len(X.columns)), dtype=np.float32)
        for i, col in enumerate(X.columns):
            freq_map = self.freq_maps_.get(col, {})
            result[:, i] = X[col].map(freq_map).fillna(self.min_frequency).values
        return result

# Use in a pipeline
from sklearn.pipeline import Pipeline
from sklearn.preprocessing import StandardScaler
from sklearn.linear_model import Ridge
from sklearn.compose import ColumnTransformer
from sklearn.impute import SimpleImputer
from sklearn.metrics import mean_absolute_error

NUM_COLS = ['distance_km', 'traffic_score', 'restaurant_prep', 'order_value']
CAT_COLS = ['restaurant', 'city', 'time_slot']

preprocessor = ColumnTransformer([
    ('num', Pipeline([
        ('imp',   SimpleImputer(strategy='median')),
        ('scale', StandardScaler()),
    ]), NUM_COLS),
    ('cat_freq', FrequencyEncoder(), CAT_COLS),
])

pipe_freq = Pipeline([
    ('prep',  preprocessor),
    ('model', Ridge(alpha=1.0)),
])
pipe_freq.fit(X_train, y_train)
mae_freq = mean_absolute_error(y_test, pipe_freq.predict(X_test))
print(f"
Pipeline with frequency encoding — MAE: {mae_freq:.4f} min")

The two safe approaches:

import numpy as np
import pandas as pd
from sklearn.model_selection import KFold
from sklearn.linear_model import Ridge
from sklearn.metrics import mean_absolute_error
from sklearn.preprocessing import StandardScaler

# ── Cross-fold target encoding — the correct manual implementation ────

def target_encode_cv(
    X_train: pd.DataFrame,
    y_train: pd.Series,
    X_test:  pd.DataFrame,
    col:     str,
    n_folds: int  = 5,
    smoothing: float = 10.0,
) -> tuple:
    """
    Safe target encoding via K-fold cross-validation.

    For each row in training: compute category mean using OTHER folds only.
    For test set: use all training data to compute the encoding.
    Smoothing: blend category mean with global mean for rare categories.
      encoded = (n * cat_mean + m * global_mean) / (n + m)
      where n = count of category in fold, m = smoothing parameter
    """
    global_mean   = y_train.mean()
    encoded_train = pd.Series(np.nan, index=X_train.index)
    kf            = KFold(n_splits=n_folds, shuffle=True, random_state=42)

    for fold_tr_idx, fold_val_idx in kf.split(X_train):
        fold_X = X_train.iloc[fold_tr_idx]
        fold_y = y_train.iloc[fold_tr_idx]

        # Compute smoothed mean per category on this fold's training portion
        stats = pd.DataFrame({
            'count': fold_y.groupby(fold_X[col].values).count(),
            'mean':  fold_y.groupby(fold_X[col].values).mean(),
        })
        stats['smoothed'] = (
            (stats['count'] * stats['mean'] + smoothing * global_mean)
            / (stats['count'] + smoothing)
        )
        # Apply to the validation rows of this fold
        val_cats = X_train.iloc[fold_val_idx][col]
        encoded_train.iloc[fold_val_idx] = val_cats.map(stats['smoothed']).fillna(global_mean)

    # Test encoding: use all training data
    all_stats = pd.DataFrame({
        'count': y_train.groupby(X_train[col].values).count(),
        'mean':  y_train.groupby(X_train[col].values).mean(),
    })
    all_stats['smoothed'] = (
        (all_stats['count'] * all_stats['mean'] + smoothing * global_mean)
        / (all_stats['count'] + smoothing)
    )
    encoded_test = X_test[col].map(all_stats['smoothed']).fillna(global_mean)

    return encoded_train, encoded_test

# Encode restaurant and city
rest_tr, rest_te = target_encode_cv(X_train, y_train, X_test, 'restaurant')
city_tr, city_te = target_encode_cv(X_train, y_train, X_test, 'city')

X_train_te = X_train[['distance_km','traffic_score','restaurant_prep','order_value']].copy()
X_test_te  = X_test[['distance_km','traffic_score','restaurant_prep','order_value']].copy()
X_train_te['restaurant_te'] = rest_tr.values
X_test_te['restaurant_te']  = rest_te.values
X_train_te['city_te']       = city_tr.values
X_test_te['city_te']        = city_te.values

sc = StandardScaler()
model_te = Ridge(alpha=1.0)
model_te.fit(sc.fit_transform(X_train_te), y_train)
mae_te = mean_absolute_error(y_test, model_te.predict(sc.transform(X_test_te)))
print(f"Target encoding (cross-fold) — MAE: {mae_te:.4f} min")

# What the encoding learned
print("
Target-encoded restaurant values (≈ mean delivery time per restaurant):")
enc_vals = dict(zip(X_train['restaurant'], rest_tr.values))
# Deduplicate — take first occurrence per restaurant
seen = {}
for rest, val in enc_vals.items():
    if rest not in seen:
        seen[rest] = val
for rest, val in sorted(seen.items(), key=lambda x: x[1]):
    print(f"  {rest:<20}: {val:.1f} min")

# ── sklearn TargetEncoder — available since sklearn 1.3 ───────────────
try:
    from sklearn.preprocessing import TargetEncoder
    te = TargetEncoder(cv=5, smooth='auto', random_state=42)
    # TargetEncoder handles cross-fold internally — safe to use directly
    X_tr_sk_te = te.fit_transform(X_train[['restaurant','city']], y_train)
    X_te_sk_te = te.transform(X_test[['restaurant','city']])
    print(f"
sklearn TargetEncoder output shape: {X_tr_sk_te.shape}")
except ImportError:
    print("
sklearn TargetEncoder requires sklearn >= 1.3")

import numpy as np

# Demonstrating the effect of smoothing
global_mean = 36.5   # overall mean delivery time

# Restaurant A: 500 orders, mean delivery 42 min
# Restaurant B: 3 orders, mean delivery 55 min (noisy estimate)

def smoothed_mean(cat_mean, cat_count, global_mean, m=10):
    return (cat_count * cat_mean + m * global_mean) / (cat_count + m)

for name, count, cat_mean in [
    ('Restaurant A (500 orders)', 500, 42.0),
    ('Restaurant B (3 orders)',     3, 55.0),
    ('Restaurant C (1 order)',       1, 70.0),
]:
    for m in [1, 5, 10, 50]:
        enc = smoothed_mean(cat_mean, count, global_mean, m)
        print(f"  {name:<30} m={m:<4}: {enc:.2f}")
    print()

# Restaurant A: 500 samples → smoothing barely affects it, stays near 42
# Restaurant B: 3 samples   → smoothing pulls it toward global mean 36.5
# Restaurant C: 1 sample    → smoothing almost entirely uses global mean
# This prevents the model from memorising rare-category noise

import numpy as np
import pandas as pd

# ── Manual binary encoding ────────────────────────────────────────────
def binary_encode(X_train_col: pd.Series,
                  X_apply_col: pd.Series) -> pd.DataFrame:
    """
    Binary encode a categorical column.
    Assigns integers in order of frequency (most common = 1).
    Converts integers to binary bit columns.
    """
    # Map categories to integers (fit on training)
    freq_order = X_train_col.value_counts().index.tolist()
    cat_to_int = {cat: i + 1 for i, cat in enumerate(freq_order)}
    n_bits     = max(1, int(np.ceil(np.log2(len(cat_to_int) + 1))))

    # Apply mapping (unknown categories → 0)
    int_encoded = X_apply_col.map(cat_to_int).fillna(0).astype(int)

    # Convert integers to binary columns
    bit_cols = {}
    for bit in range(n_bits):
        bit_cols[f'{X_train_col.name}_bit{bit}'] = (int_encoded >> bit) & 1

    return pd.DataFrame(bit_cols, index=X_apply_col.index)

# Encode restaurant (8 values → 4 binary columns instead of 7 OHE columns)
rest_bin_train = binary_encode(X_train['restaurant'], X_train['restaurant'])
rest_bin_test  = binary_encode(X_train['restaurant'], X_test['restaurant'])

print(f"Restaurant binary encoding:")
print(f"  OHE would create:    7 columns (drop=first, 8 categories)")
print(f"  Binary creates:      {rest_bin_train.shape[1]} columns")
print(f"  For 100 categories:  OHE=99 cols, Binary=7 cols")
print()

# Show the encoding
sample_rest = pd.DataFrame({'restaurant': X_train['restaurant'].unique()[:5]})
encoded = binary_encode(X_train['restaurant'], sample_rest['restaurant'])
print("Sample binary encoding:")
print(pd.concat([sample_rest, encoded], axis=1).to_string(index=False))

# ── category_encoders library — full encoding toolkit ─────────────────
# pip install category_encoders
# Has: BinaryEncoder, TargetEncoder, HashingEncoder, WOEEncoder, etc.
try:
    import category_encoders as ce
    be = ce.BinaryEncoder(cols=['restaurant'], drop_invariant=True)
    X_tr_be = be.fit_transform(X_train[['restaurant', 'city']], y_train)
    X_te_be = be.transform(X_test[['restaurant', 'city']])
    print(f"
category_encoders BinaryEncoder output shape: {X_tr_be.shape}")
except ImportError:
    print("
category_encoders not installed: pip install category_encoders")

# Encoding benchmark: compare all strategies on the same dataset
import numpy as np
import pandas as pd
from sklearn.pipeline import Pipeline
from sklearn.compose import ColumnTransformer
from sklearn.preprocessing import StandardScaler, OneHotEncoder, OrdinalEncoder
from sklearn.impute import SimpleImputer
from sklearn.linear_model import Ridge
from sklearn.metrics import mean_absolute_error

NUM_COLS = ['distance_km', 'traffic_score', 'restaurant_prep', 'order_value']

def make_pipeline(cat_transformer, cat_cols):
    return Pipeline([
        ('prep', ColumnTransformer([
            ('num', Pipeline([
                ('imp',   SimpleImputer(strategy='median')),
                ('scale', StandardScaler()),
            ]), NUM_COLS),
            ('cat', cat_transformer, cat_cols),
        ])),
        ('model', Ridge(alpha=1.0)),
    ])

CAT_COLS = ['restaurant', 'city', 'time_slot']

strategies = {
    'OHE (drop=first)': make_pipeline(
        OneHotEncoder(sparse_output=False, handle_unknown='ignore', drop='first'),
        CAT_COLS,
    ),
    'OHE (no drop)': make_pipeline(
        OneHotEncoder(sparse_output=False, handle_unknown='ignore'),
        CAT_COLS,
    ),
    'Ordinal (alphabetical)': make_pipeline(
        OrdinalEncoder(handle_unknown='use_encoded_value', unknown_value=-1),
        CAT_COLS,
    ),
    'Frequency': make_pipeline(FrequencyEncoder(), CAT_COLS),
}

print("Encoding strategy comparison (Ridge regression):")
print(f"{'Strategy':<30} {'Test MAE (min)'}")
print("─" * 48)
for name, pipe in strategies.items():
    pipe.fit(X_train, y_train)
    mae = mean_absolute_error(y_test, pipe.predict(X_test))
    print(f"  {name:<28}: {mae:.4f}")

# Add target encoding result (computed earlier)
print(f"  {'Target (cross-fold)':<28}: {mae_te:.4f}")
print()
print("Note: for tree models, ordinal and OHE perform similarly.")

import numpy as np
import pandas as pd
from sklearn.preprocessing import OneHotEncoder, OrdinalEncoder

# Simulate: training had 8 restaurants, production has a 9th
training_restaurants = ['Pizza Hut', 'Biryani Blues', "McDonald's",
                         "Haldiram's", 'Dominos', 'KFC', 'Subway', 'Burger King']
new_restaurant = ['Cloud Kitchen XYZ']   # never seen in training

# ── OneHotEncoder(handle_unknown='ignore') ────────────────────────────
# Unknown → all zeros for that feature's columns (safe, model uses baseline)
ohe_safe = OneHotEncoder(handle_unknown='ignore', sparse_output=False, drop='first')
ohe_safe.fit(pd.DataFrame({'restaurant': training_restaurants}))

new_row = pd.DataFrame({'restaurant': new_restaurant})
encoded = ohe_safe.transform(new_row)
print("OHE with handle_unknown='ignore':")
print(f"  Unknown restaurant → {encoded[0]}  (all zeros)")
print(f"  Model predicts as if no restaurant was specified (baseline intercept)")

# ── OrdinalEncoder with unknown_value ─────────────────────────────────
ord_safe = OrdinalEncoder(handle_unknown='use_encoded_value', unknown_value=-1)
ord_safe.fit(pd.DataFrame({'restaurant': training_restaurants}))
encoded_ord = ord_safe.transform(new_row)
print(f"
OrdinalEncoder with unknown_value=-1:")
print(f"  Unknown restaurant → {encoded_ord[0]}")
print(f"  -1 signals 'unknown' — tree models can learn to handle this")

# ── Target encoding: fallback to global mean ──────────────────────────
# In our cross-fold implementation: fillna(global_mean) handles unknowns
global_mean = float(y_train.mean())
print(f"
Target encoding with global mean fallback:")
print(f"  Unknown restaurant → {global_mean:.2f} (overall mean delivery time)")
print(f"  Model predicts average delivery — conservative, reasonable")

# ── Best practice: add an 'other' category during training ────────────
# If you know rare categories exist, create a catch-all bucket during training
def collapse_rare_categories(series: pd.Series,
                              min_count: int = 50,
                              other_label: str = 'OTHER') -> pd.Series:
    """
    Replace categories with fewer than min_count occurrences with OTHER.
    Ensures OHE always has an 'other' bucket for rare/unseen values.
    """
    counts   = series.value_counts()
    rare     = counts[counts < min_count].index
    return series.replace(rare, other_label)

X_train_coll = X_train.copy()
X_test_coll  = X_test.copy()
X_train_coll['restaurant'] = collapse_rare_categories(X_train['restaurant'], min_count=300)
X_test_coll['restaurant']  = X_test['restaurant'].apply(
    lambda x: x if x in X_train_coll['restaurant'].unique() else 'OTHER'
)

n_other_train = (X_train_coll['restaurant'] == 'OTHER').sum()
print(f"
After collapse_rare_categories (min_count=300):")
print(f"  Rows collapsed to OTHER: {n_other_train:,}")
print(f"  Unique values remaining: {X_train_coll['restaurant'].nunique()}")

import joblib
from sklearn.pipeline import Pipeline
from sklearn.compose import ColumnTransformer
from sklearn.preprocessing import OneHotEncoder, StandardScaler
from sklearn.linear_model import Ridge

NUM_COLS = ['distance_km', 'traffic_score', 'restaurant_prep', 'order_value']
CAT_COLS = ['restaurant', 'city', 'time_slot']

pipeline = Pipeline([
    ('prep', ColumnTransformer([
        ('num', StandardScaler(), NUM_COLS),
        ('cat', OneHotEncoder(handle_unknown='ignore', drop='first'), CAT_COLS),
    ])),
    ('model', Ridge(alpha=1.0)),
])
pipeline.fit(X_train, y_train)

# The fitted encoder's category list is frozen the moment fit() returns —
# persist encoder + model together, exactly like the scaler in Module 12
joblib.dump(pipeline, 'delivery_model_v9.joblib')

# ── Six months later — the platform has added 40 new restaurants ──────
# None of them existed when this pipeline was fit. Loading the OLD
# pipeline and scoring a request for one of the new restaurants still
# works, because handle_unknown='ignore' maps it to an all-zero row:
loaded = joblib.load('delivery_model_v9.joblib')
new_restaurant_order = [[3.2, 7.0, 15.0, 210.0, 'Cloud Kitchen 47', 'Austin', 'lunch']]
# The model does not error out — it silently falls back to treating this
# order as if no specific restaurant were known, which is SAFE but is
# also a real loss of signal for every one of those 40 new restaurants.
# This is exactly why growing platforms retrain on a schedule rather
# than waiting for accuracy to visibly degrade — every day between
# retraining runs, the newest restaurants get the least informative
# encoding the model has available.

One-hot encoding is safe specifically for low-cardinality nominal features feeding a linear model, SVM, or neural network — that is a narrower claim than "always safe." At high cardinality it produces a huge, mostly-empty feature matrix that slows training, gives rare categories almost no data to learn from, and in some production systems is simply too expensive to serve. It is also unnecessary overhead for tree-based models, which can consume integer-encoded categories directly. "Safe default" describes a specific, common situation, not a universal rule for every categorical column you will ever encounter.

Label encoding is fine for tree-based models because a tree only ever asks "is this value above or below a threshold," and it can learn to carve out any arbitrary subset of integers it needs regardless of the order you assigned them in. A linear model, an SVM, or a neural network instead treats the encoded integer as a literal number in an equation — it implicitly assumes that a category coded as 6 is "more" of something than a category coded as 2, and will learn a relationship based on that false ordering. The same encoding that is harmless for one model family actively teaches wrong relationships to another.

The leakage in target encoding is far more subtle than accidentally including the label column. If you compute each category's mean target value using the entire training set and then use that mean as a feature for every row in that category — including the very rows whose own target values were used to compute it — each row's encoded feature contains a sliver of information about its own label. No column named "delivery_time" appears in X, but the leakage is there anyway. Safe target encoding requires cross-fold computation (or sklearn's built-in TargetEncoder), not just visually checking that the target column is absent.

It solves the crash — without it, a genuinely new category value raises an error and the request fails outright, which is worse. But handle_unknown='ignore' encodes an unseen category as all zeros, which the model interprets as "none of the categories I know about," effectively discarding that feature's information for that row. That is a safe fallback, not a correct one: a brand-new restaurant or a first-time customer gets treated as generically as possible precisely when more specific information might matter most. Preventing a crash and preserving signal are two different problems, and this only solves the first one.

Scaling up one-hot encoding is not a matter of throwing more compute at the same approach — it changes the nature of the problem. At extreme cardinality, most categories appear in only a handful of training rows, so their corresponding one-hot column has almost no signal to learn a weight from, and the resulting feature matrix becomes enormous and overwhelmingly sparse for very little benefit. The right response at that scale is usually a completely different technique — frequency encoding, target encoding, hashing, or a learned embedding — not a larger version of the same one-hot approach that worked at low cardinality.

Without any handling, OneHotEncoder and OrdinalEncoder both raise an error by default the moment they see a category they were not fit on, which takes down the request. Setting handle_unknown='ignore' on OneHotEncoder maps the unseen category to an all-zero row instead of crashing; OrdinalEncoder can be configured with unknown_value=-1 to give tree models an explicit "unknown" signal to split on. For target encoding, the safe fallback is the global target mean. I would set one of these fallbacks explicitly rather than relying on defaults, and I would also track how often unknown categories actually occur in production as a signal for when the model needs retraining.

I would reach for target encoding when the categorical feature has high cardinality and a strong, genuine relationship with the target — one-hot encoding at that cardinality produces a huge sparse matrix with too little data per category, while target encoding compresses each category into a single, highly informative number. The risk is leakage: if the encoding for a row uses that row's own target value in computing its category's mean, the model partially memorises training labels instead of learning a generalisable pattern, which looks great in training and falls apart in production. The fix is cross-fold encoding or sklearn's built-in TargetEncoder, never a plain groupby-mean computed on the full training set.

I would rule out plain one-hot encoding immediately — a 200,000-column sparse matrix is both computationally expensive and starves most categories of enough data to learn anything reliable. For a linear or tree-based model, I would reach for frequency encoding as a cheap, leakage-free baseline, or cross-fold target encoding if the feature has a strong relationship with the target and I can afford the leakage-prevention machinery. For a neural network, I would instead use a learned embedding table, mapping each product ID to a small dense vector trained jointly with the rest of the model — this is the standard approach in recommendation and ranking systems with IDs at this scale.

A linear model multiplies the encoded integer by a learned coefficient, so it is implicitly assuming the numeric value is meaningful — that a category encoded as 6 contributes twice as much as a category encoded as 3. For a nominal feature with no real order, that assumption is simply false, and the model learns a spurious relationship baked in by the arbitrary integer assignment. A tree-based model instead only ever asks whether a value is above or below some threshold at each split, and can carve out any subset of category codes it needs regardless of their numeric order, so an arbitrary integer assignment costs it nothing.

I would compute each category's encoded value using k-fold cross-validation: for every row, the category mean used to encode it comes only from the folds that row is not part of, never from a mean that includes its own target value. For unseen categories or those with very few samples, I would blend the category mean with the global mean (smoothing) so a category with three noisy samples does not get encoded as if those three samples were a reliable estimate. To verify it worked, I would check that cross-validation performance and holdout test performance are close to each other — a big gap where training performance looks great but validation performance is much worse is the classic signature that leakage crept back in.