Decision Trees — Loan Approval at Capital One

What this module covers:

import numpy as np
import pandas as pd
from sklearn.model_selection import train_test_split

np.random.seed(42)
n = 10_000

# ── Capital One loan applicant dataset ────────────────────────────────────────
credit_score    = np.random.randint(300, 900, n)
monthly_income  = np.abs(np.random.normal(55_000, 25_000, n)).clip(8_000, 300_000)
existing_emi    = np.abs(np.random.normal(8_000, 6_000, n)).clip(0, 80_000)
loan_amount     = np.abs(np.random.normal(500_000, 300_000, n)).clip(50_000, 5_000_000)
employment_yrs  = np.abs(np.random.normal(5, 4, n)).clip(0, 35)
loan_to_income  = loan_amount / (monthly_income * 12)

# Approval logic — mirrors real credit scoring
score = (
    (credit_score / 900) * 40          # credit score: 40% weight
    + (monthly_income / 300_000) * 25   # income: 25%
    - (existing_emi / monthly_income) * 20  # EMI burden: -20%
    - (loan_to_income) * 10             # loan-to-income: -10%
    + (employment_yrs / 35) * 5         # employment stability: 5%
    + np.random.randn(n) * 0.05         # noise
)
approved = (score > 0.45).astype(int)

df = pd.DataFrame({
    'credit_score':   credit_score,
    'monthly_income': monthly_income.round(0),
    'existing_emi':   existing_emi.round(0),
    'loan_amount':    loan_amount.round(0),
    'employment_yrs': employment_yrs.round(1),
    'loan_to_income': loan_to_income.round(3),
    'approved':       approved,
})

print(f"Dataset: {len(df):,} applications")
print(f"Approval rate: {approved.mean()*100:.1f}%")
print(df.describe().round(1).to_string())

X = df.drop(columns=['approved'])
y = df['approved']
X_train, X_test, y_train, y_test = train_test_split(
    X, y, test_size=0.2, stratify=y, random_state=42
)
print(f"
Train: {len(X_train):,}  Test: {len(X_test):,}")

import numpy as np

def gini_impurity(y: np.ndarray) -> float:
    """
    Gini impurity for a node.
    = 1 - sum(p_i^2) for each class i
    = 0 when all samples belong to one class (pure node)
    = 0.5 when samples are split 50/50 (maximally impure, binary case)
    """
    n = len(y)
    if n == 0:
        return 0.0
    classes, counts = np.unique(y, return_counts=True)
    probs   = counts / n
    return 1.0 - np.sum(probs ** 2)

# Verify on Capital One loan scenarios
print("Gini impurity examples:")
scenarios = [
    ("All rejected (pure)",      np.array([0]*100)),
    ("90% rejected",             np.array([0]*90 + [1]*10)),
    ("70% rejected",             np.array([0]*70 + [1]*30)),
    ("50/50 split (max impure)", np.array([0]*50 + [1]*50)),
    ("All approved (pure)",      np.array([1]*100)),
]
for name, y_node in scenarios:
    g = gini_impurity(y_node)
    bar = '█' * int(g * 40)
    print(f"  {name:<30}: {bar} {g:.3f}")

def weighted_gini(y_left: np.ndarray, y_right: np.ndarray) -> float:
    """
    Weighted Gini impurity after a split.
    = (n_left/n_total)*Gini(left) + (n_right/n_total)*Gini(right)
    The tree minimises this across all possible splits.
    """
    n = len(y_left) + len(y_right)
    return (len(y_left)/n)  * gini_impurity(y_left) +            (len(y_right)/n) * gini_impurity(y_right)

# Find best split for credit_score on training data
print("
Finding best credit_score split threshold:")
y_arr     = y_train.values
scores_arr = X_train['credit_score'].values

best_threshold, best_gini = None, np.inf
thresholds = np.unique(scores_arr)[::10]   # sample every 10th unique value

for t in thresholds:
    left  = y_arr[scores_arr <= t]
    right = y_arr[scores_arr >  t]
    if len(left) == 0 or len(right) == 0:
        continue
    wg = weighted_gini(left, right)
    if wg < best_gini:
        best_gini, best_threshold = wg, t

print(f"  Best threshold: credit_score <= {best_threshold}")
print(f"  Weighted Gini:  {best_gini:.4f}")
print(f"  Parent Gini:    {gini_impurity(y_arr):.4f}")
print(f"  Gini reduction: {gini_impurity(y_arr) - best_gini:.4f}")

import numpy as np

def entropy(y: np.ndarray) -> float:
    """Shannon entropy. H = -sum(p_i * log2(p_i))"""
    n = len(y)
    if n == 0: return 0.0
    _, counts = np.unique(y, return_counts=True)
    probs = counts / n
    probs = probs[probs > 0]
    return -np.sum(probs * np.log2(probs))

def information_gain(y_parent, y_left, y_right) -> float:
    """IG = H(parent) - weighted_H(children)"""
    n = len(y_parent)
    weighted_child = (
        (len(y_left)  / n) * entropy(y_left) +
        (len(y_right) / n) * entropy(y_right)
    )
    return entropy(y_parent) - weighted_child

y_arr = y_train.values
scores_arr = X_train['credit_score'].values
left  = y_arr[scores_arr <= best_threshold]
right = y_arr[scores_arr >  best_threshold]

print(f"Entropy of parent node:     {entropy(y_arr):.4f} bits")
print(f"Entropy of left child:      {entropy(left):.4f} bits")
print(f"Entropy of right child:     {entropy(right):.4f} bits")
print(f"Information gain:           {information_gain(y_arr, left, right):.4f} bits")

# Gini vs Entropy — comparison on same split
gini_parent = gini_impurity(y_arr)
gini_after  = weighted_gini(left, right)
print(f"
Gini reduction:             {gini_parent - gini_after:.4f}")
print(f"Info gain:                  {information_gain(y_arr, left, right):.4f}")
print("Both criteria select the same split — just different scales")

import numpy as np
from dataclasses import dataclass, field
from typing import Optional

@dataclass
class TreeNode:
    """One node in the decision tree."""
    # Leaf node attributes
    is_leaf:      bool  = False
    prediction:   int   = 0       # class label (leaf only)
    probability:  float = 0.0     # P(class=1) at this leaf

    # Split node attributes
    feature_idx:  int   = 0       # which feature to split on
    threshold:    float = 0.0     # split value: left if x <= threshold
    gini:         float = 0.0     # impurity before split
    n_samples:    int   = 0       # samples reaching this node
    left:  Optional['TreeNode'] = field(default=None)
    right: Optional['TreeNode'] = field(default=None)


class DecisionTreeScratch:
    """
    Binary decision tree classifier built from scratch.
    Splitting criterion: Gini impurity minimisation.
    """

    def __init__(self, max_depth: int = 5, min_samples_leaf: int = 20):
        self.max_depth        = max_depth
        self.min_samples_leaf = min_samples_leaf
        self.root: Optional[TreeNode] = None
        self.feature_names: list = []

    # ── Gini helpers ────────────────────────────────────────────────────
    @staticmethod
    def _gini(y: np.ndarray) -> float:
        if len(y) == 0: return 0.0
        _, counts = np.unique(y, return_counts=True)
        probs = counts / len(y)
        return 1.0 - np.sum(probs ** 2)

    def _best_split(self, X: np.ndarray, y: np.ndarray):
        """Find the feature and threshold that minimises weighted Gini."""
        best_feat, best_thresh, best_gini = None, None, np.inf

        for feat in range(X.shape[1]):
            col        = X[:, feat]
            thresholds = np.unique(col)
            # Sample at most 50 candidate thresholds for speed
            if len(thresholds) > 50:
                thresholds = np.percentile(col, np.linspace(0, 100, 50))

            for t in thresholds:
                left_mask  = col <= t
                right_mask = ~left_mask
                if left_mask.sum() < self.min_samples_leaf:  continue
                if right_mask.sum() < self.min_samples_leaf: continue

                n = len(y)
                wg = (left_mask.sum()/n)  * self._gini(y[left_mask]) +                      (right_mask.sum()/n) * self._gini(y[right_mask])

                if wg < best_gini:
                    best_gini  = wg
                    best_feat  = feat
                    best_thresh = t

        return best_feat, best_thresh, best_gini

    def _build(self, X: np.ndarray, y: np.ndarray, depth: int) -> TreeNode:
        node = TreeNode(n_samples=len(y), gini=self._gini(y))
        n_pos = y.sum()
        node.probability  = n_pos / len(y)
        node.prediction   = int(n_pos >= len(y) / 2)

        # Stopping conditions → leaf node
        if (depth >= self.max_depth
                or len(y) < 2 * self.min_samples_leaf
                or self._gini(y) == 0.0):
            node.is_leaf = True
            return node

        feat, thresh, wg = self._best_split(X, y)

        # No valid split found → leaf
        if feat is None:
            node.is_leaf = True
            return node

        node.feature_idx = feat
        node.threshold   = thresh

        mask = X[:, feat] <= thresh
        node.left  = self._build(X[mask],  y[mask],  depth + 1)
        node.right = self._build(X[~mask], y[~mask], depth + 1)
        return node

    def fit(self, X: np.ndarray, y: np.ndarray,
            feature_names: list = None):
        self.feature_names = feature_names or [f'f{i}' for i in range(X.shape[1])]
        self.root = self._build(X, y, depth=0)
        return self

    def _predict_one(self, x: np.ndarray, node: TreeNode) -> float:
        if node.is_leaf:
            return node.probability
        if x[node.feature_idx] <= node.threshold:
            return self._predict_one(x, node.left)
        return self._predict_one(x, node.right)

    def predict_proba(self, X: np.ndarray) -> np.ndarray:
        return np.array([self._predict_one(x, self.root) for x in X])

    def predict(self, X: np.ndarray, threshold: float = 0.5) -> np.ndarray:
        return (self.predict_proba(X) >= threshold).astype(int)

    def score(self, X: np.ndarray, y: np.ndarray) -> float:
        return (self.predict(X) == y).mean()

    def print_tree(self, node: TreeNode = None, depth: int = 0, prefix: str = ''):
        if node is None: node = self.root
        indent = '  ' * depth
        if node.is_leaf:
            cls = 'APPROVE' if node.prediction == 1 else 'REJECT'
            print(f"{indent}{prefix}→ {cls} "
                  f"(p={node.probability:.2f}, n={node.n_samples})")
        else:
            fname = self.feature_names[node.feature_idx]
            print(f"{indent}{prefix}[{fname} <= {node.threshold:.1f}]  "
                  f"gini={node.gini:.3f}  n={node.n_samples}")
            self.print_tree(node.left,  depth+1, 'L ')
            self.print_tree(node.right, depth+1, 'R ')


# ── Train and evaluate ────────────────────────────────────────────────
from sklearn.metrics import classification_report

tree_scratch = DecisionTreeScratch(max_depth=4, min_samples_leaf=50)
tree_scratch.fit(
    X_train.values, y_train.values,
    feature_names=X_train.columns.tolist()
)

print("Decision tree structure:")
tree_scratch.print_tree()
print()

from sklearn.metrics import accuracy_score
y_pred_scratch = tree_scratch.predict(X_test.values)
print(f"From-scratch tree accuracy: {accuracy_score(y_test, y_pred_scratch):.4f}")

from sklearn.tree import DecisionTreeClassifier
from sklearn.metrics import accuracy_score, roc_auc_score
import numpy as np

# ── Demonstrate overfitting as depth increases ────────────────────────
print(f"{'max_depth':<12} {'Train acc':<12} {'Test acc':<12} {'AUC':<10} {'n_leaves'}")
print("─" * 58)

for depth in [1, 2, 3, 4, 5, 7, 10, 15, None]:
    dt = DecisionTreeClassifier(
        max_depth=depth,
        min_samples_leaf=1,
        random_state=42,
    )
    dt.fit(X_train, y_train)
    tr_acc = accuracy_score(y_train, dt.predict(X_train))
    te_acc = accuracy_score(y_test,  dt.predict(X_test))
    auc    = roc_auc_score(y_test, dt.predict_proba(X_test)[:, 1])
    leaves = dt.get_n_leaves()
    gap    = " ← overfit" if tr_acc - te_acc > 0.03 else ""
    print(f"  {str(depth):<10} {tr_acc:<12.4f} {te_acc:<12.4f} {auc:<10.4f} {leaves}{gap}")

# Output pattern:
# depth=1:  train=0.78  test=0.77  — underfitting, too simple
# depth=4:  train=0.89  test=0.88  — good generalisation
# depth=10: train=0.99  test=0.83  — overfitting, memorising noise
# depth=None: train=1.00 test=0.79 — severe overfitting

# ── Cost-complexity pruning — finding optimal ccp_alpha ────────────────
from sklearn.model_selection import cross_val_score

dt_full = DecisionTreeClassifier(random_state=42)
dt_full.fit(X_train, y_train)

# Get the pruning path — effective alphas and corresponding impurities
path = dt_full.cost_complexity_pruning_path(X_train, y_train)
ccp_alphas = path.ccp_alphas[::5]   # sample every 5th value

print(f"
CCP alpha search ({len(ccp_alphas)} values):")
best_alpha, best_auc = 0.0, 0.0
for alpha in ccp_alphas:
    dt = DecisionTreeClassifier(ccp_alpha=alpha, random_state=42)
    scores = cross_val_score(dt, X_train, y_train, cv=5, scoring='roc_auc')
    mean_auc = scores.mean()
    if mean_auc > best_auc:
        best_auc, best_alpha = mean_auc, alpha
    print(f"  alpha={alpha:.6f}: CV AUC={mean_auc:.4f}")

print(f"
Best ccp_alpha: {best_alpha:.6f}  (CV AUC={best_auc:.4f})")

# ── Final model with best hyperparameters ──────────────────────────────
final_tree = DecisionTreeClassifier(
    max_depth=5,
    min_samples_leaf=30,
    ccp_alpha=best_alpha,
    random_state=42,
)
final_tree.fit(X_train, y_train)
final_auc = roc_auc_score(y_test, final_tree.predict_proba(X_test)[:, 1])
print(f"
Final model — Test AUC: {final_auc:.4f}  "
      f"Leaves: {final_tree.get_n_leaves()}")

import numpy as np
import pandas as pd
from sklearn.tree import DecisionTreeClassifier, export_text, plot_tree

# Train final model
dt = DecisionTreeClassifier(max_depth=5, min_samples_leaf=30, random_state=42)
dt.fit(X_train, y_train)

# ── Feature importances ────────────────────────────────────────────────
importances = pd.Series(
    dt.feature_importances_,
    index=X_train.columns
).sort_values(ascending=False)

print("Feature importances (Gini-based):")
for feat, imp in importances.items():
    bar = '█' * int(imp * 50)
    print(f"  {feat:<20}: {bar} {imp:.4f}")

# ── Text representation of the tree ───────────────────────────────────
print("
Tree structure (text):")
print(export_text(dt, feature_names=X_train.columns.tolist(), max_depth=3))

# ── Trace one decision path ────────────────────────────────────────────
# sklearn's decision_path gives the nodes each sample passes through
sample_idx = 0
sample     = X_test.iloc[[sample_idx]]
actual     = y_test.iloc[sample_idx]
pred_proba = dt.predict_proba(sample)[0, 1]
pred_class = dt.predict(sample)[0]

print(f"
Trace for applicant {sample_idx}:")
print(f"  credit_score:   {sample['credit_score'].values[0]}")
print(f"  monthly_income: ${sample['monthly_income'].values[0]:,.0f}")
print(f"  existing_emi:   ${sample['existing_emi'].values[0]:,.0f}")
print(f"  Actual:         {'APPROVE' if actual == 1 else 'REJECT'}")
print(f"  Predicted:      {'APPROVE' if pred_class == 1 else 'REJECT'} "
      f"(confidence: {pred_proba:.2%})")

# Decision path node indices
node_indicator = dt.decision_path(sample)
node_ids       = node_indicator.indices
tree_          = dt.tree_
feature        = tree_.feature
threshold      = tree_.threshold
feat_names     = X_train.columns.tolist()

print(f"
  Decision path ({len(node_ids)-1} splits):")
for node_id in node_ids[:-1]:   # exclude leaf
    feat_idx  = feature[node_id]
    thresh    = threshold[node_id]
    val       = sample.iloc[0, feat_idx]
    direction = "<=" if val <= thresh else ">"
    print(f"    Node {node_id}: {feat_names[feat_idx]} "
          f"({val:.1f}) {direction} {thresh:.1f}")

from sklearn.tree import DecisionTreeRegressor
from sklearn.metrics import mean_absolute_error, r2_score
import numpy as np

# Regression target: predict exact loan interest rate offered (continuous)
np.random.seed(42)
interest_rate = (
    12.0
    - (X_train['credit_score'] / 900) * 4.0   # better score = lower rate
    + (X_train['loan_to_income']) * 2.0         # higher LTI = higher rate
    + (X_train['existing_emi'] / X_train['monthly_income']) * 3.0
    + np.random.randn(len(X_train)) * 0.5       # noise
).clip(7.5, 18.0)

interest_rate_test = (
    12.0
    - (X_test['credit_score'] / 900) * 4.0
    + (X_test['loan_to_income']) * 2.0
    + (X_test['existing_emi'] / X_test['monthly_income']) * 3.0
    + np.random.randn(len(X_test)) * 0.5
).clip(7.5, 18.0)

# ── DecisionTreeRegressor ─────────────────────────────────────────────
print(f"{'max_depth':<12} {'Train MAE':<13} {'Test MAE':<13} {'Test R²'}")
print("─" * 50)

for depth in [2, 3, 4, 5, 8, None]:
    dtr = DecisionTreeRegressor(
        max_depth=depth,
        min_samples_leaf=30,
        random_state=42,
    )
    dtr.fit(X_train, interest_rate)
    tr_mae = mean_absolute_error(interest_rate,      dtr.predict(X_train))
    te_mae = mean_absolute_error(interest_rate_test, dtr.predict(X_test))
    te_r2  = r2_score(interest_rate_test,            dtr.predict(X_test))
    print(f"  {str(depth):<10} {tr_mae:<13.4f} {te_mae:<13.4f} {te_r2:.4f}")

# Regression tree leaf output = mean target in that leaf
best_reg = DecisionTreeRegressor(max_depth=4, min_samples_leaf=50, random_state=42)
best_reg.fit(X_train, interest_rate)

# Sample prediction with explanation
sample = X_test.iloc[[0]]
rate_pred = best_reg.predict(sample)[0]
print(f"
Predicted interest rate for applicant 0: {rate_pred:.2f}%")
print(f"Actual interest rate:                    {interest_rate_test.iloc[0]:.2f}%")

The key insight: once you understand how a single decision tree chooses splits, computes impurity, and makes predictions, Random Forest and XGBoost become straightforward — they are just collections of trees combined in different ways. The algorithm you built from scratch in this module IS the algorithm inside every XGBoost model at Stripe, Instacart, and every unicorn running tabular ML.

from sklearn.tree import DecisionTreeClassifier
from sklearn.pipeline import Pipeline
from sklearn.model_selection import GridSearchCV, StratifiedKFold
from sklearn.metrics import classification_report, roc_auc_score
import pandas as pd
import joblib

# ── Day-one task: build an interpretable loan screening model ──────────

# Decision trees need no scaling — direct pipeline
pipe = Pipeline([
    ('model', DecisionTreeClassifier(random_state=42)),
])

# Hyperparameter search
param_grid = {
    'model__max_depth':        [3, 4, 5, 6],
    'model__min_samples_leaf': [20, 30, 50],
    'model__ccp_alpha':        [0.0, 0.0001, 0.001],
}

cv = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)
grid = GridSearchCV(
    pipe, param_grid,
    cv=cv, scoring='roc_auc',
    n_jobs=-1, verbose=0,
)
grid.fit(X_train, y_train)

print(f"Best params: {grid.best_params_}")
print(f"Best CV AUC: {grid.best_score_:.4f}")

# Evaluate on test set
best_model = grid.best_estimator_
y_pred      = best_model.predict(X_test)
y_proba     = best_model.predict_proba(X_test)[:, 1]
test_auc    = roc_auc_score(y_test, y_proba)

print(f"
Test AUC:  {test_auc:.4f}")
print(f"Test accuracy: {(y_pred == y_test).mean():.4f}")
print("
Classification report:")
print(classification_report(y_test, y_pred,
                             target_names=['Reject', 'Approve']))

# ── Feature importance report for the risk team ───────────────────────
tree_model = best_model.named_steps['model']
importance_df = pd.DataFrame({
    'feature':    X_train.columns,
    'importance': tree_model.feature_importances_,
}).sort_values('importance', ascending=False)

print("
Feature importance report (for credit risk team):")
print(importance_df.to_string(index=False))

# ── Save model + decision rules for compliance ─────────────────────────
from sklearn.tree import export_text
rules = export_text(tree_model, feature_names=X_train.columns.tolist())
with open('/tmp/capitalone_loan_rules.txt', 'w') as f:
    f.write(rules)

joblib.dump(best_model, '/tmp/capitalone_loan_tree.pkl')
print("
Model and decision rules saved for compliance audit.")

The depth sweep earlier in this module shows exactly why this fails: accuracy improves as depth grows from 1 to around 4–5, then reverses — depth=10 hits 99% training accuracy but only 83% test accuracy, and an unconstrained tree (depth=None) hits 100% training accuracy while test accuracy actually drops to 79%. Past the point where the tree has captured the real signal, every additional split is fitting noise specific to the training rows — a scenario built for memorising exceptions, not generalising to new applicants. Depth is a capacity dial, and more capacity only helps up to where the true pattern is fully captured.

The code comparison earlier in this module makes the actual answer explicit: "Both criteria select the same split — just different scales." Gini ranges 0–0.5 for binary classification, entropy ranges 0–1 bit, but they rank candidate splits almost identically in practice. sklearn defaults to Gini because it skips a logarithm and is marginally faster to compute — not because it produces better trees. If you are spending tuning time on this instead of max_depth, min_samples_leaf, or ccp_alpha, you are optimising the wrong hyperparameter.

True for a shallow tree with a handful of splits, which is exactly why Capital One wanted one over logistic regression in the first place. It stops being true once max_depth grows past 6–8: the tree now has dozens of splits and hundreds of leaves, and "interpretable" degrades into a rulebook too large for a regulator or customer to meaningfully audit, even though every individual rule is still printable with export_text. There is also a subtler limit even in a shallow tree: the printed threshold (credit_score <= 650) is whichever value happened to minimise weighted Gini on this training sample — a few points either side can be an arbitrary artifact of the data rather than a meaningful business cutoff.

The module explicitly frames the actual motivation as instability, not marginal accuracy: "A single tree overfits. It is also unstable — small changes in the training data produce dramatically different trees." That instability is the formal definition of high variance — swap out a handful of borderline applicants near a split boundary and the chosen feature or threshold can change, cascading into a differently-shaped tree downstream. Random Forest's bagging specifically targets this variance by averaging many trees trained on different random samples; the accuracy improvement is a side effect of fixing the instability, not the primary goal.

Feature importance measures how much impurity reduction this specific tree credited to a feature across the splits where it was greedily chosen — not how predictive that feature fundamentally is. If two features are correlated (loan_amount and loan_to_income both encode overlapping information here), the tree picks whichever one narrowly wins the Gini comparison at the first opportunity and credits it with all the importance, while the other can show near-zero importance despite carrying comparable signal. This is exactly the leakage-detection trap flagged in the errors section above — always check correlated feature groups together, and prefer permutation importance or Random Forest's averaged importance (computed across many trees seeing different feature subsets) over a single tree's numbers.

At every node the tree evaluates every feature and every candidate threshold on that feature, computes the weighted Gini impurity (or entropy) of the two resulting child groups for each candidate, and greedily picks whichever single split produces the lowest weighted impurity — equivalently, the largest reduction from the parent node's impurity. It then repeats this process recursively and independently inside each child, continuing until a stopping condition is met: max depth reached, too few samples to split further, or a node is already pure. It is a greedy, locally-optimal algorithm at every step — it never looks ahead to see if a worse split now would enable a better one two levels down.

Left unconstrained, the tree keeps splitting until every leaf is pure — in the extreme, one training sample per leaf — which drives training accuracy to 100% by memorising individual rows, including noise like one applicant approved despite a 400 credit score purely because the analyst had an inconsistent day. That memorised noise does not generalise. The fix is controlling tree capacity: pre-pruning parameters like max_depth (start at 3–5), min_samples_leaf, and min_samples_split stop the tree from growing that deep in the first place, while post-pruning with ccp_alpha grows the full tree first and then removes branches whose removal doesn't meaningfully hurt impurity, with the right alpha chosen via cross-validation.

In practice, barely. Gini impurity (1 − Σpᵢ²) and entropy-based information gain almost always select the same splits — they're just on different numeric scales (Gini tops out at 0.5 for binary classes, entropy at 1 bit). Gini is marginally cheaper to compute since it avoids a logarithm, which is why sklearn defaults to it; entropy can occasionally favour a very slightly more balanced split. I'd spend tuning effort on max_depth, min_samples_leaf, and ccp_alpha before ever touching this — those control overfitting directly, while the splitting criterion barely changes the resulting tree.

High variance means small changes to the training data produce large changes in the fitted model. A tree is a clear example: change which handful of borderline applicants near a split threshold happened to land in the training set, and the chosen feature or threshold at that node can flip, which cascades into a differently structured tree below it. Random Forest directly targets this: train many trees, each on a bootstrapped sample of rows and a random subset of features, so their individual errors are decorrelated, then average their predictions. Averaging over many high-variance, low-bias trees cancels out the instability while keeping the low bias — which is exactly why the ensemble generalises far better than any one tree in it.

Not necessarily, and this is a common trap. Feature importance in a single tree only reflects how much impurity reduction got credited to a feature at the specific splits where the greedy algorithm happened to choose it. If two features are correlated, the tree picks whichever one narrowly wins the impurity comparison first and gives it all the credit — the other can look completely uninformative despite carrying comparable signal, and this is also a classic symptom to check for label leakage. I'd validate with permutation importance, which measures the actual performance drop from shuffling a feature rather than relying on a single greedy tree's internal bookkeeping, and I'd prefer Random Forest's importance, averaged across many trees that each see different feature subsets, over any single tree's numbers.