Lesson 5 of 10

Models

Logistic Regression baseline vs XGBoost, XGBoost regression, CatBoost. Pros, cons, importance.

Outputs will appear here after the full run. The code is final.

Left: what & whyRight: the code and its real output

5. Models (each fold checkpointed)

model	output	why it's here
`logreg` Logistic Regression (impute → scale → LR pipeline)	4 probabilities	simple, fast, interpretable baseline; shows what non-linear models add
`lgb_multi` LightGBM multiclass	4 probabilities	strong default for tabular data
`lgb_reg` LightGBM regression on tier 1–4	one score	uses the order of the tiers (H4)
`xgb` XGBoost multiclass	4 probabilities	a different boosting implementation, so its mistakes differ
`xgb_reg` XGBoost regression on tier 1–4	one score	a second ordered model, on GPU
`cat` CatBoost multiclass	4 probabilities	symmetric trees, strong regularisation, a third opinion
`lgb_ord` (optional)	4 probabilities	v1's ordinal model: 3 binary models P(tier > k)

Every model uses early stopping on its validation fold; test predictions are averaged over the 5 fold models.

def pick_lgb_device():
    """LightGBM's GPU mode (OpenCL) isn't always available or faster: time 30 rounds on 60k rows on each device, keep the faster."""
    if not GPU or not any(m.startswith("lgb") for m in MODELS):
        return "cpu"
    rows_ = np.random.RandomState(0).choice(n_tr, min(60000, n_tr), replace=False)
    times = {}
    for dev in ("cpu", "gpu"):
        try:
            pa = dict(objective="multiclass", num_class=K, learning_rate=0.05, num_leaves=127, max_bin=63, feature_fraction=0.5,
                      verbose=-1, num_threads=N_THREADS, device_type=dev)
            t0 = time.time()
            lgb.train(pa, lgb.Dataset(X[rows_], y[rows_], categorical_feature=CAT_IDX), 30)
            times[dev] = time.time() - t0
        except Exception as e:
            print(f"LightGBM on {dev} not available: {str(e)[:150]}")
    print("LightGBM speed test (30 rounds, 60k rows):", {k: f"{v:.1f}s" for k, v in times.items()})
    return min(times, key=times.get) if times else "cpu"


LGB_DEVICE = pick_lgb_device()
print("LightGBM device:", LGB_DEVICE, "| XGBoost device:", "cuda" if GPU else "cpu", "| CatBoost:", "GPU" if GPU else "CPU")
LGB_BASE = dict(learning_rate=0.05, num_leaves=127, min_child_samples=40, feature_fraction=0.5, bagging_fraction=0.8,
                bagging_freq=1, lambda_l2=1.0, max_bin=63, verbose=-1, seed=SEED, num_threads=N_THREADS, device_type=LGB_DEVICE)
XGB_PARAMS = dict(objective="multi:softprob", num_class=K, eta=0.05, max_depth=8, min_child_weight=5, subsample=0.8,
                  colsample_bytree=0.5, max_bin=128, tree_method="hist", eval_metric="mlogloss", seed=SEED,
                  device="cuda" if GPU else "cpu", nthread=N_THREADS)
CAT_PARAMS = dict(loss_function="MultiClass", iterations=MAX_ROUNDS, learning_rate=0.08, depth=8, border_count=128,
                  random_seed=SEED, task_type="GPU" if GPU else "CPU", thread_count=N_THREADS, od_type="Iter", od_wait=100)


def train_lgb(params, Xtr, ytr, Xva, yva, Xte):
    dtr = lgb.Dataset(Xtr, ytr, categorical_feature=CAT_IDX)
    dva = lgb.Dataset(Xva, yva, categorical_feature=CAT_IDX, reference=dtr)
    m = lgb.train(params, dtr, MAX_ROUNDS, valid_sets=[dva],
                  callbacks=[lgb.early_stopping(100, verbose=False), lgb.log_evaluation(LOG_EVERY)])
    print(f"   best iteration: {m.best_iteration}")
    return m.predict(Xva, num_iteration=m.best_iteration), m.predict(Xte, num_iteration=m.best_iteration)


def cumulative_to_classes(p_above):
    p_above = np.minimum.accumulate(p_above, axis=1)
    edges = np.column_stack([np.ones(len(p_above)), p_above, np.zeros(len(p_above))])
    return np.clip(edges[:, :-1] - edges[:, 1:], 1e-6, 1)


def fit_lgb_multi(Xtr, ytr, Xva, yva, Xte):
    return train_lgb({**LGB_BASE, "objective": "multiclass", "num_class": K}, Xtr, ytr, Xva, yva, Xte)


def fit_lgb_reg(Xtr, ytr, Xva, yva, Xte):
    pv, pt = train_lgb({**LGB_BASE, "objective": "regression"}, Xtr, ytr + 1.0, Xva, yva + 1.0, Xte)
    return pv[:, None], pt[:, None]


def fit_lgb_ord(Xtr, ytr, Xva, yva, Xte):
    av, at = [], []
    for k in range(K - 1):
        v, t = train_lgb({**LGB_BASE, "objective": "binary"}, Xtr, (ytr > k).astype(int), Xva, (yva > k).astype(int), Xte)
        av.append(v); at.append(t)
    return cumulative_to_classes(np.column_stack(av)), cumulative_to_classes(np.column_stack(at))


from sklearn.impute import SimpleImputer
from sklearn.linear_model import LogisticRegression
from sklearn.pipeline import make_pipeline
from sklearn.preprocessing import StandardScaler


def fit_logreg(Xtr, ytr, Xva, yva, Xte):
    """baseline: impute -> scale -> multinomial logistic regression (a reusable sklearn pipeline)"""
    clean = lambda A: np.where(np.isfinite(A), A, np.nan)
    m = make_pipeline(SimpleImputer(strategy="median"), StandardScaler(), LogisticRegression(max_iter=300, C=0.5))
    m.fit(clean(Xtr), ytr)
    return m.predict_proba(clean(Xva)), m.predict_proba(clean(Xte))


LAST_IMP = {}


def fit_xgb(Xtr, ytr, Xva, yva, Xte):
    dtr, dva = xgb.DMatrix(Xtr, label=ytr), xgb.DMatrix(Xva, label=yva)
    b = xgb.train(XGB_PARAMS, dtr, MAX_ROUNDS, evals=[(dva, "valid")], early_stopping_rounds=100, verbose_eval=LOG_EVERY)
    LAST_IMP.clear(); LAST_IMP.update(b.get_score(importance_type="gain"))
    it = (0, b.best_iteration + 1)
    print(f"   best iteration: {b.best_iteration}")
    return b.predict(dva, iteration_range=it), b.predict(xgb.DMatrix(Xte), iteration_range=it)


XGB_REG = {**{k: v for k, v in XGB_PARAMS.items() if k not in ("objective", "num_class", "eval_metric")},
           "objective": "reg:squarederror", "eval_metric": "rmse"}


def fit_xgb_reg(Xtr, ytr, Xva, yva, Xte):
    dtr, dva = xgb.DMatrix(Xtr, label=ytr + 1.0), xgb.DMatrix(Xva, label=yva + 1.0)
    b = xgb.train(XGB_REG, dtr, MAX_ROUNDS, evals=[(dva, "valid")], early_stopping_rounds=100, verbose_eval=LOG_EVERY)
    it = (0, b.best_iteration + 1)
    print(f"   best iteration: {b.best_iteration}")
    return b.predict(dva, iteration_range=it)[:, None], b.predict(xgb.DMatrix(Xte), iteration_range=it)[:, None]


def fit_cat(Xtr, ytr, Xva, yva, Xte):
    m = CatBoostClassifier(**CAT_PARAMS)
    m.fit(Xtr, ytr, eval_set=(Xva, yva), verbose=LOG_EVERY, use_best_model=True)
    return m.predict_proba(Xva), m.predict_proba(Xte)


TRAINERS = {"logreg": fit_logreg, "lgb_multi": fit_lgb_multi, "lgb_reg": fit_lgb_reg, "lgb_ord": fit_lgb_ord, "xgb": fit_xgb, "xgb_reg": fit_xgb_reg,
            "cat": fit_cat}
OOF, TEST = {}, {}
for key in MODELS:
    for f, (tr, va) in enumerate(folds):
        name = f"{TAG}_{SET_TAG}_{key}_fold{f}"
        prog.start(f"{key}: fold {f + 1}/{FOLDS}")
        ck = ck_load(name)
        if ck is None:
            pv, pt = TRAINERS[key](X[tr], y[tr], X[va], y[va], Xt)
            ck = {"oof": np.asarray(pv, np.float32), "test": np.asarray(pt, np.float32)}
            if key == "xgb":  # keep feature importance (gain) for section 5c
                ck["imp"] = np.array([LAST_IMP.get(f"f{i}", 0.0) for i in range(X.shape[1])], np.float32)
            ck_save(name, **ck)
            prog.finish()
        else:
            prog.finish(from_checkpoint=True)
        if key not in OOF:
            OOF[key] = np.zeros((n_tr, ck["oof"].shape[1]), np.float32)
            TEST[key] = np.zeros((len(Xt), ck["test"].shape[1]), np.float32)
        OOF[key][va] = ck["oof"]
        if "imp" in ck:
            XGB_IMP = XGB_IMP + ck["imp"] if "XGB_IMP" in globals() else ck["imp"].copy()
        TEST[key] += ck["test"] / FOLDS
        if ck["oof"].shape[1] == K:
            print(f"   fold macro F1: {fast_f1(y[va], ck['oof'].argmax(1)):.4f}")
for key in OOF:
    if OOF[key].shape[1] == K:
        print(f"{key:10s} grouped CV macro F1 (plain argmax): {fast_f1(y, OOF[key].argmax(1)):.4f}")

5b. Model comparison

Every model is scored on the same operator-grouped folds. Probability models use the most likely tier; regression models round to the nearest tier.

cmp_rows = []
for key in OOF:
    pred_ = OOF[key].argmax(1) if OOF[key].shape[1] == K else np.clip(np.rint(OOF[key][:, 0]) - 1, 0, K - 1).astype(int)
    cmp_rows.append({"model": key, "grouped CV macro F1": round(fast_f1(y, pred_), 4)})
display(pd.DataFrame(cmp_rows).sort_values("grouped CV macro F1", ascending=False).reset_index(drop=True))

model	advantages	disadvantages
Logistic Regression	fast, stable, coefficients are interpretable, little overfitting	only linear boundaries; needs scaling/imputation; can't use interactions like "zone × size" unless we build them
XGBoost	strong on tabular data, handles missing values, captures interactions, fast on GPU	many hyperparameters; can overfit without early stopping
XGBoost regression	uses the order of tiers (predicting 2.6 is "between 2 and 3")	needs cut-points to turn a number into a tier
CatBoost	symmetric trees regularise well; robust with default settings	slower on CPU; less flexible trees
LightGBM (optional)	very fast on CPU with many cores, leaf-wise growth	its GPU mode was slower than CPU here; on Kaggle's 4 cores it is the bottleneck

5c. Which features matter? (XGBoost gain, summed over folds)

import matplotlib.pyplot as plt
if "XGB_IMP" in globals():
    imp = pd.Series(XGB_IMP, index=COLS).sort_values(ascending=False)
    fig, ax = plt.subplots(figsize=(8, 7))
    imp.head(25)[::-1].plot.barh(ax=ax, title="Top 25 features by XGBoost gain")
    plt.tight_layout(); plt.show()
    groups = {"target encodings (te_/res_)": ("te_", "res_"), "geo neighbours (geo*/knn*)": ("geo", "knn"),
              "operator profile (h_/b_)": ("h_", "b_"), "facilities (f_)": ("f_",)}
    share = {g: imp[[c for c in imp.index if c.startswith(p)]].sum() / imp.sum() for g, p in groups.items()}
    share["listing's own columns"] = 1 - sum(share.values())
    display(pd.Series(share, name="share of total gain").round(3).sort_values(ascending=False).to_frame())