v3 · Colab run · part 4 of 7
Models
The notebook exactly as it ran on Colab's T4 GPU, with its real outputs.
Download the executed notebook
Left: what & whyRight: the code and its real output
4. Level-1 models (H3), each fold checkpointed
| model | output | why it's here |
|---|---|---|
lgb_multi LightGBM multiclass |
4 probabilities | strong default for tabular data |
lgb_reg LightGBM regression on tier 1–4 |
one score | uses the order of the tiers (H4) |
xgb XGBoost multiclass |
4 probabilities | a different boosting implementation, so its mistakes differ |
xgb_reg XGBoost regression on tier 1–4 |
one score | a second ordered model, on GPU |
cat CatBoost multiclass |
4 probabilities | symmetric trees, strong regularisation, a third opinion |
lgb_ord (optional) |
4 probabilities | v1's ordinal model: 3 binary models P(tier > k) |
Every model uses early stopping on its validation fold; test predictions are averaged over the 5 fold models.
def pick_lgb_device(): """LightGBM's GPU mode (OpenCL) isn't always available or faster: time 30 rounds on 60k rows on each device, keep the faster.""" if not GPU or not any(m.startswith("lgb") for m in MODELS): return "cpu" rows_ = np.random.RandomState(0).choice(n_tr, min(60000, n_tr), replace=False) times = {} for dev in ("cpu", "gpu"): try: pa = dict(objective="multiclass", num_class=K, learning_rate=0.05, num_leaves=127, max_bin=63, feature_fraction=0.5, verbose=-1, num_threads=N_THREADS, device_type=dev) t0 = time.time() lgb.train(pa, lgb.Dataset(X[rows_], y[rows_], categorical_feature=CAT_IDX), 30) times[dev] = time.time() - t0 except Exception as e: print(f"LightGBM on {dev} not available: {str(e)[:150]}") print("LightGBM speed test (30 rounds, 60k rows):", {k: f"{v:.1f}s" for k, v in times.items()}) return min(times, key=times.get) if times else "cpu" LGB_DEVICE = pick_lgb_device() print("LightGBM device:", LGB_DEVICE, "| XGBoost device:", "cuda" if GPU else "cpu", "| CatBoost:", "GPU" if GPU else "CPU") LGB_BASE = dict(learning_rate=0.05, num_leaves=127, min_child_samples=40, feature_fraction=0.5, bagging_fraction=0.8, bagging_freq=1, lambda_l2=1.0, max_bin=63, verbose=-1, seed=SEED, num_threads=N_THREADS, device_type=LGB_DEVICE) XGB_PARAMS = dict(objective="multi:softprob", num_class=K, eta=0.05, max_depth=8, min_child_weight=5, subsample=0.8, colsample_bytree=0.5, max_bin=128, tree_method="hist", eval_metric="mlogloss", seed=SEED, device="cuda" if GPU else "cpu", nthread=N_THREADS) CAT_PARAMS = dict(loss_function="MultiClass", iterations=MAX_ROUNDS, learning_rate=0.08, depth=8, border_count=128, random_seed=SEED, task_type="GPU" if GPU else "CPU", thread_count=N_THREADS, od_type="Iter", od_wait=100) def train_lgb(params, Xtr, ytr, Xva, yva, Xte): dtr = lgb.Dataset(Xtr, ytr, categorical_feature=CAT_IDX) dva = lgb.Dataset(Xva, yva, categorical_feature=CAT_IDX, reference=dtr) m = lgb.train(params, dtr, MAX_ROUNDS, valid_sets=[dva], callbacks=[lgb.early_stopping(100, verbose=False), lgb.log_evaluation(LOG_EVERY)]) print(f" best iteration: {m.best_iteration}") return m.predict(Xva, num_iteration=m.best_iteration), m.predict(Xte, num_iteration=m.best_iteration) def cumulative_to_classes(p_above): p_above = np.minimum.accumulate(p_above, axis=1) edges = np.column_stack([np.ones(len(p_above)), p_above, np.zeros(len(p_above))]) return np.clip(edges[:, :-1] - edges[:, 1:], 1e-6, 1) def fit_lgb_multi(Xtr, ytr, Xva, yva, Xte): return train_lgb({**LGB_BASE, "objective": "multiclass", "num_class": K}, Xtr, ytr, Xva, yva, Xte) def fit_lgb_reg(Xtr, ytr, Xva, yva, Xte): pv, pt = train_lgb({**LGB_BASE, "objective": "regression"}, Xtr, ytr + 1.0, Xva, yva + 1.0, Xte) return pv[:, None], pt[:, None] def fit_lgb_ord(Xtr, ytr, Xva, yva, Xte): av, at = [], [] for k in range(K - 1): v, t = train_lgb({**LGB_BASE, "objective": "binary"}, Xtr, (ytr > k).astype(int), Xva, (yva > k).astype(int), Xte) av.append(v); at.append(t) return cumulative_to_classes(np.column_stack(av)), cumulative_to_classes(np.column_stack(at)) def fit_xgb(Xtr, ytr, Xva, yva, Xte): dtr, dva = xgb.DMatrix(Xtr, label=ytr), xgb.DMatrix(Xva, label=yva) b = xgb.train(XGB_PARAMS, dtr, MAX_ROUNDS, evals=[(dva, "valid")], early_stopping_rounds=100, verbose_eval=LOG_EVERY) it = (0, b.best_iteration + 1) print(f" best iteration: {b.best_iteration}") return b.predict(dva, iteration_range=it), b.predict(xgb.DMatrix(Xte), iteration_range=it) XGB_REG = {**{k: v for k, v in XGB_PARAMS.items() if k not in ("objective", "num_class", "eval_metric")}, "objective": "reg:squarederror", "eval_metric": "rmse"} def fit_xgb_reg(Xtr, ytr, Xva, yva, Xte): dtr, dva = xgb.DMatrix(Xtr, label=ytr + 1.0), xgb.DMatrix(Xva, label=yva + 1.0) b = xgb.train(XGB_REG, dtr, MAX_ROUNDS, evals=[(dva, "valid")], early_stopping_rounds=100, verbose_eval=LOG_EVERY) it = (0, b.best_iteration + 1) print(f" best iteration: {b.best_iteration}") return b.predict(dva, iteration_range=it)[:, None], b.predict(xgb.DMatrix(Xte), iteration_range=it)[:, None] def fit_cat(Xtr, ytr, Xva, yva, Xte): m = CatBoostClassifier(**CAT_PARAMS) m.fit(Xtr, ytr, eval_set=(Xva, yva), verbose=LOG_EVERY, use_best_model=True) return m.predict_proba(Xva), m.predict_proba(Xte) TRAINERS = {"lgb_multi": fit_lgb_multi, "lgb_reg": fit_lgb_reg, "lgb_ord": fit_lgb_ord, "xgb": fit_xgb, "xgb_reg": fit_xgb_reg, "cat": fit_cat} OOF, TEST = {}, {} for key in MODELS: for f, (tr, va) in enumerate(folds): name = f"{TAG}_{SET_TAG}_{key}_fold{f}" prog.start(f"{key}: fold {f + 1}/{FOLDS}") ck = ck_load(name) if ck is None: pv, pt = TRAINERS[key](X[tr], y[tr], X[va], y[va], Xt) ck = {"oof": np.asarray(pv, np.float32), "test": np.asarray(pt, np.float32)} ck_save(name, **ck) prog.finish() else: prog.finish(from_checkpoint=True) if key not in OOF: OOF[key] = np.zeros((n_tr, ck["oof"].shape[1]), np.float32) TEST[key] = np.zeros((len(Xt), ck["test"].shape[1]), np.float32) OOF[key][va] = ck["oof"] TEST[key] += ck["test"] / FOLDS if ck["oof"].shape[1] == K: print(f" fold macro F1: {fast_f1(y[va], ck['oof'].argmax(1)):.4f}") for key in OOF: if OOF[key].shape[1] == K: print(f"{key:10s} grouped CV macro F1 (plain argmax): {fast_f1(y, OOF[key].argmax(1)):.4f}")
LightGBM device: cpu | XGBoost device: cuda | CatBoost: GPU ▶ xgb: fold 1/5 [████░░░░░░░░░░░░░░░░░░░░░░░░░░ 3/21 (14%)] [0] valid-mlogloss:1.34464 [250] valid-mlogloss:0.68007 [500] valid-mlogloss:0.66389 [750] valid-mlogloss:0.65959 [1000] valid-mlogloss:0.65905 [1048] valid-mlogloss:0.65932 best iteration: 948 ✓ xgb: fold 1/5: took 1.8 min | [█████░░░░░░░░░░░░░░░░░░░░░░░░░ 4/21 (19%)] | elapsed 6.0 min | ETA ~7 min + 13 steps not timed yet fold macro F1: 0.7135 ▶ xgb: fold 2/5 [█████░░░░░░░░░░░░░░░░░░░░░░░░░ 4/21 (19%)] [0] valid-mlogloss:1.34317 [250] valid-mlogloss:0.67086 [500] valid-mlogloss:0.65344 [750] valid-mlogloss:0.64839 [992] valid-mlogloss:0.64763 best iteration: 892 ✓ xgb: fold 2/5: took 1.8 min | [███████░░░░░░░░░░░░░░░░░░░░░░░ 5/21 (24%)] | elapsed 7.8 min | ETA ~5 min + 13 steps not timed yet fold macro F1: 0.7140 ▶ xgb: fold 3/5 [███████░░░░░░░░░░░░░░░░░░░░░░░ 5/21 (24%)] [0] valid-mlogloss:1.34376 ... (156 lines hidden) ... bestIteration = 2247 Shrink model to first 2248 iterations. ✓ cat: fold 4/5: took 1.7 min | [████████████████████████░░░░░░ 17/21 (81%)] | elapsed 24.5 min | ETA ~2 min + 3 steps not timed yet fold macro F1: 0.7160 ▶ cat: fold 5/5 [████████████████████████░░░░░░ 17/21 (81%)] 0: learn: 1.3093647 test: 1.3122034 best: 1.3122034 (0) total: 91.8ms remaining: 9m 10s 250: learn: 0.6286168 test: 0.7022837 best: 0.7022837 (250) total: 10.3s remaining: 3m 55s 500: learn: 0.5762297 test: 0.6866022 best: 0.6866022 (500) total: 20.1s remaining: 3m 40s 750: learn: 0.5384355 test: 0.6791796 best: 0.6791789 (745) total: 29.9s remaining: 3m 28s 1000: learn: 0.5079056 test: 0.6749869 best: 0.6749869 (1000) total: 39.6s remaining: 3m 17s 1250: learn: 0.4810091 test: 0.6723687 best: 0.6723687 (1250) total: 49.3s remaining: 3m 7s 1500: learn: 0.4565549 test: 0.6701565 best: 0.6701030 (1490) total: 58.9s remaining: 2m 56s 1750: learn: 0.4342202 test: 0.6684288 best: 0.6684288 (1750) total: 1m 8s remaining: 2m 46s 2000: learn: 0.4134261 test: 0.6676836 best: 0.6676835 (1999) total: 1m 18s remaining: 2m 36s 2250: learn: 0.3946886 test: 0.6668935 best: 0.6668866 (2249) total: 1m 27s remaining: 2m 26s 2500: learn: 0.3767622 test: 0.6665843 best: 0.6665651 (2494) total: 1m 37s remaining: 2m 16s 2750: learn: 0.3600809 test: 0.6664604 best: 0.6661767 (2697) total: 1m 47s remaining: 2m 6s bestTest = 0.6661766598 bestIteration = 2697 Shrink model to first 2698 iterations. ✓ cat: fold 5/5: took 2.0 min | [█████████████████████████░░░░░ 18/21 (86%)] | elapsed 26.5 min | ETA ~0 min + 3 steps not timed yet fold macro F1: 0.7114 xgb grouped CV macro F1 (plain argmax): 0.7148 cat grouped CV macro F1 (plain argmax): 0.7127