v3 · Colab run · part 2 of 7
Features
The notebook exactly as it ran on Colab's T4 GPU, with its real outputs.
Download the executed notebook
Left: what & whyRight: the code and its real output
2. Features (checkpointed)
Blocks (the ablation in section 3 decides which to keep):
- v2 set: base columns, facility one-hots (≥ 0.5% of listings), geo transforms, demand/space ratios, local density, zone context, label-free operator profile, and out-of-fold target features on grouped folds (zone / comparables / geo neighbours / kNN comparables).
- H1
rel_*: listing minus the average of its comparables (same zone + access type + seats), for bookings, open days, reviews, terms, facilities, scores. No labels used. - H2
facte_*: out-of-fold facility effects. For each facility, the average tier residual after removing the metro average; per listing, the mean / sum / max / min over its facilities.
prog.start("build features") feat = ck_load(f"{TAG}_features") if feat is None: raw = pd.concat([train.drop(columns="price_tier"), test], ignore_index=True) rawx = raw.assign(seats_b=raw.max_seats.clip(upper=12)) df = raw.copy() for c in ["operator_certified", "operator_has_avatar", "operator_id_verified", "bookable"]: df[c] = df[c].map({"t": 1, "f": 0}) for c in ["snapshot_start_date", "snapshot_end_date", "first_review_date", "last_review_date"]: df[c] = (pd.to_datetime(df[c]) - pd.Timestamp("2000-01-01")).dt.days for c in ["operator_ref", "zone", "borough"]: df[c] = raw[c].map(raw[c].value_counts()) nfac = raw["facilities"].str.count("FCL-").fillna(0) df["n_facilities"] = nfac df["occupancy"] = df.booked_load / 365 df["review_speed_12m"] = df.review_count_12m / (df.review_count + 1) df["review_trend"] = (df.review_count_12m + 1) / (df.review_count_prev_yr + 1) df["days_since_last_review"] = df.snapshot_end_date - df.last_review_date df["listing_age_days"] = df.snapshot_end_date - df.first_review_date df["open_ratio_a_d"] = df.open_slots_a / (df.open_slots_d + 1) df["open_ratio_e_d"] = df.open_slots_e / (df.open_slots_d + 1) df["seats_per_ws"] = df.max_seats / df.workstations.replace(0, np.nan) df["seats_per_zone"] = df.max_seats / df.zones.replace(0, np.nan) df["seats_per_util"] = df.max_seats / df.utility_rooms.replace(0, np.nan) df["term_span"] = df.max_term_days - df.min_term_days codes = raw["facilities"].fillna("[]").map(json.loads).explode().dropna() fac = pd.crosstab(codes.index, codes).reindex(raw.index, fill_value=0).clip(upper=1).astype(np.uint8) fac = fac.loc[:, fac.mean() >= 0.005] fac.columns = ["f_" + c[4:] for c in fac.columns] code_id, code_names = pd.factorize(codes.values) F_all = sparse.csr_matrix((np.ones(len(codes), np.float32), (codes.index.to_numpy(), code_id)), shape=(len(raw), len(code_names))) F_all.data[:] = 1.0 F_all = F_all[:, np.where(np.asarray(F_all.sum(0)).ravel() >= 30)[0]].tocsr() lat, lon = raw["geo_lat"], raw["geo_lon"] geo = pd.DataFrame({"lat_p_lon": lat + lon, "lat_m_lon": lat - lon}) for a in (15, 30, 60): r = np.deg2rad(a) geo[f"rot{a}_x"] = lat * np.cos(r) - lon * np.sin(r) geo[f"rot{a}_y"] = lat * np.sin(r) + lon * np.cos(r) for k in ("metro", "zone"): center = raw.groupby(k)[["geo_lat", "geo_lon"]].transform("median") geo[f"dist_{k}_center"] = np.hypot(lat - center["geo_lat"], lon - center["geo_lon"]) km = MiniBatchKMeans(100, random_state=0, n_init=3, batch_size=4096).fit(raw[["geo_lat", "geo_lon"]]) geo["geo_cluster"] = km.labels_ geo["geo_cluster_dist"] = km.transform(raw[["geo_lat", "geo_lon"]]).min(1) dist_all, _ = NearestNeighbors(n_neighbors=51).fit(raw[["geo_lat", "geo_lon"]]).kneighbors(raw[["geo_lat", "geo_lon"]]) geo["density_d10"], geo["density_d50"] = dist_all[:, 10], dist_all[:, 50] geo["zone_mean_seats"] = df.max_seats.groupby(raw.zone).transform("mean") geo["zone_mean_nfac"] = nfac.groupby(raw.zone).transform("mean") geo["seats_vs_zone"] = df.max_seats - geo.zone_mean_seats geo["nfac_vs_zone"] = nfac - geo.zone_mean_nfac # label-free operator profile: what kind of operator is this, and how does this listing compare with its portfolio? d = pd.DataFrame({"seats": raw.max_seats, "ws": raw.workstations, "zones": raw.zones, "util": raw.utility_rooms, "nfac": nfac, "score": raw.score_overall, "rev": raw.review_count, "minterm": raw.min_term_days, "open_d": raw.open_slots_d, "load": raw.booked_load}) for c in ["access_type", "space_class"]: for v in raw[c].value_counts().index[:4]: d[f"{c}_{v}"] = (raw[c] == v).astype(float) g = d.groupby(host) prof = pd.DataFrame(index=d.index) prof["h_rows"] = g.seats.transform("size") prof["h_zones_n"] = raw.groupby(host).zone.transform("nunique") prof["h_metros_n"] = raw.groupby(host).metro.transform("nunique") for c in d.columns: m = g[c].transform("mean") prof[f"h_mean_{c}"] = m if c in ("seats", "ws", "zones", "util", "nfac", "score", "minterm"): prof[f"h_std_{c}"] = g[c].transform("std") prof[f"h_diff_{c}"] = d[c] - m prof["h_rank_seats"] = g.seats.rank(pct=True) prof["h_rank_nfac"] = g.nfac.rank(pct=True) bld = raw.groupby(["geo_lat", "geo_lon"]).ngroup().to_numpy() gb = d.groupby(bld) prof["b_rows"] = gb.seats.transform("size") prof["b_hosts"] = pd.Series(host).groupby(bld).transform("nunique") prof["b_mean_seats"] = gb.seats.transform("mean") prof["b_mean_nfac"] = gb.nfac.transform("mean") # H1: demand and quality relative to comparable listings (same zone + access type + seats); no labels comp = rawx.groupby(["zone", "access_type", "seats_b"], dropna=False, sort=False).ngroup().to_numpy() value_gap = raw.score_value - raw[["score_accuracy", "score_condition", "score_onboarding", "score_support", "score_access"]].mean(axis=1) rel = pd.DataFrame({"rel_value_gap": value_gap, "rel_comp_size": pd.Series(comp).map(pd.Series(comp).value_counts()).to_numpy()}) for c, s in {"load": raw.booked_load, "open_a": raw.open_slots_a, "open_d": raw.open_slots_d, "rpm": raw.reviews_per_month, "rev12": raw.review_count_12m, "minterm": raw.min_term_days, "nfac": nfac, "score": raw.score_overall, "value": value_gap, "ws": raw.workstations}.items(): rel[f"rel_{c}_vs_comps"] = s - s.groupby(comp).transform("mean") metro_comp = rawx.groupby(["metro", "access_type", "seats_b"], dropna=False, sort=False).ngroup().to_numpy() rel["rel_load_vs_metro_comps"] = raw.booked_load - raw.booked_load.groupby(metro_comp).transform("mean") df = df.drop(columns=["facilities", "space_id"]) CAT_COLS = list(df.select_dtypes(exclude="number").columns) for c in CAT_COLS: df[c] = df[c].astype("category").cat.codes # -1 = missing (LightGBM treats negative categories as missing) base = pd.concat([df, fac, geo, prof, rel], axis=1) # ---- target features, out-of-fold on grouped folds ---- TE_GROUPS = { "zone": ["zone"], "boro": ["borough"], "sc": ["space_class"], "metro_sc": ["metro", "space_class"], "zone_acc": ["zone", "access_type"], "zone_seats": ["zone", "max_seats"], "zone_sc": ["zone", "space_class"], "geo": ["geo_lat", "geo_lon"], "metro_acc_seats": ["metro", "access_type", "max_seats"], "zone_acc_seats": ["zone", "access_type", "seats_b"], "zone_acc_seats_ur": ["zone", "access_type", "seats_b", "utility_rooms"], "boro_acc_seats": ["borough", "access_type", "seats_b"], "metro_sc_seats": ["metro", "space_class", "seats_b"], "bld_acc": ["geo_lat", "geo_lon", "access_type"], "metro_acc_rooms": ["metro", "access_type", "zones", "utility_rooms"], } RES_GROUPS = ["zone", "geo", "zone_sc", "sc"] gid = {k: rawx[c].groupby(c, sort=False, dropna=False).ngroup().to_numpy() for k, c in TE_GROUPS.items()} base_ids = raw[["access_type", "max_seats"]].groupby(["access_type", "max_seats"], dropna=False).ngroup().to_numpy() metro_id = pd.factorize(raw.metro)[0] coords = raw[["geo_lat", "geo_lon"]].to_numpy() prior = np.bincount(y, minlength=K) / len(y) tiers = np.arange(1, K + 1) space = np.column_stack([(lat - lat.mean()) / lat.std(), (lon - lon.mean()) / lon.std(), np.log1p(raw.max_seats) * 0.3, raw.zones.fillna(1) * 0.05, raw.utility_rooms.fillna(1) * 0.05, nfac * 0.002]).astype(np.float32) acc = raw.access_type.astype(str).to_numpy() def target_features(fit, apply): cols = {} for name, g_ in gid.items(): # smoothed tier distribution of each group cnt = np.zeros((g_.max() + 1, K)); np.add.at(cnt, (g_[fit], y[fit]), 1) tot = cnt.sum(1) p = ((cnt + 20 * prior) / (tot[:, None] + 20))[g_[apply]] for j in range(K): cols[f"te_{name}_p{j + 1}"] = p[:, j] cols[f"te_{name}_mean"] = p @ tiers cols[f"te_{name}_n"] = np.log1p(tot[g_[apply]]) yt = y[fit] + 1.0 # residuals vs. an (access type, seats) baseline nb = base_ids.max() + 1 bm = (np.bincount(base_ids[fit], yt, nb) + 20 * yt.mean()) / (np.bincount(base_ids[fit], minlength=nb) + 20) res = yt - bm[base_ids[fit]] for name in RES_GROUPS: g_ = gid[name]; n = g_.max() + 1 c_ = np.bincount(g_[fit], minlength=n)[g_[apply]] + 10 mres = np.bincount(g_[fit], res, n)[g_[apply]] / c_ cols[f"res_{name}_mean"] = mres cols[f"res_{name}_pred"] = bm[base_ids[apply]] + mres pts, inv = np.unique(coords[fit], axis=0, return_inverse=True) # tiers of nearest locations cnt = np.zeros((len(pts), K)); np.add.at(cnt, (inv.ravel(), y[fit]), 1) dist, idx = NearestNeighbors(n_neighbors=50).fit(pts).kneighbors(coords[apply]) for k in (1, 5, 20, 50): c_ = cnt[idx[:, :k]].sum(1) p = (c_ + 5 * prior) / (c_.sum(1)[:, None] + 5) for j in range(K): cols[f"geo{k}_p{j + 1}"] = p[:, j] cols[f"geo{k}_mean"] = p @ tiers cols[f"geo{k}_dist"] = dist[:, k - 1] for k in (10, 50): # comparable listings: nearest in (location, size) with the same access type for s in ("mean", "p1", "p4", "dist"): cols[f"knn{k}_{s}"] = np.full(len(apply), np.nan) for a in np.unique(acc): fa, aa = fit[acc[fit] == a], np.where(acc[apply] == a)[0] if len(fa) < 50 or len(aa) == 0: continue dist, idx = NearestNeighbors(n_neighbors=50).fit(space[fa]).kneighbors(space[apply[aa]]) lab = y[fa][idx] for k in (10, 50): oh = np.stack([(lab[:, :k] == j).mean(1) for j in range(K)], 1) cols[f"knn{k}_mean"][aa] = oh @ tiers cols[f"knn{k}_p1"][aa], cols[f"knn{k}_p4"][aa] = oh[:, 0], oh[:, 3] cols[f"knn{k}_dist"][aa] = dist[:, k - 1] # H2: facility effects after removing the metro average (shrunk towards 0 for rare facilities) nm = metro_id.max() + 1 mm = np.bincount(metro_id[fit], yt, nm) / np.maximum(np.bincount(metro_id[fit], minlength=nm), 1) r_ = yt - mm[metro_id[fit]] Ff = F_all[fit] dev = (Ff.T @ r_) / (np.asarray(Ff.sum(0)).ravel() + 50) Fa = F_all[apply] nf = np.asarray(Fa.sum(1)).ravel() has = nf > 0 cols["facte_sum"] = Fa @ dev cols["facte_mean"] = np.where(has, cols["facte_sum"] / np.maximum(nf, 1), np.nan) cols["facte_max"] = np.where(has, Fa.multiply(dev + 10).max(1).toarray().ravel() - 10, np.nan) cols["facte_min"] = np.where(has, 10 - Fa.multiply(10 - dev).max(1).toarray().ravel(), np.nan) cols["facte_n_premium"] = Fa @ (dev > 0.05).astype(np.float32) cols["facte_n_budget"] = Fa @ (dev < -0.05).astype(np.float32) return pd.DataFrame(cols) T_tr = pd.concat([target_features(f, v).set_index(v) for f, v in folds]).sort_index() T_te = target_features(np.arange(n_tr), np.arange(n_tr, len(raw))) X_df = pd.concat([base.iloc[:n_tr].reset_index(drop=True), T_tr], axis=1) Xt_df = pd.concat([base.iloc[n_tr:].reset_index(drop=True), T_te], axis=1) feat = dict(X=X_df.to_numpy(np.float32), Xt=Xt_df.to_numpy(np.float32), cols=np.array(X_df.columns, dtype=str), cat_cols=np.array(CAT_COLS, dtype=str), bld=bld) ck_save(f"{TAG}_features", **feat) del raw, rawx, df, base, X_df, Xt_df, T_tr, T_te, F_all prog.finish() else: prog.finish(from_checkpoint=True) X_full, Xt_full = feat["X"], feat["Xt"] ALL_COLS = list(feat["cols"]) bld = feat["bld"] MT_tr = np.maximum(X_full[:, ALL_COLS.index("metro")], 0).astype(int) MT_te = np.maximum(Xt_full[:, ALL_COLS.index("metro")], 0).astype(int) print("all features:", X_full.shape, "| test:", Xt_full.shape)
▶ build features [█░░░░░░░░░░░░░░░░░░░░░░░░░░░░░ 1/21 (5%)] ✓ build features: took 3.1 min | [██░░░░░░░░░░░░░░░░░░░░░░░░░░░░ 2/21 (10%)] | elapsed 4.2 min | ETA ~0 min + 19 steps not timed yet all features: (279746, 411) | test: (114538, 411)