"""Next-best SKU picks as of 20 Oct 2026 with the picked model (LightGBM ranker), memory-lean: each labelled frame is
subsampled (all positives + 15% of negatives) as soon as it is built. Also stores the fair-pick and bootstrap results
printed by reco_final_v2.py (that run was stopped for memory after the bootstrap)."""
import sys, re, ast, gc, pickle, time, warnings
import numpy as np, pandas as pd, scipy.sparse as sp, lightgbm as lgb
warnings.filterwarnings('ignore')
sys.path.insert(0, '/home/claude/binzagr/boost'); sys.path.insert(0, '/home/claude/binzagr/pred')
import reco_race as RR
from reco_final_v2 import enrich, FEATS, fit_ranker
from p03_reco import build, B, P
OUT = '/home/claude/binzagr/boost/work'
DATA_END = pd.Timestamp('2026-10-20')

# ---- results from the reco_final_v2 log
log = open(f'{OUT}/reco_final_v2.log').read()
m = re.search(r'fair validation p@10: blend ([\d.]+) \(weights (\([^)]*\))\) vs ranker ([\d.]+) -> pick (\w+); test weights (\([^)]*\))', log)
res = {}
for k in ('site_style', 'ease', 'blend', 'ranker'):
    line = re.search(rf'^{k} (\{{.*\}})$', log, re.M).group(1)
    res[k] = ast.literal_eval(re.sub(r'np\.float64\(([^)]*)\)', r'\1', line))
pickle.dump({'res': res, 'pick': m.group(4), 'fair_blend_val': float(m.group(1)), 'ranker_val': float(m.group(3)),
             'w_fair': ast.literal_eval(m.group(2)), 'w_test': ast.literal_eval(m.group(5))}, open(f'{OUT}/reco_final_v2.pkl', 'wb'))
print('stored results', res, flush=True)

# ---- training sample for the final ranker: three labelled cut-offs, subsampled as built
t0 = time.time(); rs = np.random.RandomState(11); parts = []
for i, c in enumerate((RR.TRAIN0, RR.VAL, RR.TEST)):
    F, X, Xw = RR.frame(c); enrich(F, X)
    F = F[(F.y == 1) | (rs.rand(len(F)) < 0.15)][['outlet_id', 'y'] + FEATS].copy()
    F['outlet_id'] = 'abc'[i] + F.outlet_id; parts.append(F)
    del X, Xw; gc.collect(); print('frame', c.date(), len(F), f'{time.time() - t0:.0f}s', flush=True)
D = pd.concat(parts, ignore_index=True).sort_values('outlet_id', kind='stable'); del parts; gc.collect()
grp = D.groupby('outlet_id', sort=False).size().values
mdl = lgb.LGBMRanker(objective='lambdarank', n_estimators=300, learning_rate=0.05, num_leaves=63, min_child_samples=100, subsample=0.8,
                     subsample_freq=1, colsample_bytree=0.8, verbose=-1, n_jobs=2, lambdarank_truncation_level=20).fit(D[FEATS], D.y, group=grp)
del D; gc.collect()
# ---- score every outlet as of 20 Oct (SKUs sold somewhere in the last 4 weeks only)
Fn = build(DATA_END, with_label=False, scope='all'); past = B[B.date <= DATA_END]
outs = np.array(sorted(Fn.outlet_id.unique())); skus = np.array(sorted(Fn.sku.unique()))
oi = {o: i for i, o in enumerate(outs)}; si = {s_: j for j, s_ in enumerate(skus)}
pp = past[past.outlet_id.isin(oi) & past.sku.isin(si)].drop_duplicates(['outlet_id', 'sku'])
Xn = sp.csr_matrix((np.ones(len(pp)), (pp.outlet_id.map(oi).values, pp.sku.map(si).values)), shape=(len(outs), len(skus)))
Fn['oi'] = Fn.outlet_id.map(oi).values; Fn['si'] = Fn.sku.map(si).values; Fn['ocode'] = Fn['oi']; Fn['y'] = 0
enrich(Fn, Xn)
last_sale = B.groupby('sku').date.max(); Fn = Fn[Fn.sku.isin(last_sale[last_sale > DATA_END - pd.Timedelta(days=28)].index)].copy()
Fn['score'] = mdl.predict(Fn[FEATS])
Fn = Fn.sort_values(['outlet_id', 'score'], ascending=[True, False]); Fn['rank'] = Fn.groupby('outlet_id').cumcount() + 1
top = Fn[Fn['rank'] <= 10][['outlet_id', 'rank', 'sku', 'score', 'mainvan']].copy()
top['sku_name'] = top.sku.map(P.prd_desc_en); top['principal'] = top.sku.map(P.l1_desc)
top['model'] = 'LightGBM ranker over EASE, RP3beta and the current scores'
top.to_parquet(f'{OUT}/picks_v2.parquet', index=False)
print('picks', top.outlet_id.nunique(), 'outlets', len(top), 'rows', f'{time.time() - t0:.0f}s')