{ "title": "Binzagr PoV: before and after figures for binzagr.snop.live", "as_of": "2026-10-23", "site_read": "2026-10-23", "types": { "Replace": "Measured on the same test after switching the model; change it once the new model is live on the site", "Fix": "A wrong or mislabelled figure; change it now, no model change needed", "Add": "Keep the figure and add the note", "Verify first": "Our time-honest re-test is lower; re-run the site test on a clean split, then use whichever result it gives", "Recompute": "Needs the site pipeline re-run; we cannot produce the exact figure", "Keep": "No change" }, "rows": [ { "no": 1, "page": "Value cockpit", "where_on_page": "KPI tile, Forecast proof", "on_site_now": "22% \u00b7 less error than last week repeated", "replace_with": "22% \u00b7 less error than last week repeated (forecast 15% below actual)", "type": "Add", "basis": "Site SKU run bias -15.0% (base demand, weekly run)", "before_value": 22, "after_value": 22, "unit": "%" }, { "no": 2, "page": "Value cockpit", "where_on_page": "Proof card: Demand forecast", "on_site_now": "ML forecast 64% \u00b7 4-week weekday average 72% \u00b7 Repeat last week 82%", "replace_with": "Keep the three values; add \"ML bias \u221215%\" under ML forecast", "type": "Add", "basis": "Site scores, base demand weekly run", "before_value": 64, "after_value": 64, "unit": "%" }, { "no": 3, "page": "Value cockpit", "where_on_page": "Proof card: Next-best SKU", "on_site_now": "3.0\u00d7 \u00b7 Model picks 12% \u00b7 Best-sellers 4.0% \u00b7 Random SKUs 0.7%", "replace_with": "3.3\u00d7 \u00b7 Model picks 12.4% \u00b7 Best-sellers 3.8% \u00b7 Random SKUs 0.8%", "type": "Replace", "basis": "Site test re-run: picks 25 Aug, 4,097 outlets, 8 weeks; LightGBM ranker picked on 30 Jun validation (next_best_sku_models.csv)", "before_value": 12.0, "after_value": 12.4, "unit": "%" }, { "no": 4, "page": "Value cockpit", "where_on_page": "Proof card: Outlet churn warning", "on_site_now": "0.77 \u00b7 Churn model 0.77 \u00b7 Recency rule 0.68 \u00b7 top 20% of risk holds 48% of churners", "replace_with": "0.80 \u00b7 Churn model 0.80 \u00b7 Recency rule 0.73 \u00b7 top 20% of risk holds 59% of churners", "type": "Replace", "basis": "Extra trees picked on 2 validation folds; 1,314 held-out outlets at 25 Aug (churn_models.csv)", "before_value": 0.77, "after_value": 0.797, "unit": "AUC" }, { "no": 5, "page": "Value cockpit", "where_on_page": "Proof card: Cluster adoption", "on_site_now": "65% \u00b7 Adoption model 65% \u00b7 Channel popularity 40% \u00b7 What neighbours buy 12% \u00b7 AUC 0.90", "replace_with": "32% \u00b7 Adoption model 32% \u00b7 Channel popularity 30% \u00b7 What neighbours buy 9% \u00b7 AUC 0.79", "type": "Verify first", "basis": "Time-honest re-test: trained on adoptions to 31 Jul, tested Aug-Oct on held-out outlets, 5 clusters (adoption_models.csv)", "before_value": 65, "after_value": 32, "unit": "%" }, { "no": 6, "page": "Value cockpit", "where_on_page": "Scorecard: Next-best SKU", "on_site_now": "vs best-sellers \u00b7 3.0\u00d7 as good", "replace_with": "vs best-sellers \u00b7 3.3\u00d7 as good", "type": "Replace", "basis": "As row 3", "before_value": 3.0, "after_value": 3.3, "unit": "\u00d7" }, { "no": 7, "page": "Value cockpit", "where_on_page": "Scorecard: Outlet churn warning", "on_site_now": "vs recency rule \u00b7 +0.09 AUC", "replace_with": "vs recency rule \u00b7 +0.07 AUC", "type": "Replace", "basis": "As row 4; the recency rule is stronger on this test (0.73 vs 0.68), so the gap is smaller", "before_value": 0.09, "after_value": 0.07, "unit": "AUC" }, { "no": 8, "page": "Value cockpit", "where_on_page": "Scorecard: Cluster adoption", "on_site_now": "vs channel popularity \u00b7 1.6\u00d7 as good", "replace_with": "vs channel popularity \u00b7 1.1\u00d7 as good", "type": "Verify first", "basis": "As row 5", "before_value": 1.6, "after_value": 1.1, "unit": "\u00d7" }, { "no": 9, "page": "Value cockpit", "where_on_page": "Scorecard: Demand forecast", "on_site_now": "vs 4-week weekday average \u00b7 12% less error", "replace_with": "Keep; add \"3% at zero bias\"", "type": "Add", "basis": "Best blend scaled to zero bias: 74.1% vs 76.5% for the 4-week average (forecast_by_level.csv)", "before_value": 12, "after_value": 12, "unit": "%" }, { "no": 10, "page": "Value cockpit", "where_on_page": "Recommendation 1 (leads)", "on_site_now": "AUC 0.902 on unseen outlets", "replace_with": "AUC 0.79 on a time-honest test (0.75 within cluster)", "type": "Verify first", "basis": "As row 5", "before_value": 0.902, "after_value": 0.79, "unit": "AUC" }, { "no": 11, "page": "Value cockpit", "where_on_page": "Recommendation 3 (P1 accounts)", "on_site_now": "SAR 927,163 past due (24%)", "replace_with": "SAR 590,932 past due (15%) + SAR 336,231 older than the invoice window", "type": "Fix", "basis": "The site's own collections-lever evidence (matched invoices only)", "before_value": 927163, "after_value": 590932, "unit": "SAR" }, { "no": 12, "page": "Value cockpit", "where_on_page": "Recommendation 5 (win-back)", "on_site_now": "churn model AUC 0.77 on a later period it never saw \u00b7 Top 20% risk holds 48% of churners", "replace_with": "churn model AUC 0.80 on outlets it never saw \u00b7 Top 20% risk holds 59% of churners", "type": "Replace", "basis": "As row 4", "before_value": 0.77, "after_value": 0.8, "unit": "AUC" }, { "no": 13, "page": "Value cockpit", "where_on_page": "Explore card: Receivables", "on_site_now": "SAR 927K past due, 24% of the credit book", "replace_with": "SAR 591K past due, 15% of the credit book (+ SAR 336K unmatched)", "type": "Fix", "basis": "As row 11", "before_value": 927, "after_value": 591, "unit": "SAR K" }, { "no": 14, "page": "Value cockpit", "where_on_page": "Explore card: Next-best SKU", "on_site_now": "12% of the model's picks were bought \u00b7 3.0\u00d7 best-sellers", "replace_with": "12.4% of the model's picks were bought \u00b7 3.3\u00d7 best-sellers", "type": "Replace", "basis": "As row 3", "before_value": 12, "after_value": 12.4, "unit": "%" }, { "no": 15, "page": "Value cockpit", "where_on_page": "Hero and impact tree", "on_site_now": "SAR 3.45M a year", "replace_with": "Keep (all model upgrades add about SAR 12K, under 1%)", "type": "Keep", "basis": "Model-dependent levers rescaled by the measured gains", "before_value": 3454548, "after_value": 3466091, "unit": "SAR" }, { "no": 16, "page": "Forecast proof", "where_on_page": "Hero", "on_site_now": "the ML forecast made 22% less error than repeating last week", "replace_with": "Keep; add \"with the forecast 15% below actual sales\"", "type": "Add", "basis": "As row 1", "before_value": 22, "after_value": 22, "unit": "%" }, { "no": 17, "page": "Forecast proof", "where_on_page": "Tile: Van-week error removed", "on_site_now": "38.6% vs 47.2% weekly error", "replace_with": "40.0% vs 47.2% weekly error", "type": "Fix", "basis": "The level table on the same page uses the weekly run (40.0%); the tile used the daily run", "before_value": 38.6, "after_value": 40.0, "unit": "%" }, { "no": 18, "page": "Forecast proof", "where_on_page": "Section heading (Model vs planner)", "on_site_now": "Less error at every level where orders are placed", "replace_with": "Less error at SKU and van level; level with a simple average above that", "type": "Fix", "basis": "Principal: ML 28.3% vs 27.4% for the 4-week average; cluster 21.7% vs 21.5%", "before_value": null, "after_value": null, "unit": "" }, { "no": 19, "page": "Forecast proof", "where_on_page": "Principal level chip", "on_site_now": "\u22128% error", "replace_with": "Level with the 4-week average (28.3% vs 27.4%)", "type": "Fix", "basis": "Site scores, base demand weekly run", "before_value": null, "after_value": null, "unit": "" }, { "no": 20, "page": "Forecast proof", "where_on_page": "Company total level", "on_site_now": "About level \u00b7 ML 14.6% vs 14.9%", "replace_with": "Keep; add \"four weeks cannot rank models at total\"", "type": "Add", "basis": "Any flat forecast of SAR 1.08M-1.38M a week scores 16.2% (raw sales) in these four weeks", "before_value": null, "after_value": null, "unit": "" }, { "no": 21, "page": "Forecast proof", "where_on_page": "SKU level: ML forecast", "on_site_now": "63.8%", "replace_with": "Keep until re-run; expect about 1.4 points lower at the same bias", "type": "Recompute", "basis": "Raw-sales re-run: one equal blend for all SKUs 65.5% vs 66.9% at -17% bias (forecast_by_level.csv)", "before_value": 63.8, "after_value": null, "unit": "%" }, { "no": 22, "page": "Forecast proof", "where_on_page": "Van, principal, cluster: ML forecast", "on_site_now": "40.0% \u00b7 28.3% \u00b7 21.7%", "replace_with": "Keep", "type": "Keep", "basis": "Site blends beat our best by 0.6-1.4 points at the same bias (forecast_by_level.csv)", "before_value": null, "after_value": null, "unit": "" }, { "no": 23, "page": "Forecast proof", "where_on_page": "Model race text", "on_site_now": "Its forecast is a blend of the best one to three models", "replace_with": "SKUs: one equal blend of the three models best on validation. Vans: a blend of the best one to three", "type": "Recompute", "basis": "Per-SKU racing overfits: our copy scored 77.9%", "before_value": null, "after_value": null, "unit": "" }, { "no": 24, "page": "Forecast proof", "where_on_page": "Next 12 weeks: weekly range", "on_site_now": "The weekly range is the sum of daily p10 and p90", "replace_with": "Weekly bands calibrated on validation errors", "type": "Recompute", "basis": "Summed daily bands are too wide (the page says so)", "before_value": null, "after_value": null, "unit": "" }, { "no": 25, "page": "Forecast proof", "where_on_page": "Cluster cards: days in range", "on_site_now": "Ice Cream 54% \u00b7 Tea & Dairy Foods 93% of days in range", "replace_with": "Bands calibrated per cluster (target 80%)", "type": "Recompute", "basis": "Site cluster coverage 54%-93%", "before_value": null, "after_value": null, "unit": "" }, { "no": 26, "page": "Forecast proof", "where_on_page": "Next 12 weeks: company model", "on_site_now": "Company model: LightGBM + AutoETS + TiRex", "replace_with": "Company model: TiRex + AutoETS + AutoTheta (the blend tested on the hold-out)", "type": "Fix", "basis": "The forward blend was not the one tested", "before_value": null, "after_value": null, "unit": "" }, { "no": 27, "page": "Next-best SKU", "where_on_page": "Hero", "on_site_now": "outlets bought 12.0% of their ten recommended SKUs, against 4.0% for the best-sellers and 0.7% for random SKUs", "replace_with": "outlets bought 12.4% of their ten recommended SKUs, against 3.8% for the best-sellers and 0.8% for random SKUs", "type": "Replace", "basis": "As row 3", "before_value": 12.0, "after_value": 12.4, "unit": "%" }, { "no": 28, "page": "Next-best SKU", "where_on_page": "Tile", "on_site_now": "58.2% bought at least one pick", "replace_with": "61.4% bought at least one pick", "type": "Replace", "basis": "As row 3", "before_value": 58.2, "after_value": 61.4, "unit": "%" }, { "no": 29, "page": "Next-best SKU", "where_on_page": "Hold-out proof", "on_site_now": "4,097 outlets judged. Lift: 3.0\u00d7 over best-sellers, 18.2\u00d7 over random.", "replace_with": "4,097 outlets judged. Lift: 3.3\u00d7 over best-sellers, 16.3\u00d7 over random.", "type": "Replace", "basis": "As row 3", "before_value": 3.0, "after_value": 3.3, "unit": "\u00d7" }, { "no": 30, "page": "Next-best SKU", "where_on_page": "New line under the proof", "on_site_now": "(not shown)", "replace_with": "Top five picks: 14.6% bought (the list the value uses)", "type": "Add", "basis": "As row 3", "before_value": null, "after_value": 14.6, "unit": "%" }, { "no": 31, "page": "Next-best SKU", "where_on_page": "Method text", "on_site_now": "similar outlets' baskets, neighbourhood and van penetration, popularity", "replace_with": "similar outlets' baskets, neighbourhood and van penetration, popularity, plus EASE and RP3beta scores, ranked by a LightGBM ranker", "type": "Replace", "basis": "Model change", "before_value": null, "after_value": null, "unit": "" }, { "no": 32, "page": "Outlets", "where_on_page": "Hero", "on_site_now": "AUC 0.77 against 0.68 for the recency rule", "replace_with": "AUC 0.80 against 0.73 for the recency rule", "type": "Replace", "basis": "As row 4", "before_value": 0.77, "after_value": 0.8, "unit": "AUC" }, { "no": 33, "page": "Outlets", "where_on_page": "Tile: riskiest 20%", "on_site_now": "47.9% of the outlets that lapsed sat in the top 20% of risk", "replace_with": "59.2% of the outlets that lapsed sat in the top 20% of risk", "type": "Replace", "basis": "As row 4", "before_value": 47.9, "after_value": 59.2, "unit": "%" }, { "no": 34, "page": "Outlets", "where_on_page": "Text under the AUC", "on_site_now": "Top risk decile churns at 2.9\u00d7 the average rate (6.7% in the test window)", "replace_with": "Top risk decile churns at 4.0\u00d7 the average rate (7.8% in the test window)", "type": "Replace", "basis": "As row 4", "before_value": 2.9, "after_value": 4.0, "unit": "\u00d7" }, { "no": 35, "page": "Outlets", "where_on_page": "Test description", "on_site_now": "trained to 21 Aug", "replace_with": "trained on cut-offs to 30 Jun, tested at 25 Aug on outlets it never saw", "type": "Replace", "basis": "Test protocol of the new model", "before_value": null, "after_value": null, "unit": "" }, { "no": 36, "page": "Outlets", "where_on_page": "Risk tiers", "on_site_now": "High risk 57 outlets \u00b7 Medium risk 129 outlets", "replace_with": "Re-score every outlet with the new model (data/churn_scores_v2.csv)", "type": "Recompute", "basis": "Tier thresholds depend on the site scale", "before_value": null, "after_value": null, "unit": "" }, { "no": 37, "page": "Cluster harmonisation", "where_on_page": "Tile", "on_site_now": "0.90 model AUC on unseen outlets", "replace_with": "0.79 model AUC on unseen outlets (time-honest test)", "type": "Verify first", "basis": "As row 5", "before_value": 0.9, "after_value": 0.79, "unit": "AUC" }, { "no": 38, "page": "Cluster harmonisation", "where_on_page": "Proof: top 10% of ranked pairs", "on_site_now": "Adoption model 65% \u00b7 Channel popularity 40% \u00b7 Neighbours buy it 12%", "replace_with": "Adoption model 32% \u00b7 Channel popularity 30% \u00b7 Neighbours buy it 9%", "type": "Verify first", "basis": "As row 5", "before_value": 65, "after_value": 32, "unit": "%" }, { "no": 39, "page": "Cluster harmonisation", "where_on_page": "Proof: lift text", "on_site_now": "6.5\u00d7 more adoptions in the model's top 10% than in a random 10% (29.3% of top-10% pairs adopted vs 4.54% on average)", "replace_with": "3.2\u00d7 more adoptions in the model's top 10% than in a random 10% (18.0% of top-10% pairs adopted vs 5.66% on average)", "type": "Verify first", "basis": "As row 5", "before_value": 6.5, "after_value": 3.2, "unit": "\u00d7" }, { "no": 40, "page": "Cluster harmonisation", "where_on_page": "Ranking accuracy (AUC)", "on_site_now": "Adoption model 0.90 \u00b7 Channel popularity 0.71 \u00b7 Neighbours buy it 0.56", "replace_with": "Adoption model 0.79 \u00b7 Channel popularity 0.73 \u00b7 Neighbours buy it 0.46", "type": "Verify first", "basis": "As row 5", "before_value": 0.9, "after_value": 0.79, "unit": "AUC" }, { "no": 41, "page": "Cluster harmonisation", "where_on_page": "How well it ranks each cluster", "on_site_now": "Per-cluster AUC 0.67 to 0.86", "replace_with": "Keep; add \"mean within cluster 0.75\" next to the pooled AUC", "type": "Add", "basis": "Pooled AUC mostly reflects that Unilever HPC adopts far more often than the other teams", "before_value": null, "after_value": null, "unit": "" }, { "no": 42, "page": "Receivables", "where_on_page": "Headline", "on_site_now": "SAR 927K past due (24%)", "replace_with": "SAR 591K past due (15%) + SAR 336K older than the invoice window", "type": "Fix", "basis": "As row 11", "before_value": 927, "after_value": 591, "unit": "SAR K" }, { "no": 43, "page": "Value roll-up", "where_on_page": "Labour lever evidence", "on_site_now": "625 badge-sharing sessions flagged", "replace_with": "625 high or medium-risk badge sessions (198 high)", "type": "Fix", "basis": "Site badge-risk tiers: high 198, medium 427", "before_value": 625, "after_value": 625, "unit": "sessions" }, { "no": 44, "page": "Value roll-up", "where_on_page": "ML forecast to safety stock", "on_site_now": "SAR 65.2K a year \u00b7 + SAR 296K once", "replace_with": "Keep", "type": "Keep", "basis": "Same bias after re-run: SAR 69.8K + SAR 317K once; if forecasts must be unbiased SAR 42.6K + SAR 193K", "before_value": 65179, "after_value": 69779, "unit": "SAR" }, { "no": 45, "page": "Value roll-up", "where_on_page": "Whitespace / next-best SKU", "on_site_now": "SAR 105K a year", "replace_with": "Keep", "type": "Keep", "basis": "Top-5 precision barely moves: SAR 105.3K", "before_value": 104524, "after_value": 105335, "unit": "SAR" }, { "no": 46, "page": "Value roll-up", "where_on_page": "Churn early-warning and win-back", "on_site_now": "SAR 47.3K a year", "replace_with": "Up to SAR 53.4K after the new model is live", "type": "Recompute", "basis": "Upper bound: whole lever scaled by churners caught in the riskiest 20%", "before_value": 47302, "after_value": 53434, "unit": "SAR" } ] }