{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Notebook v2b \u2014 SHAP + net-demand completion for the v2 design\n",
    "Refits only the best model per hotel (XGBoost) with the identical pipeline and split as v2, then produces the SHAP tables (Section 3.5) and the net-demand translation (Section 3.6) that the v2 SHAP cell failed to produce."
   ]
  },
  {
   "cell_type": "code",
   "metadata": {},
   "execution_count": null,
   "outputs": [],
   "source": [
    "# ---------- Cell 1: Setup + load ----------\n",
    "import warnings; warnings.filterwarnings(\"ignore\")\n",
    "from pathlib import Path\n",
    "import numpy as np, pandas as pd\n",
    "import matplotlib; matplotlib.use(\"Agg\")\n",
    "import matplotlib.pyplot as plt\n",
    "from google.colab import files as colab_files\n",
    "OUT = Path(\"outputs\"); OUT.mkdir(exist_ok=True)\n",
    "CSV = Path(\"hotel_bookings.csv\")\n",
    "if not CSV.exists():\n",
    "    print(\"Please upload hotel_bookings.csv ...\")\n",
    "    up = colab_files.upload(); CSV = Path(list(up.keys())[0])\n",
    "df = pd.read_csv(CSV)\n",
    "MONTHS = {m:i+1 for i,m in enumerate([\"January\",\"February\",\"March\",\"April\",\"May\",\"June\",\"July\",\"August\",\"September\",\"October\",\"November\",\"December\"])}\n",
    "df[\"arrival_date\"] = pd.to_datetime(dict(year=df.arrival_date_year, month=df.arrival_date_month.map(MONTHS), day=df.arrival_date_day_of_month))\n",
    "DROP_ALWAYS = [\"reservation_status\",\"reservation_status_date\",\"assigned_room_type\",\"arrival_date_year\",\"arrival_date_month\",\"arrival_date_day_of_month\",\"arrival_date_week_number\"]\n",
    "DROP_MAIN = [\"country\",\"agent\",\"company\"]\n",
    "TARGET, DATE, HOTEL = \"is_canceled\", \"arrival_date\", \"hotel\"\n",
    "print(\"Loaded:\", df.shape)\n"
   ]
  },
  {
   "cell_type": "code",
   "metadata": {},
   "execution_count": null,
   "outputs": [],
   "source": [
    "# ---------- Cell 2: Refit XGBoost per hotel (same pipeline/split as v2) ----------\n",
    "from sklearn.compose import ColumnTransformer\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.impute import SimpleImputer\n",
    "from sklearn.preprocessing import OneHotEncoder, StandardScaler\n",
    "from xgboost import XGBClassifier\n",
    "fits = {}\n",
    "for h, g0 in df.groupby(HOTEL):\n",
    "    g = g0.sort_values(DATE).reset_index(drop=True)\n",
    "    feats = [c for c in g.columns if c not in DROP_ALWAYS + DROP_MAIN + [TARGET, DATE, HOTEL]]\n",
    "    k = int(0.75*len(g)); tr, te = g.iloc[:k], g.iloc[k:]\n",
    "    num = [c for c in feats if g[c].dtype != object]; cat = [c for c in feats if g[c].dtype == object]\n",
    "    pre = ColumnTransformer([(\"num\", Pipeline([(\"imp\", SimpleImputer(strategy=\"median\")), (\"sc\", StandardScaler())]), num),\n",
    "                             (\"cat\", Pipeline([(\"imp\", SimpleImputer(strategy=\"most_frequent\")), (\"oh\", OneHotEncoder(handle_unknown=\"ignore\"))]), cat)])\n",
    "    pipe = Pipeline([(\"pre\", pre), (\"mdl\", XGBClassifier(n_estimators=500, max_depth=6, learning_rate=0.05, subsample=0.9, colsample_bytree=0.9, eval_metric=\"logloss\", n_jobs=-1, random_state=42))])\n",
    "    pipe.fit(tr[feats], tr[TARGET].values)\n",
    "    fits[h] = (pipe, te, feats)\n",
    "    print(h, \"refit done | test n =\", len(te))\n"
   ]
  },
  {
   "cell_type": "code",
   "metadata": {},
   "execution_count": null,
   "outputs": [],
   "source": [
    "# ---------- Cell 3: SHAP per hotel ----------\n",
    "import shap\n",
    "ACTIONABLE = {\"deposit_type\",\"market_segment\",\"distribution_channel\",\"total_of_special_requests\",\"booking_changes\",\"meal\"}\n",
    "for h, (pipe, te, feats) in fits.items():\n",
    "    Xte = pipe.named_steps[\"pre\"].transform(te[feats])\n",
    "    feat_names = list(pipe.named_steps[\"pre\"].get_feature_names_out())\n",
    "    expl = shap.TreeExplainer(pipe.named_steps[\"mdl\"])\n",
    "    sv = np.asarray(expl.shap_values(Xte))\n",
    "    if sv.ndim == 3: sv = sv[:, :, 1]\n",
    "    def base_name(f):\n",
    "        s = f.split(\"__\",1)[-1]\n",
    "        if s in feats: return s\n",
    "        parts = s.split(\"_\")\n",
    "        for i in range(len(parts)-1, 0, -1):\n",
    "            cand = \"_\".join(parts[:i])\n",
    "            if cand in feats: return cand\n",
    "        return s\n",
    "    imp = pd.Series(np.abs(sv).mean(0), index=feat_names)\n",
    "    base = imp.groupby([base_name(f) for f in imp.index]).sum().sort_values(ascending=False)\n",
    "    out = base.rename(\"mean_abs_shap\").reset_index().rename(columns={\"index\":\"feature\"})\n",
    "    # recover base feature name without the one-hot suffix for common categorical patterns\n",
    "    print(f\"\\n===== SHAP {h} (XGBoost, test period) =====\")\n",
    "    print(out.head(15).round(4).to_string(index=False))\n",
    "    out.to_csv(OUT / f\"shap_global_onehot_{h.replace(' ','_')}.csv\", index=False)\n",
    "    tot = base.sum()\n",
    "    act = base[[i for i in base.index if i in ACTIONABLE]].sum()\n",
    "    print(\"Actionable share of total SHAP importance:\", round(float(act/tot), 3))\n",
    "    seg = te[\"market_segment\"].values\n",
    "    rows = {}\n",
    "    for s in sorted(set(seg)):\n",
    "        m = seg == s\n",
    "        if m.sum() >= 30:\n",
    "            v = pd.Series(np.abs(sv[m]).mean(0), index=feat_names)\n",
    "            rows[s] = v.groupby([base_name(f) for f in v.index]).sum()\n",
    "    segtab = pd.DataFrame(rows)\n",
    "    print(segtab.round(4).head(15).to_string())\n",
    "    segtab.to_csv(OUT / f\"shap_by_segment_{h.replace(' ','_')}.csv\")\n"
   ]
  },
  {
   "cell_type": "code",
   "metadata": {},
   "execution_count": null,
   "outputs": [],
   "source": [
    "# ---------- Cell 4: Net demand per hotel (XGBoost; threshold = train-derived FN 3x from v2 tables) ----------\n",
    "THR = {\"City Hotel\": 0.20, \"Resort Hotel\": 0.25}\n",
    "for h, (pipe, te, feats) in fits.items():\n",
    "    p = pipe.predict_proba(te[feats])[:,1]\n",
    "    d = te.copy(); d[\"p_cancel\"] = p; d[\"actual_cancel\"] = d[TARGET].values\n",
    "    daily = d.groupby(DATE).agg(bookings=(\"p_cancel\",\"size\"), expected_cancellations=(\"p_cancel\",\"sum\"), actual_cancellations=(\"actual_cancel\",\"sum\"), flagged_high_risk=(\"p_cancel\", lambda s: int((s >= THR[h]).sum()))).reset_index()\n",
    "    daily[\"net_expected_demand\"] = daily[\"bookings\"] - daily[\"expected_cancellations\"]\n",
    "    daily.to_csv(OUT / f\"daily_net_demand_{h.replace(' ','_')}.csv\", index=False)\n",
    "    print(f\"\\n===== {h} \u2014 threshold in use (FN 3x, from training): {THR[h]} =====\")\n",
    "    print(\"test days:\", len(daily), \"| bookings:\", int(daily.bookings.sum()), \"| expected canc:\", round(daily.expected_cancellations.sum(),1), \"| actual canc:\", int(daily.actual_cancellations.sum()), \"| flagged:\", int(daily.flagged_high_risk.sum()))\n",
    "    print(daily.head(10).round(2).to_string(index=False))\n",
    "    plt.figure(figsize=(11,4)); plt.plot(daily[DATE], daily[\"bookings\"], label=\"Bookings\", lw=1)\n",
    "    plt.plot(daily[DATE], daily[\"net_expected_demand\"], label=\"Net expected demand\", lw=1)\n",
    "    plt.plot(daily[DATE], daily[\"bookings\"]-daily[\"actual_cancellations\"], label=\"Actual arrivals\", lw=1, alpha=.7)\n",
    "    plt.legend(); plt.title(f\"Daily demand translation \u2014 {h} (XGBoost)\"); plt.tight_layout()\n",
    "    plt.savefig(OUT / f\"fig_daily_net_demand_{h.replace(' ','_')}.png\", dpi=200); plt.close()\n",
    "print(\"\\nv2b outputs written.\")\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "name": "python",
   "version": "3.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}