{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Predicting Hotel Booking Cancellations with Explainable Machine Learning\n",
    "### Analysis notebook v2 \u2014 aligned to the paper's Section 3 (per-hotel models)\n",
    "\n",
    "Run order: Runtime \u2192 Run all. When the data cell asks, upload `hotel_bookings.csv`.\n",
    "\n",
    "Design implemented here (exactly as written in the paper):\n",
    "- One model set **per hotel** (Resort Hotel, City Hotel), reported side by side\n",
    "- Predictors restricted to booking-time information; `reservation_status`, `reservation_status_date`, `assigned_room_type` excluded as post-outcome\n",
    "- **Main specification excludes `country`** (following Yang & Miao, 2024) and excludes `agent` / `company` identifiers; a **sensitivity run with country included** is reported at the end\n",
    "- Out-of-time split **within each hotel**: earliest 75% of arrivals = training, latest 25% = test\n",
    "- Operating threshold chosen **inside the training data only** (out-of-fold predictions), under FN:FP cost ratios 2x, 3x, 5x\n",
    "- SHAP computed on the **test period** for the best model of each hotel (global + market-segment level)\n",
    "- Predicted probabilities summed by arrival date \u2192 expected cancellations, net demand, overbooking buffer\n"
   ]
  },
  {
   "cell_type": "code",
   "metadata": {},
   "execution_count": null,
   "outputs": [],
   "source": [
    "# ---------- Cell 1: Setup ----------\n",
    "import warnings; warnings.filterwarnings(\"ignore\")\n",
    "from pathlib import Path\n",
    "import numpy as np, pandas as pd\n",
    "import matplotlib; matplotlib.use(\"Agg\")\n",
    "import matplotlib.pyplot as plt\n",
    "from google.colab import files as colab_files\n",
    "\n",
    "OUT = Path(\"outputs\"); OUT.mkdir(exist_ok=True)\n",
    "print(\"Setup done.\")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 1 \u2014 Load the data"
   ]
  },
  {
   "cell_type": "code",
   "metadata": {},
   "execution_count": null,
   "outputs": [],
   "source": [
    "# ---------- Cell 2: Load data ----------\n",
    "CSV = Path(\"hotel_bookings.csv\")\n",
    "if not CSV.exists():\n",
    "    print(\"Please upload hotel_bookings.csv ...\")\n",
    "    up = colab_files.upload()\n",
    "    CSV = Path(list(up.keys())[0])\n",
    "df = pd.read_csv(CSV)\n",
    "MONTHS = {m:i+1 for i,m in enumerate([\"January\",\"February\",\"March\",\"April\",\"May\",\"June\",\"July\",\"August\",\"September\",\"October\",\"November\",\"December\"])}\n",
    "df[\"arrival_date\"] = pd.to_datetime(dict(year=df.reservation_status_date.str.slice(0,4).astype(int)*0 + df.arrival_date_year, month=df.arrival_date_month.map(MONTHS), day=df.arrival_date_day_of_month))\n",
    "print(\"Shape:\", df.shape)\n",
    "print(\"Hotels:\", df.hotel.value_counts().to_dict())\n",
    "print(\"Overall cancellation rate:\", round(df.is_canceled.mean(),4))\n",
    "for h,g in df.groupby(\"hotel\"):\n",
    "    print(h, \"n =\", len(g), \"| cancel rate:\", round(g.is_canceled.mean(),4))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 2 \u2014 Leakage control and predictor set (Section 3.2)"
   ]
  },
  {
   "cell_type": "code",
   "metadata": {},
   "execution_count": null,
   "outputs": [],
   "source": [
    "# ---------- Cell 3: Predictor set ----------\n",
    "DROP_ALWAYS = [\"reservation_status\",\"reservation_status_date\",\"assigned_room_type\",\"arrival_date_year\",\"arrival_date_month\",\"arrival_date_day_of_month\",\"arrival_date_week_number\"]\n",
    "DROP_MAIN   = [\"country\",\"agent\",\"company\"]   # main specification (Section 3.2)\n",
    "TARGET, DATE, HOTEL = \"is_canceled\", \"arrival_date\", \"hotel\"\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 3 \u2014 Per-hotel estimation: split, tune-free pipelines, train-only threshold (Sections 3.2\u20133.4)"
   ]
  },
  {
   "cell_type": "code",
   "metadata": {},
   "execution_count": null,
   "outputs": [],
   "source": [
    "# ---------- Cell 4: Per-hotel modelling ----------\n",
    "from sklearn.compose import ColumnTransformer\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.impute import SimpleImputer\n",
    "from sklearn.preprocessing import OneHotEncoder, StandardScaler\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.model_selection import cross_val_predict\n",
    "from sklearn.metrics import roc_auc_score, average_precision_score, brier_score_loss, confusion_matrix\n",
    "from xgboost import XGBClassifier\n",
    "\n",
    "COST_RATIOS = [2.0, 3.0, 5.0]\n",
    "def best_threshold(y, p, fn_cost):\n",
    "    grid = np.arange(0.05, 0.96, 0.01); best, bt = np.inf, 0.5\n",
    "    for t in grid:\n",
    "        tn, fp, fn, tp = confusion_matrix(y, (p >= t).astype(int)).ravel()\n",
    "        c = fn_cost * fn + fp\n",
    "        if c < best: best, bt = c, float(t)\n",
    "    return bt\n",
    "\n",
    "def run_hotel(data, extra_drop):\n",
    "    data = data.sort_values(DATE).reset_index(drop=True)\n",
    "    feats = [c for c in data.columns if c not in DROP_ALWAYS + extra_drop + [TARGET, DATE, HOTEL]]\n",
    "    k = int(0.75 * len(data))\n",
    "    tr, te = data.iloc[:k], data.iloc[k:]\n",
    "    ytr, yte = tr[TARGET].values, te[TARGET].values\n",
    "    num = [c for c in feats if data[c].dtype != object]\n",
    "    cat = [c for c in feats if data[c].dtype == object]\n",
    "    pre = ColumnTransformer([\n",
    "        (\"num\", Pipeline([(\"imp\", SimpleImputer(strategy=\"median\")), (\"sc\", StandardScaler())]), num),\n",
    "        (\"cat\", Pipeline([(\"imp\", SimpleImputer(strategy=\"most_frequent\")), (\"oh\", OneHotEncoder(handle_unknown=\"ignore\"))]), cat)])\n",
    "    models = {\n",
    "        \"Logistic regression\": LogisticRegression(max_iter=1000, class_weight=\"balanced\"),\n",
    "        \"Random forest\": RandomForestClassifier(n_estimators=400, n_jobs=-1, random_state=42),\n",
    "        \"XGBoost\": XGBClassifier(n_estimators=500, max_depth=6, learning_rate=0.05, subsample=0.9, colsample_bytree=0.9, eval_metric=\"logloss\", n_jobs=-1, random_state=42)}\n",
    "    rows, store = [], {}\n",
    "    for name, mdl in models.items():\n",
    "        pipe = Pipeline([(\"pre\", pre), (\"mdl\", mdl)])\n",
    "        # threshold inside training data only: out-of-fold probabilities on the training period\n",
    "        oof = cross_val_predict(Pipeline([(\"pre\", pre), (\"mdl\", mdl)]), tr[feats], ytr, cv=3, method=\"predict_proba\")[:,1]\n",
    "        ths = {r: best_threshold(ytr, oof, r) for r in COST_RATIOS}\n",
    "        pipe.fit(tr[feats], ytr)\n",
    "        p = pipe.predict_proba(te[feats])[:,1]\n",
    "        pred = (p >= 0.5).astype(int)\n",
    "        tn, fp, fn, tp = confusion_matrix(yte, pred).ravel()\n",
    "        rows.append({\"Model\": name, \"ROC-AUC\": roc_auc_score(yte,p), \"PR-AUC\": average_precision_score(yte,p),\n",
    "                     \"Brier\": brier_score_loss(yte,p), \"Accuracy@0.5\": (pred==yte).mean(),\n",
    "                     \"Precision@0.5\": tp/(tp+fp), \"Recall@0.5\": tp/(tp+fn), \"F1@0.5\": 2*tp/(2*tp+fp+fn),\n",
    "                     \"Thr (FN 2x)\": ths[2.0], \"Thr (FN 3x)\": ths[3.0], \"Thr (FN 5x)\": ths[5.0]})\n",
    "        store[name] = (pipe, p)\n",
    "    res = pd.DataFrame(rows).round(4)\n",
    "    return res, store, tr, te, feats\n",
    "\n",
    "results_by_hotel, store_by_hotel, split_info = {}, {}, {}\n",
    "for h, g in df.groupby(HOTEL):\n",
    "    res, store, tr, te, feats = run_hotel(g, DROP_MAIN)\n",
    "    results_by_hotel[h] = res; store_by_hotel[h] = store\n",
    "    split_info[h] = (len(tr), len(te), tr[TARGET].mean(), te[TARGET].mean(), feats)\n",
    "    print(f\"\\n===== {h} =====  train n={len(tr)} (cancel {tr[TARGET].mean():.3f}) | test n={len(te)} (cancel {te[TARGET].mean():.3f}) | predictors={len(feats)}\")\n",
    "    print(res.to_string(index=False))\n",
    "    res.to_csv(OUT / f\"results_{h.replace(' ','_')}.csv\", index=False)\n"
   ]
  },
  {
   "cell_type": "code",
   "metadata": {},
   "source": [
    "\n",
    "# ---------- Cell 8: Sensitivity specifications ----------\n",
    "TRIO = [\"required_car_parking_spaces\",\"total_of_special_requests\",\"booking_changes\"]\n",
    "specs = {\"A_main\": DROP_MAIN, \"B_no_accrual\": DROP_MAIN + TRIO, \"C_no_deposit\": DROP_MAIN + [\"deposit_type\"], \"D_no_accrual_no_deposit\": DROP_MAIN + TRIO + [\"deposit_type\"]}\n",
    "stores = {}\n",
    "for spec, drops in specs.items():\n",
    "    print(f\"\\n########## SPEC {spec} (extra drops: {drops}) ##########\")\n",
    "    stores[spec] = {}\n",
    "    for h, g in df.groupby(HOTEL):\n",
    "        res, store, tr, te, feats = run_hotel(g, drops)\n",
    "        stores[spec][h] = (store, feats, te)\n",
    "        print(f\"--- {h}: predictors={len(feats)}\")\n",
    "        print(res[[\"Model\",\"ROC-AUC\",\"PR-AUC\",\"Brier\"]].round(4).to_string(index=False))\n"
   ],
   "outputs": [],
   "execution_count": null
  },
  {
   "cell_type": "code",
   "metadata": {},
   "source": [
    "\n",
    "# ---------- Cell 9: SHAP for Spec B (no accrual trio), XGBoost; strict actionable shares ----------\n",
    "import shap\n",
    "STRICT_ACTIONABLE = {\"deposit_type\",\"distribution_channel\"}\n",
    "BROAD_ACTIONABLE = {\"deposit_type\",\"market_segment\",\"distribution_channel\",\"total_of_special_requests\",\"booking_changes\",\"meal\"}\n",
    "def shap_base(pipe, te, feats):\n",
    "    Xte = pipe.named_steps[\"pre\"].transform(te[feats])\n",
    "    try: feat_names = list(pipe.named_steps[\"pre\"].get_feature_names_out())\n",
    "    except Exception: feat_names = [f\"f{i}\" for i in range(Xte.shape[1])]\n",
    "    expl = shap.TreeExplainer(pipe.named_steps[\"mdl\"]); sv = np.asarray(expl.shap_values(Xte))\n",
    "    if isinstance(sv, list): sv = sv[1]\n",
    "    if sv.ndim == 3: sv = sv[:, :, 1]\n",
    "    imp = pd.DataFrame({\"feature\": feat_names, \"m\": np.abs(sv).mean(0)})\n",
    "    imp[\"base\"] = imp[\"feature\"].map(lambda f: f.split(\"__\",1)[-1])\n",
    "    return imp.groupby(\"base\")[\"m\"].sum().sort_values(ascending=False)\n",
    "for spec in [\"A_main\",\"B_no_accrual\"]:\n",
    "    for h in results_by_hotel:\n",
    "        store, feats, te = stores[spec][h]\n",
    "        pipe, p = store[\"XGBoost\"]\n",
    "        g = df[df[HOTEL]==h].sort_values(DATE).reset_index(drop=True)\n",
    "        k = int(0.75*len(g)); te2 = g.iloc[k:]\n",
    "        base = shap_base(pipe, te2, feats)\n",
    "        tot = base.sum()\n",
    "        print(f\"\\n===== SHAP {spec} \u2014 {h} (XGBoost) =====\")\n",
    "        print(base.head(12).round(4).to_string())\n",
    "        print(\"strict actionable share (deposit+channel):\", round(base[base.index.isin(STRICT_ACTIONABLE)].sum()/tot, 3))\n",
    "        print(\"broad actionable share:\", round(base[base.index.isin(BROAD_ACTIONABLE)].sum()/tot, 3))\n",
    "        dep = base.get(\"deposit_type\", 0.0)\n",
    "        print(\"deposit share alone:\", round(dep/tot, 3), \"| strict share excluding deposit:\", round(base[base.index.isin(STRICT_ACTIONABLE-{\"deposit_type\"})].sum()/tot, 3))\n",
    "print(\"\\nALL DONE\")\n"
   ],
   "outputs": [],
   "execution_count": null
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "name": "python",
   "version": "3.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}