# -*- coding: utf-8 -*-
"""
Heterogeneous effects of intrapartum interventions on labor duration:
honest causal forest analysis of a real-world obstetric cohort.

Exposure 1 (primary): maintained epidural analgesia vs Lamaze breathing
  (sensitivity: epidural vs no analgesia)
Exposure 2: intrapartum uterotonic augmentation (oxytocin or artificial
  rupture of membranes) vs none.

Method: honest causal forest (Athey & Imbens 2016; Athey, Tibshirani &
Wager 2019) with DML residualization for observational data, implemented
in pure scikit-learn/numpy. Adjusted ATEs via cross-fitted DML
(continuous outcomes) and doubly robust AIPW estimation (binary outcomes).

Input: Supplementary_File_S2_Dataset.xlsx (de-identified analytic dataset,
n = 2,111 singleton parturients who attempted vaginal birth; all covariates
are pre-intrapartum characteristics; Chinese source-registry fields were
parsed upstream as described in the Methods section).

Dependencies: pandas, numpy, scipy, scikit-learn, statsmodels, matplotlib
Usage:    python Supplementary_File_S3_Analysis_code.py [path_to_S2_xlsx]
"""
import sys
import numpy as np
import pandas as pd
import scipy.stats as st
import statsmodels.api as sm
import matplotlib.pyplot as plt
from sklearn.impute import SimpleImputer
from sklearn.ensemble import RandomForestRegressor, RandomForestClassifier
from sklearn.model_selection import KFold

DATA = sys.argv[1] if len(sys.argv) > 1 else "Supplementary_File_S2_Dataset.xlsx"
OUT = "."
SEED = 42

# ---------------- Data loading ----------------
def load_data(path):
    """Load the de-identified analytic dataset (already cleaned upstream:
    gestational ages parsed to weeks, labor stages to minutes, binary
    indicators coded 0/1, planned cesareans and multiple gestations excluded)."""
    return pd.read_excel(path)

COVS = ['maternal_age_y', 'gravidity', 'parity', 'ga_delivery_weeks',
        'ga_registration_weeks', 'risk_score_0_4', 'scarred_uterus',
        'accompanied_labor', 'hbsag_positive', 'syphilis_ab_positive',
        'nonlocal_registration']
LABELS = dict(zip(COVS, ['Maternal age', 'Gravidity', 'Parity',
    'GA at delivery', 'GA at registration', 'Risk score', 'Scarred uterus',
    'Accompanied labor', 'HBsAg positive', 'Syphilis Ab positive',
    'Non-local registration']))

def prep_X(d):
    return SimpleImputer(strategy='median').fit_transform(d[COVS].astype(float).values)

# ---------------- Honest Causal Forest ----------------
class HonestCausalForest:
    """Honest causal forest with DML residualization.

    Each tree: a 70% subsample is split into a structure half A (used only
    to select splits; splitting criterion = maximize between-child
    difference in tau) and an honest estimation half B (used only to
    estimate leaf effects). Leaf effects tau = sum(Yr*Tr)/sum(Tr^2),
    where Yr and Tr are cross-fitted residuals of outcome and treatment.
    """
    def __init__(self, n_trees=400, min_leaf=60, max_depth=6, n_try=4, n_thresh=16,
                 subsample_ratio=0.7, honest_split=0.5, seed=42):
        self.n_trees, self.min_leaf, self.max_depth = n_trees, min_leaf, max_depth
        self.n_try, self.n_thresh = n_try, n_thresh
        self.subsample_ratio, self.honest_split = subsample_ratio, honest_split
        self.seed = seed
    def _tau(self, yr, tr):
        d = (tr**2).sum()
        return (yr*tr).sum()/d if d > 1e-6 else 0.0
    def _best_split(self, X, yr, tr, idx, rng):
        n = len(idx); best = (0.0, None, None); p = X.shape[1]
        for j in rng.choice(p, size=min(self.n_try, p), replace=False):
            xj = X[idx, j]; order = np.argsort(xj, kind='mergesort')
            s_idx = idx[order]; xs = xj[order]
            for c in np.unique(np.quantile(xs, np.linspace(0.1, 0.9, self.n_thresh))):
                if c <= xs[0] or c >= xs[-1]: continue
                k = np.searchsorted(xs, c, side='right')
                if k < self.min_leaf or n-k < self.min_leaf: continue
                yrL, trL = yr[s_idx[:k]], tr[s_idx[:k]]; yrR, trR = yr[s_idx[k:]], tr[s_idx[k:]]
                if (trL**2).sum() < 1.0 or (trR**2).sum() < 1.0: continue
                gain = k*(n-k)/n * (self._tau(yrL,trL) - self._tau(yrR,trR))**2
                if gain > best[0]: best = (gain, j, c)
        return best
    def _grow(self, X, yr, tr, idx, depth, rng):
        gain, j, c = self._best_split(X, yr, tr, idx, rng) if depth < self.max_depth else (0,None,None)
        if gain <= 1e-9 or j is None: return ('leaf', None, idx)
        L = idx[X[idx,j]<=c]; R = idx[X[idx,j]>c]
        return ('node', j, float(c), self._grow(X,yr,tr,L,depth+1,rng), self._grow(X,yr,tr,R,depth+1,rng))
    def _honest_fill(self, tree, X, yr, tr, B, fb):
        if tree[0]=='leaf':
            if len(B) >= max(10, self.min_leaf//3) and (tr[B]**2).sum() > 1.0:
                return ('leaf', self._tau(yr[B], tr[B]), None)
            return ('leaf', fb, None)
        return ('node', tree[1], tree[2],
                self._honest_fill(tree[3], X, yr, tr, B[X[B,tree[1]]<=tree[2]], fb),
                self._honest_fill(tree[4], X, yr, tr, B[X[B,tree[1]]>tree[2]], fb))
    def _predict_tree(self, tree, x):
        while tree[0]=='node': tree = tree[3] if x[tree[1]]<=tree[2] else tree[4]
        return tree[1]
    def _count(self, tree):
        if tree[0]=='node':
            self.split_count_[tree[1]] += 1; self._count(tree[3]); self._count(tree[4])
    def fit_predict(self, X, T, Y, mhat, ehat):
        rng = np.random.default_rng(self.seed); n = len(Y)
        self.trees_, self.split_count_ = [], np.zeros(X.shape[1])
        yr, tr = Y-mhat, T-np.clip(ehat,0.05,0.95)
        for b in range(self.n_trees):
            sub = rng.choice(n, size=int(self.subsample_ratio*n), replace=False)
            half = int(self.honest_split*len(sub)); A, B = sub[:half], sub[half:]
            raw = self._grow(X, yr, tr, A, 0, rng)
            tree = self._honest_fill(raw, X, yr, tr, B, self._tau(yr[A], tr[A]))
            self.trees_.append(tree); self._count(tree)
        return np.array([np.mean([self._predict_tree(t, X[i]) for t in self.trees_]) for i in range(n)])
    def predict(self, Xnew):
        return np.array([np.mean([self._predict_tree(t, Xnew[i]) for t in self.trees_]) for i in range(len(Xnew))])

# ---------------- Nuisances and effect estimators ----------------
def nuisances(d, tcol, ocol, seed=SEED, n_splits=5):
    """5-fold cross-fitted random-forest nuisances; returns AIPW pseudo-outcomes."""
    dd = d.dropna(subset=[ocol, tcol]).reset_index(drop=True)
    X = prep_X(dd); T = dd[tcol].values.astype(float); Y = dd[ocol].values.astype(float)
    kf = KFold(n_splits=n_splits, shuffle=True, random_state=seed)
    mhat=np.zeros(len(Y)); ehat=np.zeros(len(Y)); mu1=np.zeros(len(Y)); mu0=np.zeros(len(Y))
    for tr, te in kf.split(X):
        mhat[te] = RandomForestRegressor(n_estimators=400, min_samples_leaf=20, n_jobs=-1, random_state=seed).fit(X[tr], Y[tr]).predict(X[te])
        mu1[te] = RandomForestRegressor(n_estimators=300, min_samples_leaf=20, n_jobs=-1, random_state=seed).fit(X[tr][T[tr]==1], Y[tr][T[tr]==1]).predict(X[te])
        mu0[te] = RandomForestRegressor(n_estimators=300, min_samples_leaf=20, n_jobs=-1, random_state=seed).fit(X[tr][T[tr]==0], Y[tr][T[tr]==0]).predict(X[te])
        ehat[te] = np.clip(RandomForestClassifier(n_estimators=400, min_samples_leaf=20, n_jobs=-1, random_state=seed).fit(X[tr], T[tr]).predict_proba(X[te])[:,1], 0.05, 0.95)
    aipw = (mu1-mu0) + T*(Y-mu1)/ehat - (1-T)*(Y-mu0)/(1-ehat)
    return dd, X, T, Y, mhat, ehat, aipw

def dml_plr(X, T, Y, seed=SEED):
    """Cross-fitted DML (partially linear model) ATE with standard error."""
    kf = KFold(5, shuffle=True, random_state=seed)
    mhat=np.zeros(len(Y)); ehat=np.zeros(len(Y))
    for tr, te in kf.split(X):
        mhat[te] = RandomForestRegressor(400, min_samples_leaf=20, n_jobs=-1, random_state=seed).fit(X[tr], Y[tr]).predict(X[te])
        ehat[te] = np.clip(RandomForestClassifier(400, min_samples_leaf=20, n_jobs=-1, random_state=seed).fit(X[tr], T[tr]).predict_proba(X[te])[:,1], 0.02, 0.98)
    Yr, Tr = Y-mhat, T-ehat
    ate = np.mean(Yr*Tr)/np.mean(Tr**2)
    psi = Yr*Tr - ate*Tr**2
    return ate, np.sqrt(np.mean(psi**2)/len(Y))/np.mean(Tr**2)

def dr_ate(X, T, Y, seed=SEED):
    """Cross-fitted doubly robust (AIPW) ATE with standard error."""
    kf = KFold(5, shuffle=True, random_state=seed)
    mu1=np.zeros(len(Y)); mu0=np.zeros(len(Y)); ehat=np.zeros(len(Y))
    for tr, te in kf.split(X):
        mu1[te]=RandomForestRegressor(300, min_samples_leaf=20, n_jobs=-1, random_state=seed).fit(X[tr][T[tr]==1],Y[tr][T[tr]==1]).predict(X[te])
        mu0[te]=RandomForestRegressor(300, min_samples_leaf=20, n_jobs=-1, random_state=seed).fit(X[tr][T[tr]==0],Y[tr][T[tr]==0]).predict(X[te])
        ehat[te]=np.clip(RandomForestClassifier(400, min_samples_leaf=20, n_jobs=-1, random_state=seed).fit(X[tr],T[tr]).predict_proba(X[te])[:,1],0.02,0.98)
    psi=(mu1-mu0)+T*(Y-mu1)/ehat-(1-T)*(Y-mu0)/(1-ehat)
    return psi.mean(), psi.std(ddof=1)/np.sqrt(len(Y))

def run_forest(d, tcol, ocol, seed):
    dd, X, T, Y, mhat, ehat, aipw = nuisances(d, tcol, ocol, seed=seed)
    cf = HonestCausalForest(n_trees=400, min_leaf=60, max_depth=6, n_try=4, seed=seed)
    cate = cf.fit_predict(X, T, Y, mhat, ehat)
    return dd, X, T, Y, mhat, ehat, aipw, cate, cf

def diagnose(res, label):
    """Heterogeneity diagnostics: quartile calibration, calibration
    regression, best linear projection (BLP), variable importance."""
    dd, X, T, Y, mhat, ehat, aipw, cate, cf = res
    print(f"\n===== {label} (n={len(dd)}) =====")
    print("CATE: mean=%.1f sd=%.1f" % (cate.mean(), cate.std()))
    q = pd.qcut(cate, 4, labels=['Q1','Q2','Q3','Q4'])
    qt = []
    for qi, idx in pd.Series(np.arange(len(dd))).groupby(q, observed=True):
        a = aipw[idx.values]
        qt.append({'quartile':qi,'n':len(a),'AIPW_mean':round(a.mean(),1),'SE':round(a.std(ddof=1)/np.sqrt(len(a)),1)})
    qt = pd.DataFrame(qt).set_index('quartile'); print(qt)
    diff = qt['AIPW_mean'].iloc[-1]-qt['AIPW_mean'].iloc[0]
    se = np.sqrt(qt['SE'].iloc[-1]**2+qt['SE'].iloc[0]**2)
    print("Q4-Q1 = %.1f +- %.1f, z=%.2f, p=%.4f" % (diff, se, diff/se, 2*(1-st.norm.cdf(abs(diff/se)))))
    fit = sm.OLS(aipw, sm.add_constant(cate)).fit()
    print("Calibration regression: intercept=%.1f (p=%.3f), CATE slope=%.2f (p=%.3f)" % (fit.params[0], fit.pvalues[0], fit.params[1], fit.pvalues[1]))
    fb = sm.OLS(aipw, sm.add_constant(np.column_stack([cate, X]))).fit()
    print("BLP joint F-test p=%.3f" % fb.f_test(np.eye(X.shape[1]+2)[2:]).pvalue)
    vi = pd.Series(cf.split_count_/cf.split_count_.sum(), index=[LABELS[c] for c in COVS]).sort_values(ascending=False)
    print("VI:", dict(vi.round(3)))
    return qt, vi

def stratified_outcome(d, tcol, ocol, res, label):
    """Doubly robust effect on a secondary outcome within labor-CATE quartiles."""
    dd_o = d.dropna(subset=[ocol, tcol]).reset_index(drop=True)
    Xo = prep_X(dd_o); To = dd_o[tcol].values.astype(float); Yo = dd_o[ocol].values.astype(float)
    cate_o = res[8].predict(Xo)
    _,_,_,_,_,_, aipw_o = nuisances(d, tcol, ocol, seed=99)
    q = pd.qcut(cate_o, 4, labels=['Q1','Q2','Q3','Q4'])
    rows=[]
    for qi, idx in pd.Series(np.arange(len(dd_o))).groupby(q, observed=True):
        a = aipw_o[idx.values]
        rows.append({'quartile':qi,'n':len(a),'DR_effect':round(a.mean(),3),'SE':round(a.std(ddof=1)/np.sqrt(len(a)),3)})
    r = pd.DataFrame(rows).set_index('quartile')
    print(f"\n{label}: {ocol} stratified by labor-CATE quartile"); print(r)
    d_ = r['DR_effect'].iloc[-1]-r['DR_effect'].iloc[0]
    se = np.sqrt(r['SE'].iloc[-1]**2+r['SE'].iloc[0]**2)
    print("Q4-Q1 = %.3f +- %.3f, z=%.2f, p=%.4f" % (d_, se, d_/se, 2*(1-st.norm.cdf(abs(d_/se)))))
    return r

# ---------------- Main pipeline ----------------
if __name__ == "__main__":
    labor = load_data(DATA)
    d1  = labor.dropna(subset=['epidural_vs_none'])     # sensitivity: epidural vs no analgesia
    d1b = labor.dropna(subset=['epidural_vs_lamaze'])   # primary: epidural vs Lamaze
    d2  = labor.dropna(subset=['augmentation'])         # exposure 2: augmentation vs none
    print("Attempted vaginal birth n=%d | primary contrast n=%d | augmentation n=%d"
          % (len(labor), len(d1b), len(d2)))

    res1m = run_forest(d1b, 'epidural_vs_lamaze', 'labor_total_min', seed=11)
    res1s = run_forest(d1,  'epidural_vs_none',   'labor_total_min', seed=13)
    res2  = run_forest(d2,  'augmentation',       'labor_total_min', seed=7)

    diagnose(res1m, "Primary contrast: epidural vs Lamaze")
    diagnose(res1s, "Sensitivity contrast: epidural vs no analgesia")
    qt2, vi2 = diagnose(res2, "Exposure 2: augmentation vs none")

    r_ces1 = stratified_outcome(d1b, 'epidural_vs_lamaze', 'intrapartum_cesarean', res1m, "Primary contrast")
    r_epi1 = stratified_outcome(d1b, 'epidural_vs_lamaze', 'episiotomy', res1m, "Primary contrast")
    r_ces2 = stratified_outcome(d2,  'augmentation', 'intrapartum_cesarean', res2, "Exposure 2")

    # Figure: CATE distributions + quartile calibration
    fig, axes = plt.subplots(2, 2, figsize=(12, 9))
    (ax1, ax2), (ax3, ax4) = axes
    for ax, res, lab, col in [(ax1, res1m, 'Primary: epidural vs Lamaze', '#4C72B0'),
                              (ax2, res2, 'Exposure 2: augmentation vs none', '#55A868')]:
        cate = res[7]
        ax.hist(cate, bins=30, color=col, alpha=0.85, edgecolor='white')
        ax.axvline(cate.mean(), color='k', ls='--', lw=1.2, label=f'mean {cate.mean():.0f} min')
        ax.set_title(lab, fontsize=12); ax.set_xlabel('CATE (effect on total labor duration, min)')
        ax.set_ylabel('N'); ax.legend()
    for ax, res, lab, col in [(ax3, res1m, 'Primary', '#4C72B0'), (ax4, res2, 'Exposure 2', '#55A868')]:
        cate, aipw = res[7], res[6]
        q = pd.qcut(cate, 4, labels=['Q1','Q2','Q3','Q4'])
        means = [aipw[q==qi].mean() for qi in ['Q1','Q2','Q3','Q4']]
        ses = [aipw[q==qi].std(ddof=1)/np.sqrt((q==qi).sum()) for qi in ['Q1','Q2','Q3','Q4']]
        ax.errorbar(range(4), means, yerr=1.96*np.array(ses), fmt='o-', color=col, capsize=4, lw=2, ms=7)
        ax.set_xticks(range(4)); ax.set_xticklabels(['Q1\nlowest','Q2','Q3','Q4\nhighest'], fontsize=10)
        ax.set_xlabel('CATE quartile', fontsize=10)
        ax.set_ylabel('Within-quartile AIPW effect (min)')
        ax.set_title(f'{lab}: calibration by CATE quartile', fontsize=12); ax.grid(alpha=0.3)
    plt.tight_layout(); plt.savefig(f'{OUT}/Figure_CATE_distribution_calibration.png', dpi=150); plt.close()

    # Figure: VI + stratified outcomes
    dd, X, T, Y, mhat, ehat, aipw, cate, cf = res1m
    vi = pd.Series(cf.split_count_/cf.split_count_.sum(), index=[LABELS[c] for c in COVS]).sort_values(ascending=False)
    fig, axes = plt.subplots(1, 3, figsize=(15, 4.2))
    vi.round(3).iloc[::-1].plot.barh(ax=axes[0], color='#4C72B0', title='Primary contrast: variable importance (split frequency)')
    for lab, r, col, ax in [('Primary: intrapartum cesarean', r_ces1, '#4C72B0', axes[1]),
                            ('Primary: episiotomy', r_epi1, '#8172b3', axes[2])]:
        ax.errorbar(range(4), r['DR_effect'], yerr=1.96*r['SE'], fmt='o-', color=col, capsize=4, lw=2, ms=7)
        ax.axhline(0, color='gray', lw=0.8)
        ax.set_xticks(range(4)); ax.set_xticklabels(['Q1','Q2','Q3','Q4'])
        ax.set_title(f'{lab}: DR effect by labor-CATE quartile', fontsize=11)
        ax.set_ylabel('DR effect'); ax.grid(alpha=0.3)
    plt.tight_layout(); plt.savefig(f'{OUT}/Figure_VI_stratified_outcomes.png', dpi=150); plt.close()
    print("\nFigures saved.")
