#!/usr/bin/env python3
"""Reproducible analysis of diabetes treatment coverage in 28 South Asia/MENA countries.
Source data: NCD-RisC 2024 country estimates; World Bank WGI 2026/WDI exports.
Outcomes are model-based point estimates; inference for ecological regressions is conditional
on those point estimates. Does not claim posterior uncertainty propagation.
"""
import json, hashlib, sys
from pathlib import Path
import numpy as np
import pandas as pd
from scipy import stats
import statsmodels.formula.api as smf
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
from matplotlib.lines import Line2D

ROOT=Path(__file__).resolve().parents[1]
RAW=ROOT/'data'/'raw'; PROC=ROOT/'data'/'processed'; TABLES=ROOT/'results'/'tables'; FIGS=ROOT/'results'/'figures'; SUPP=ROOT/'supplementary'
for d in [PROC,TABLES,FIGS,SUPP]:d.mkdir(parents=True,exist_ok=True)
REGIONS={'South Asia':['AFG','BGD','BTN','IND','MDV','NPL','PAK','LKA'],
         'MENA':['BHR','DJI','EGY','IRN','IRQ','JOR','KWT','LBN','LBY','MAR','OMN','PSE','QAT','SAU','SOM','SDN','SYR','TUN','ARE','YEM']}
NAMES={'AFG':'Afghanistan','BHR':'Bahrain','DJI':'Djibouti','EGY':'Egypt','IRN':'Iran','IRQ':'Iraq','JOR':'Jordan','KWT':'Kuwait','LBN':'Lebanon','LBY':'Libya','MAR':'Morocco','OMN':'Oman','PSE':'Palestine','PAK':'Pakistan','QAT':'Qatar','SAU':'Saudi Arabia','SOM':'Somalia','SDN':'Sudan','SYR':'Syria','TUN':'Tunisia','ARE':'United Arab Emirates','YEM':'Yemen','BGD':'Bangladesh','BTN':'Bhutan','IND':'India','MDV':'Maldives','NPL':'Nepal','LKA':'Sri Lanka'}
CODES=[c for li in REGIONS.values() for c in li]
assert len(CODES)==28 and len(set(CODES))==28
intervals=[(2000,2005),(2005,2010),(2010,2015),(2015,2019),(2019,2022)]

def wb(filename,key):
    v=json.loads((RAW/filename).read_text(encoding='utf-8'))
    assert isinstance(v,list) and len(v)==2 and v[1]
    out=pd.DataFrame([{'ISO':r['countryiso3code'],'Year':int(r['date']),key:r.get('value')} for r in v[1] if r and r.get('date') and r.get('countryiso3code') in CODES])
    assert not out.duplicated(['ISO','Year']).any(),filename
    return out

raw=pd.read_csv(RAW/'NCD_RisC_Lancet_2024_Diabetes_age_standardised_countries.csv')
assert {'ISO','Year','Sex','Age'}.issubset(raw.columns)
x=raw[raw.ISO.isin(CODES)&raw.Year.between(2000,2022)&raw.Age.eq('Age-standardised')].copy()
column='Proportion of people with diabetes who were treated (30+ years)'
x=x[['ISO','Year','Sex',column,column+' lower 95% uncertainty interval',column+' upper 95% uncertainty interval']].rename(columns={column:'treat',column+' lower 95% uncertainty interval':'lo',column+' upper 95% uncertainty interval':'hi'})
for k in ['treat','lo','hi']:x[k]=x[k].astype(float)*100
x['region']=x.ISO.map({c:reg for reg,cs in REGIONS.items() for c in cs});x['country']=x.ISO.map(NAMES)
assert len(x)==1288 and not x.duplicated(['ISO','Year','Sex']).any()
assert x.treat.between(0,100).all() and (x.lo<=x.treat).all() and (x.treat<=x.hi).all()
x=x.sort_values(['region','ISO','Sex','Year']).reset_index(drop=True)
x.to_csv(PROC/'ncd_treatment_country_sex_year.csv',index=False)

gov=wb('worldbank_GOV_WGI_GE.SC.json','wgi'); low=wb('worldbank_GOV_WGI_GE.SC_LB.json','wgi_lo');high=wb('worldbank_GOV_WGI_GE.SC_UB.json','wgi_hi')
gdp=wb('worldbank_NY.GDP.PCAP.KD.json','gdp_pc');urban=wb('worldbank_SP.URB.TOTL.IN.ZS.json','urban_pct')
for a in [low,high,gdp,urban]:gov=gov.merge(a,on=['ISO','Year'],how='left',validate='one_to_one')
gov=gov.sort_values(['ISO','Year'])
gov.to_csv(PROC/'worldbank_context_country_year.csv',index=False)

# Point-estimate trends and cross-country dispersion: no within-source inferential claim.
def disp(a):return pd.Series({'n_countries':a.ISO.nunique(),'mean':a.treat.mean(),'median':a.treat.median(),'q25':a.treat.quantile(.25),'q75':a.treat.quantile(.75),'iqr':a.treat.quantile(.75)-a.treat.quantile(.25),'p10_p90_gap':a.treat.quantile(.9)-a.treat.quantile(.1)})
trend=x.groupby(['region','Sex','Year']).apply(disp,include_groups=False).reset_index()
trend.to_csv(TABLES/'regional_annual_coverage_and_dispersion.csv',index=False)
end=trend[trend.Year.isin([2000,2022])].pivot(index=['region','Sex'],columns='Year',values=['mean','median','iqr','p10_p90_gap']).reset_index()
end.columns=['_'.join(str(z) for z in c if str(z)) for c in end.columns]
end['change_mean_pp']=end['mean_2022']-end['mean_2000']
end['change_iqr_pp']=end['iqr_2022']-end['iqr_2000']
end['change_p10_p90_gap_pp']=end['p10_p90_gap_2022']-end['p10_p90_gap_2000']
end.to_csv(TABLES/'table1_regional_trends_and_inequality.csv',index=False)

sex=x.pivot(index=['ISO','country','region','Year'],columns='Sex',values=['treat','lo','hi'])
sex.columns=['_'.join(v) for v in sex.columns]
sex=sex.reset_index(); sex['women_minus_men_pp']=sex['treat_Women']-sex['treat_Men']
sex.to_csv(SUPP/'Additional_file_1_country_sex_year.csv',index=False)
sex_summary=sex.groupby(['region','Year']).agg(n_countries=('ISO','nunique'),mean_sex_gap=('women_minus_men_pp','mean'),median_sex_gap=('women_minus_men_pp','median'),n_women_higher=('women_minus_men_pp',lambda a:int((a>0).sum()))).reset_index()
sex_summary.to_csv(TABLES/'regional_sex_gap_annual.csv',index=False)
sex_table=sex_summary[sex_summary.Year.isin([2000,2022])].copy()
sex_table.to_csv(TABLES/'table2_sex_gaps.csv',index=False)

cend=x[x.Year.isin([2000,2022])].pivot(index=['ISO','country','region','Sex'],columns='Year',values=['treat','lo','hi']).reset_index()
cend.columns=['_'.join(str(z) for z in c if str(z)) for c in cend.columns]
cend['change_pp']=cend['treat_2022']-cend['treat_2000']
cend['envelope_min_pp']=cend['lo_2022']-cend['hi_2000']
cend['envelope_max_pp']=cend['hi_2022']-cend['lo_2000']
cend['source_marginal_envelope_excludes_zero']=(cend.envelope_min_pp>0)|(cend.envelope_max_pp<0)
cend.to_csv(SUPP/'Additional_file_2_country_endpoint_uncertainty_envelopes.csv',index=False)
country_change=cend.groupby(['ISO','country','region'],as_index=False).agg(equal_sex_change_pp=('change_pp','mean'),min_sex_change_pp=('change_pp','min'),max_sex_change_pp=('change_pp','max'))
country_change.to_csv(TABLES/'country_change.csv',index=False)

# Country-period data: one independent governance exposure per country-period, not doubled by sex.
records=[]
for y0,y1 in intervals:
  z0=sex[sex.Year.eq(y0)].set_index('ISO');z1=sex[sex.Year.eq(y1)].set_index('ISO')
  w0=gov[gov.Year.eq(y0)].set_index('ISO');w1=gov[gov.Year.eq(y1)].set_index('ISO')
  for iso in CODES:
    dur=y1-y0
    a=z0.loc[iso];b=z1.loc[iso];va=w0.loc[iso];vb=w1.loc[iso]
    rec={'ISO':iso,'country':NAMES[iso],'region':a['region'],'interval':f'{y0}-{y1}', 'start_year':y0,'end_year':y1,'years':dur}
    for sexid in ['Men','Women']:
      rec[f'change_{sexid.lower()}_pp_year']=(b[f'treat_{sexid}']-a[f'treat_{sexid}'])/dur
    rec['change_equal_sex_pp_year']=(rec['change_men_pp_year']+rec['change_women_pp_year'])/2
    rec['wgi_annualized']=(vb.wgi-va.wgi)/dur
    rec['wgi_lb0']=va.wgi_lo;rec['wgi_ub0']=va.wgi_hi
    rec['wgi_lb1']=vb.wgi_lo;rec['wgi_ub1']=vb.wgi_hi
    rec['wgi_nonoverlap_90pct_bounds']=bool(va.wgi_hi<vb.wgi_lo or vb.wgi_hi<va.wgi_lo)
    rec['wgi_endpoint_envelope_lo']=(vb.wgi_lo-va.wgi_hi)/dur
    rec['wgi_endpoint_envelope_hi']=(vb.wgi_hi-va.wgi_lo)/dur
    rec['log_gdp_pc_annualized']=(np.log(vb.gdp_pc)-np.log(va.gdp_pc))/dur if pd.notna(va.gdp_pc) and pd.notna(vb.gdp_pc) and va.gdp_pc>0 and vb.gdp_pc>0 else np.nan
    rec['urban_pct_annualized']=(vb.urban_pct-va.urban_pct)/dur
    records.append(rec)
panel=pd.DataFrame(records)
assert len(panel)==140 and not panel.duplicated(['ISO','interval']).any()
assert panel['wgi_annualized'].notna().all()
panel.to_csv(PROC/'country_period_panel.csv',index=False)

# Country FE and period FE. Covariance clusters by ISO. All model-based inference is conditional on source point estimates.
base_formula='change_equal_sex_pp_year ~ wgi_annualized + C(ISO) + C(interval)'
adj_formula='change_equal_sex_pp_year ~ wgi_annualized + log_gdp_pc_annualized + urban_pct_annualized + C(ISO) + C(interval)'

def estimate(label,d,adjusted=False):
    f=adj_formula if adjusted else base_formula
    m=smf.ols(f,data=d).fit(cov_type='cluster',cov_kwds={'groups':d.ISO,'use_correction':True})
    g=d.ISO.nunique(); term='wgi_annualized';coef=float(m.params[term]);se=float(m.bse[term]);df=g-1
    t=coef/se; p=2*stats.t.sf(abs(t),df);q=stats.t.ppf(.975,df)
    return m,{'model':label,'n_country_period':len(d),'n_countries':g,'n_periods':d.interval.nunique(),'coef_per_one_WGI_score_point_year':coef,'se_cluster_country':se,'t_cluster':t,'df_t_reference':df,'p_cluster_t':p,'ci95_low_conditional':coef-q*se,'ci95_high_conditional':coef+q*se,'r_squared':m.rsquared}

common=panel.dropna(subset=['log_gdp_pc_annualized','urban_pct_annualized']).copy()
primary,m1=estimate('Primary unadjusted: all available',panel)
common_primary,mc=estimate('Unadjusted: adjusted-model common sample',common)
adjusted,m2=estimate('Adjusted: GDP and urbanization',common,True)
model_rows=[m1,mc,m2];extra=[]
for label,frame,adj_flag in [
 ('Excluding 2019–2022',panel[panel.interval!='2019-2022'],False),
 ('Excluding six Gulf economies',panel[~panel.ISO.isin(['BHR','KWT','OMN','QAT','SAU','ARE'])],False),
 ('South Asia exploratory',panel[panel.region=='South Asia'],False),
 ('MENA exploratory',panel[panel.region=='MENA'],False),
 ('Adjusted excluding 2019–2022',common[common.interval!='2019-2022'],True)]:
    try:_,res=estimate(label,frame,adj_flag)
    except (ValueError,IndexError,KeyError,np.linalg.LinAlgError) as e:print('MODEL FAIL',label,e,file=sys.stderr);continue
    model_rows.append(res)
for iso in CODES:
    _,r=estimate(f'Leave out {iso}',panel[panel.ISO!=iso])
    r['excluded_ISO']=iso;extra.append(r)

# Wild cluster bootstrap-t, restricted/null-imposed with Rademacher weights at country level.
# Implemented on fixed OLS design with CRV1 covariance and t_{clusters-1} reference diagnostics.
def wild_cluster_p(model, data, covariate='wgi_annualized', B=1999, seed=20261031):
    import patsy
    y,xmat=patsy.dmatrices(model.model.formula,data=data,return_type='dataframe')
    y=y.to_numpy().ravel();X=xmat.to_numpy();col=xmat.columns.get_loc(covariate)
    n,k=X.shape
    cats=np.unique(data.ISO.to_numpy());g=len(cats)
    groups=[np.flatnonzero(data.ISO.to_numpy()==c) for c in cats]
    A=np.linalg.pinv(X.T@X)@X.T
    def calc(y0):
        coef=A@y0;e=y0-X@coef
        cluster_s=0.0
        for ids in groups:
            z=A[:,ids]@e[ids]
            cluster_s+=(z[col]**2)
        v=cluster_s*(g/(g-1))*((n-1)/(n-k))
        return coef[col]/np.sqrt(v) if v>0 else np.nan
    observed=calc(y)
    X0=np.delete(X,col,axis=1)
    fitted=X0@np.linalg.lstsq(X0,y,rcond=None)[0]
    residual=y-fitted
    rng=np.random.default_rng(seed)
    exceed=0
    for _ in range(B):
        weights=rng.choice(np.array([-1.,1.]),size=g)
        ystar=fitted+residual*np.array([weights[np.searchsorted(cats,iso)] for iso in data.ISO])
        tstar=calc(ystar)
        if np.isfinite(tstar) and abs(tstar)>=abs(observed):exceed+=1
    return {'wild_cluster_bootstrap_t_observed':float(observed),'wild_cluster_bootstrap_p':(exceed+1)/(B+1),'bootstrap_repetitions':B,'bootstrap_seed':seed}

for model,data,row in [(primary,panel,m1),(adjusted,common,m2)]:
    row.update(wild_cluster_p(model,data))
models=pd.DataFrame(model_rows)
models.to_csv(TABLES/'table3_regression_and_sensitivity.csv',index=False)
loocv=pd.DataFrame(extra)
loocv.to_csv(SUPP/'Additional_file_3_leave_one_country_out.csv',index=False)
panel.to_csv(SUPP/'Additional_file_4_country_period_analysis_panel.csv',index=False)

# WGI uncertainty diagnostics: nonoverlap of marginal 90% intervals NOT a hypothesis test.
wgi_diag=panel.groupby('region').agg(n_country_periods=('ISO','size'),nonoverlap_n=('wgi_nonoverlap_90pct_bounds','sum')).reset_index()
wgi_diag['nonoverlap_pct']=100*wgi_diag.nonoverlap_n/wgi_diag.n_country_periods
wgi_diag.to_csv(TABLES/'wgi_uncertainty_diagnostics.csv',index=False)
missing=panel[panel.log_gdp_pc_annualized.isna()][['ISO','country','interval','log_gdp_pc_annualized']]
missing.to_csv(TABLES/'missing_gdp_country_periods.csv',index=False)

plt.rcParams.update({'font.family':'DejaVu Sans','font.size':10,'axes.spines.top':False,'axes.spines.right':False})
PAL={'Men':'#1E507E','Women':'#9A4465'}
# Four figures; separate transparent legends and no false uncertainty interval.
fig,axs=plt.subplots(1,2,figsize=(11.2,4.4),sharey=True)
for i,(region,ds) in enumerate(trend.groupby('region',sort=False)):
  ax=axs[i]
  for sexid in ['Men','Women']:
    h=ds[ds.Sex==sexid].sort_values('Year')
    ax.plot(h.Year,h['mean'],color=PAL[sexid],lw=2.15,label=sexid)
    ax.fill_between(h.Year.to_numpy(),h.q25.to_numpy(),h.q75.to_numpy(),color=PAL[sexid],alpha=.11)
  ax.set_title(f'{region} (n={ds.n_countries.max():.0f} countries)')
  ax.set_xlabel('Year');ax.grid(axis='y',alpha=.18);ax.set_xlim(2000,2022)
axs[0].set_ylabel('Age-standardized medication treatment coverage (%)');axs[1].legend(frameon=False,loc='upper left')
fig.tight_layout();fig.savefig(FIGS/'Figure_1_Regional_Trajectories.png',dpi=320,bbox_inches='tight');plt.close(fig)

cc=country_change.sort_values('equal_sex_change_pp')
fig,ax=plt.subplots(figsize=(9.4,8.0))
colors=['#9A4465' if r=='South Asia' else '#356E85' for r in cc.region]
ax.barh(cc.country,cc.equal_sex_change_pp,color=colors)
ax.axvline(0,color='#444444',lw=.8);ax.grid(axis='x',alpha=.18)
ax.set_xlabel('Change, 2000–2022 (percentage points; equal-sex mean)')
ax.legend(handles=[Line2D([0],[0],color='#9A4465',lw=6,label='South Asia'),Line2D([0],[0],color='#356E85',lw=6,label='MENA')],loc='lower right',frameon=False)
fig.tight_layout();fig.savefig(FIGS/'Figure_2_Country_Endpoint_Changes.png',dpi=320,bbox_inches='tight');plt.close(fig)

fig,ax=plt.subplots(figsize=(9.0,4.5))
for reg in ['South Asia','MENA']:
  for s in ['Men','Women']:
    z=trend[(trend.region==reg)&(trend.Sex==s)].sort_values('Year')
    ax.plot(z.Year,z.p10_p90_gap,lw=2,ls='-' if s=='Men' else '--',label=f'{reg}: {s}')
ax.set_xlabel('Year');ax.set_ylabel('Between-country P90–P10 gap (percentage points)');ax.grid(alpha=.18)
ax.legend(frameon=False,ncol=2,fontsize=9)
fig.tight_layout();fig.savefig(FIGS/'Figure_3_Country_Inequality.png',dpi=320,bbox_inches='tight');plt.close(fig)

loo=loocv.merge(country_change[['ISO','country']],left_on='excluded_ISO',right_on='ISO').sort_values('coef_per_one_WGI_score_point_year')
fig,ax=plt.subplots(figsize=(9.3,7.9))
ypos=np.arange(len(loo))
lo=loo.ci95_low_conditional.to_numpy();hi=loo.ci95_high_conditional.to_numpy();v=loo.coef_per_one_WGI_score_point_year.to_numpy()
ax.errorbar(v,ypos,xerr=np.vstack([v-lo,hi-v]),fmt='o',color='#356E85',ecolor='#7C9FAA',markersize=3.8,lw=1.1,capsize=0)
ax.set_yticks(ypos,loo.country);ax.axvline(0,color='#8A8A8A',lw=.9,ls='--')
ax.axvline(m1['coef_per_one_WGI_score_point_year'],color='#9A4465',lw=1.4,ls=':')
ax.set_xlabel('WGI coefficient (percentage points/year per 1 score point/year)')
ax.grid(axis='x',alpha=.17)
fig.tight_layout();fig.savefig(FIGS/'Figure_4_Leave_One_Country_Out.png',dpi=320,bbox_inches='tight');plt.close(fig)

sha={f.name:hashlib.sha256(f.read_bytes()).hexdigest() for f in sorted(RAW.glob('*'))}
(PROC/'raw_input_sha256.json').write_text(json.dumps(sha,indent=2))
summary={
'year_range':[2000,2022], 'n_countries':len(CODES),'n_south_asia':8,'n_mena':20,
'n_country_sex_year':len(x), 'n_country_period':len(panel),'n_adjusted_model':len(common),
'n_missing_gdp_periods':len(missing),
'wgi_nonoverlap_country_periods':int(panel.wgi_nonoverlap_90pct_bounds.sum()),
'wgi_total_country_periods':len(panel),
'country_negative_coverage_change_n':int((country_change.equal_sex_change_pp<0).sum()),
'south_asia_negative_n':int(((country_change.region=='South Asia')&(country_change.equal_sex_change_pp<0)).sum()),
'endpoint_regional':end.to_dict(orient='records'),
'endpoint_sex_gaps':sex_table.to_dict(orient='records'),
'pakistan':cend[cend.ISO=='PAK'].to_dict(orient='records'),
'model_primary':m1,'model_common_sample':mc,'model_adjusted':m2,
'loo_min':float(loo.coef_per_one_WGI_score_point_year.min()),'loo_max':float(loo.coef_per_one_WGI_score_point_year.max())}
(TABLES/'analysis_summary.json').write_text(json.dumps(summary,indent=2,default=lambda o:float(o) if isinstance(o,np.number) else str(o)))
print(json.dumps({k:v for k,v in summary.items() if k not in ['endpoint_regional','endpoint_sex_gaps','pakistan','model_primary','model_common_sample','model_adjusted']},indent=2))
print('PRIMARY',m1)
print('ADJUSTED',m2)
print('PAKISTAN',cend[cend.ISO=='PAK'].to_string(index=False))
print('REGIONS',end.to_string(index=False))
