#!/usr/bin/env python3
"""Reproduce aggregate analyses for the Albanian diaspora exploratory study.

Input: an author-held CSV exported from the *Data* worksheet of the cleaned
Diaspora_survey_cleaned_n35_v3.xlsx workbook (not the raw bilingual export).

Privacy: DO NOT upload the respondent-level CSV or this script's raw input to
journal supplementary material without a documented data-sharing assessment.
The script writes only aggregate tables and test statistics.

Dependencies: Python >=3.10; numpy; scipy; statsmodels.
Usage: python S4_Reproduce_Analyses.py --data /private/path/Data.csv --out /private/aggregate.json
"""
from __future__ import annotations
import argparse, csv, json, math
from pathlib import Path
import numpy as np
from scipy.stats import spearmanr, mannwhitneyu, fisher_exact, rankdata, pearsonr, chi2_contingency
from statsmodels.stats.multitest import multipletests
from statsmodels.stats.proportion import proportion_confint
import statsmodels.api as sm

COMPOSITES = {
    'Exposure': ['C1','C2','C4','C5','C6'],
    'Ethnonationalism': ['D6','D8','D9','D10'],
    'Identity importance': ['D1','D2'],
    'Connecting effect': ['F1','F2','F4','F7'],
    'Divisive effect': ['F3','F5','F9'],
    'State trust': ['E7_government','E7_parliament','E7_judiciary'],
    'Homeland attachment': ['E8a','E8b','E8d'],
}
IDX = {
    'Exposure':'IDX_exposure',
    'Ethnonationalism':'IDX_ethnonat',
    'Identity importance':'IDX_identity',
    'Connecting effect':'IDX_sm_effect',
    'Divisive effect':'IDX_sm_negative',
    'State trust':'IDX_trust_state',
    'Homeland attachment':'IDX_attachment',
}
ATTITUDE_ITEMS = [f'C{i}' for i in range(1,8)] + [f'D{i}' for i in range(1,11)] + ['E8a','E8b','E8d'] + [f'F{i}' for i in range(1,10)]
CORRS = [
    ('Exposure vs identity strengthening', 'IDX_exposure','F1'),
    ('Exposure vs ethnonationalism', 'IDX_exposure','IDX_ethnonat'),
    ('Time online vs ethnonationalism', 'B1_code','IDX_ethnonat'),
    ('National posting vs ethnonationalism', 'B6_code','IDX_ethnonat'),
    ('Historical content vs ethnonationalism','B4_history_code','IDX_ethnonat'),
    ('Identity vs ethnonationalism','IDX_identity','IDX_ethnonat'),
    ('Ethnonationalism vs trust','IDX_ethnonat','IDX_trust_state'),
    ('Unification vs coexistence','D6','D7'),
    ('Unification vs state identification','D6','D3'),
    ('Grievance vs ethnonationalism','D4','IDX_ethnonat'),
    ('Connecting vs divisive effects','IDX_sm_effect','IDX_sm_negative'),
]

def num(x):
    try:
        if x is None or str(x).strip() in ('', 'None', 'nan'): return float('nan')
        return float(x)
    except (TypeError,ValueError): return float('nan')

def valid_pair(rows,a,b):
    x=np.array([num(r.get(a)) for r in rows],float)
    y=np.array([num(r.get(b)) for r in rows],float)
    mask=np.isfinite(x)&np.isfinite(y)
    return x[mask], y[mask]

def corr(rows,a,b):
    x,y=valid_pair(rows,a,b)
    if len(x)<4 or len(set(x))<2 or len(set(y))<2: return {'n':len(x),'rho':None,'p':None}
    r,p=spearmanr(x,y)
    return {'n':len(x),'rho':round(float(r),5),'p':round(float(p),8)}

def partial_rank(rows,a,b):
    # Exploratory control for agreement tendency, excluding the focal items.
    inv = {v:k for k,v in IDX.items()}
    excluded=set()
    for var in (a,b):
        if var in inv:
            excluded.update(COMPOSITES[inv[var]])
        else:
            excluded.add(var)
    control=[q for q in ATTITUDE_ITEMS if q not in excluded]
    arr=[]
    for r in rows:
        xv,yv=num(r.get(a)),num(r.get(b))
        z=np.array([num(r.get(k)) for k in control],float)
        z=z[np.isfinite(z)]
        if np.isfinite(xv) and np.isfinite(yv) and len(z)>5:arr.append((xv,yv,z.mean()))
    if len(arr)<5:return {'n':len(arr),'rho':None,'p':None}
    A=np.array(arr)
    xr=rankdata(A[:,0]);yr=rankdata(A[:,1]);zr=rankdata(A[:,2])
    Q=sm.add_constant(zr)
    xx=sm.OLS(xr,Q).fit().resid;yy=sm.OLS(yr,Q).fit().resid
    co,p=pearsonr(xx,yy)
    return {'n':len(arr),'rho':round(float(co),5),'p':round(float(p),7),'control_item_count':len(control)}

def alpha(rows,items):
    X=np.array([[num(r.get(k)) for k in items] for r in rows],float)
    X=X[np.isfinite(X).all(axis=1)]
    if len(X)<3:return {'n_complete':len(X),'alpha':None}
    variances=X.var(axis=0,ddof=1)
    total=X.sum(axis=1).var(ddof=1)
    val=len(items)/(len(items)-1)*(1-variances.sum()/total)
    return {'n_complete':len(X),'alpha':round(float(val),5)}

def wilson(x,n):
    lo,hi=proportion_confint(x,n,method='wilson',alpha=0.05)
    return [round(float(lo),4),round(float(hi),4)]

def ordered_n(rows,var):
    vals=[num(r.get(var)) for r in rows]
    return [int(sum(x==v for x in vals)) for v in [1,2,3,4,5]],int(sum(np.isfinite(vals)))

def generation(rows):
    first=[r for r in rows if r.get('A4_born')=='North Macedonia']
    second=[r for r in rows if r.get('A4_born')=='Country of residence']
    out={}
    for var in ['F3','IDX_sm_negative','IDX_sm_effect','IDX_exposure','IDX_ethnonat','IDX_identity','IDX_attachment','F4']:
        a=np.array([num(r.get(var)) for r in first],float); a=a[np.isfinite(a)]
        b=np.array([num(r.get(var)) for r in second],float); b=b[np.isfinite(b)]
        if len(a)>2 and len(b)>2:
            stat,p=mannwhitneyu(a,b,alternative='two-sided',method='asymptotic')
            rbc=2*float(stat)/(len(a)*len(b))-1 # positive = first>second
            out[var]={'n_first':len(a),'n_second':len(b),'mean_first':round(float(a.mean()),4),
                      'mean_second':round(float(b.mean()),4),'p_MWU':round(float(p),7),
                      'rank_biserial_first_minus_second':round(rbc,4)}
    a=[num(r.get('F3')) for r in first]; b=[num(r.get('F3')) for r in second]
    a=[x for x in a if np.isfinite(x)]; b=[x for x in b if np.isfinite(x)]
    yes_a=sum(x>=4 for x in a); yes_b=sum(x>=4 for x in b)
    _,p=fisher_exact([[yes_b,len(b)-yes_b],[yes_a,len(a)-yes_a]])
    out['F3_agreement']={'first':{'agree':yes_a,'n':len(a),'wilson_95':wilson(yes_a,len(a))},
                         'second':{'agree':yes_b,'n':len(b),'wilson_95':wilson(yes_b,len(b))},
                         'fisher_exact_two_sided_p':round(float(p),7)}
    return out

def regress(rows,y,predictors):
    cols=[y]+predictors
    sub=[]
    for r in rows:
        v=[]
        for k in cols:
            if k=='second_gen':
                birth=r.get('A4_born')
                v.append(float('nan') if birth not in ('North Macedonia','Country of residence') else (1.0 if birth=='Country of residence' else 0.0))
            elif k=='male':v.append(float('nan') if not r.get('A3_gender') else (1.0 if r.get('A3_gender')=='Male' else 0.0))
            elif k=='age_ordinal':
                v.append({'18–24':1,'25–34':2,'35–44':3,'45–54':4,'55–64':5}.get(r.get('A2_age'),float('nan')))
            elif k=='university':
                v.append(float('nan') if not r.get('A7_education') else (1.0 if r.get('A7_education') in ('University (Bachelor)','Master / PhD') else 0.0))
            elif k=='switzerland':
                v.append(float('nan') if not r.get('A1_country') else (1.0 if r.get('A1_country')=='Switzerland' else 0.0))
            else:v.append(num(r.get(k)))
        if np.isfinite(v).all():sub.append(v)
    A=np.asarray(sub,float)
    if len(A)<=len(predictors)+2:return {'error':'too few complete cases','n':len(A)}
    dep=A[:,0]; x=A[:,1:]
    dep=(dep-dep.mean())/dep.std(ddof=1)
    for i,k in enumerate(predictors):
        if k not in ('second_gen','male','university','switzerland'):
            sd=x[:,i].std(ddof=1)
            if sd>0:x[:,i]=(x[:,i]-x[:,i].mean())/sd
    res=sm.OLS(dep,sm.add_constant(x)).fit(cov_type='HC3')
    d={'n':len(A),'R2':round(float(res.rsquared),4),'adjusted_R2':round(float(res.rsquared_adj),4),'predictors':{}}
    for i,k in enumerate(['intercept']+predictors):
        d['predictors'][k]={'beta':round(float(res.params[i]),5),'se_HC3':round(float(res.bse[i]),5),'p_HC3':round(float(res.pvalues[i]),6),
                             'CI95': [round(float(v),4) for v in res.conf_int()[i]]}
    return d

def main():
    ap=argparse.ArgumentParser(); ap.add_argument('--data',required=True); ap.add_argument('--out',required=True)
    args=ap.parse_args()
    with open(args.data,encoding='utf-8-sig',newline='') as f: allrows=list(csv.DictReader(f))
    mainrows=[r for r in allrows if num(r.get('include'))==1]
    strict=[r for r in mainrows if num(r.get('q_near_straightline'))==0]
    result={'dataset':{'all_responses':len(allrows),'included':len(mainrows),'strict':len(strict),
                       'excluded_zero_variance':sum(num(r.get('q_zero_variance'))==1 for r in allrows),
                       'excluded_proxy_population':sum(num(r.get('q_possible_non_target'))==1 for r in allrows)},
            'reliability':{k:alpha(mainrows,v) for k,v in COMPOSITES.items()},
            'correlations':{},'generation':generation(mainrows),'models':{},'items':{}}
    pvals=[]
    for label,a,b in CORRS:
        m=corr(mainrows,a,b);s=corr(strict,a,b)
        result['correlations'][label]={'main':m,'strict':s,'partial_agreement':partial_rank(mainrows,a,b)}
        pvals.append(m['p'])
    _,qvals,_,_=multipletests(pvals,alpha=0.05,method='fdr_bh')
    for (label,_,_),q in zip(CORRS,qvals):result['correlations'][label]['q_BH_11_tests']=round(float(q),7)
    result['models']={
      'M1_identity_strengthening':regress(mainrows,'F1',['IDX_exposure','IDX_identity','second_gen']),
      'M2_ethnonationalism_media':regress(mainrows,'IDX_ethnonat',['IDX_exposure','IDX_identity','B1_code','second_gen']),
      'M3_ethnonationalism_grievance':regress(mainrows,'IDX_ethnonat',['D4','IDX_exposure','second_gen']),
      'M4_negative_views_generation':regress(mainrows,'F3',['IDX_exposure','second_gen','age_ordinal','male']),
      'M4_sensitivity_education_country':regress(mainrows,'F3',['IDX_exposure','second_gen','university','switzerland']),
    }
    for v in ['C1','C2','C3','C7','D6','D7','F1','F3','F9']:
        counts,n=ordered_n(mainrows,v)
        result['items'][v]={'n':n,'counts_1_through_5':counts,'agree_4_or_5':sum(counts[-2:]),'agree_pct':round(100*sum(counts[-2:])/n,1)}
    cf=[r.get('F10_conflict','') for r in mainrows]
    result['items']['F10']={'n':sum(bool(v) for v in cf),'no':cf.count('No'),'sometimes':cf.count('Yes, sometimes'),'often':cf.count('Yes, often'),
                            'any_conflict':cf.count('Yes, sometimes')+cf.count('Yes, often')}
    result['homeland_by_generation']={}
    for lab,rows in [('first', [r for r in mainrows if r.get('A4_born')=='North Macedonia']),('second',[r for r in mainrows if r.get('A4_born')=='Country of residence'])]:
        from collections import Counter
        result['homeland_by_generation'][lab]=dict(Counter(r.get('E6_homeland','') for r in rows))
    Path(args.out).write_text(json.dumps(result,ensure_ascii=False,indent=2))
    print('Wrote aggregate results:',args.out, 'n=',len(mainrows),'strict=',len(strict))
    print('F3',result['generation']['F3_agreement'])
    print('Correlations', {k: v['main'] for k,v in result['correlations'].items()})
    print('Models',result['models'])
if __name__=='__main__':main()
