"""Reproduce adjusted financial-distress checks. Usage: python Expanded_Cohort_Sensitivity.py RAW_CSV ROSTER_CSV OUTPUT_CSV"""
import sys
import numpy as np
import pandas as pd
from scipy.optimize import minimize
from scipy.stats import norm
raw, roster, output = sys.argv[1:]
d = pd.read_csv(raw)
r = pd.read_csv(roster)
assert d.instanceID.is_unique and r.instanceID.is_unique
assert set(r.instanceID).issubset(set(d.instanceID))
assert d.Q02.ge(15).all() and d.Q01.isin([1, 3, 4, 5]).all()
assert not d[['Q01','Q02','Q04','Q12','Q43','Q31/3']].isna().any().any()
results = []
for cohort, a in [('Parent roster', d[d.instanceID.isin(r.instanceID)]), ('All exported records', d)]:
    X = np.column_stack([np.ones(len(a)), a.Q12.ge(3), a.Q01.ne(1), a.Q02.lt(25), a.Q04.eq(2)]).astype(float)
    for outcome, y in [('Attendance concern', a.Q43.ge(3).astype(float).to_numpy()), ('Transport support', a['Q31/3'].astype(float).to_numpy())]:
        fit = minimize(lambda b: np.sum(np.exp(X @ b) - y * (X @ b)), np.zeros(5), jac=lambda b: X.T @ (np.exp(X @ b) - y), method='BFGS', options={'gtol': 1e-10})
        b = fit.x
        mu = np.exp(X @ b)
        assert np.max(np.abs(X.T @ (mu-y))) < 1e-5
        bread = np.linalg.inv(X.T @ (mu[:,None]*X))
        scores = X*(y-mu)[:,None]
        cov = bread @ (scores.T @ scores) @ bread
        se = np.sqrt(cov[1,1])
        results.append(dict(cohort=cohort, outcome=outcome, n=len(a), events=int(y.sum()), pr=float(np.exp(b[1])), lower=float(np.exp(b[1]-norm.ppf(.975)*se)), upper=float(np.exp(b[1]+norm.ppf(.975)*se)), p=float(2*norm.sf(abs(b[1]/se))), maximum_fitted=float(mu.max())))
pd.DataFrame(results).to_csv(output, index=False)
