"""
CLASSIFIER CORPUS-SAMPLE VALIDATION (Section 4.3, Section 7).

IMPORTANT, HONEST SCOPE DISCLOSURE: this script draws a stratified random
sample of 152 real final assistant messages directly from the actual
tau-bench corpus (38 per domain/model cell) and compares the rule-based
classifier's predictions against a set of independent annotations. These
annotations were produced by the authors' AI writing assistant (Claude,
via Microsoft Copilot) reading each real message and applying the same
three-way scoring rubric as the classifier (1.0 = self-reported success,
0.0 = self-reported failure, 0.5 = ambiguous/no clear signal), NOT by an
independent human rater or a panel of multiple human raters computing
traditional inter-rater reliability.

This is a genuine methodological improvement over the 25 hand-constructed
edge cases in classifier_robustness_validation.py, since it validates
classifier behavior on a random sample of the REAL corpus rather than
author-constructed hypothetical cases. However, it is explicitly NOT
equivalent to a fully independent, blind, multi-rater human annotation
study, and should not be described as "human-annotated" validation. This
distinction is maintained transparently in the manuscript.

The 152 message texts and their AI-annotator labels are provided in
classifier_corpus_validation_sample.csv for full inspection and audit by
any reader who wishes to check individual judgments.

USAGE: the annotation labels themselves are fixed data (they cannot be
mechanically regenerated, since they required reading and judging each
message). This script demonstrates how the reported agreement statistics
(exact agreement, Cohen's kappa, confusion matrix, per-cell breakdown)
were computed from classifier_corpus_validation_sample.csv, and can be
re-run to verify those statistics are correctly derived from the provided
annotations.
"""
import csv
import json
from sklearn.metrics import cohen_kappa_score, confusion_matrix


def load_sample(path="classifier_corpus_validation_sample.csv"):
    rows = []
    with open(path, newline="") as f:
        reader = csv.DictReader(f)
        for row in reader:
            row["classifier_pred"] = float(row["classifier_pred"])
            row["ai_annotator_label"] = float(row["ai_annotator_label"])
            rows.append(row)
    return rows


def main():
    rows = load_sample()
    n = len(rows)
    clf = [str(r["classifier_pred"]) for r in rows]
    ann = [str(r["ai_annotator_label"]) for r in rows]

    n_agree = sum(1 for c, a in zip(clf, ann) if c == a)
    kappa = cohen_kappa_score(clf, ann)

    print(f"N = {n}")
    print(f"Exact agreement: {n_agree}/{n} = {n_agree/n:.1%}")
    print(f"Cohen's kappa: {kappa:.3f}")

    labels_order = ["0.0", "0.5", "1.0"]
    cm = confusion_matrix(ann, clf, labels=labels_order)
    print("\nConfusion matrix (rows=AI-annotator label, cols=classifier prediction):")
    print("           clf=0.0  clf=0.5  clf=1.0")
    for lab, row in zip(labels_order, cm):
        print(f"ann={lab}     {row[0]:5d}    {row[1]:5d}    {row[2]:5d}")

    print("\n=== By domain/model ===")
    by_cell = {}
    for domain in ["retail", "airline"]:
        for model in ["gpt-4o", "sonnet"]:
            sub = [r for r in rows if r["domain"] == domain and r["model"] == model]
            agree = sum(1 for r in sub if r["classifier_pred"] == r["ai_annotator_label"])
            by_cell[f"{domain}/{model}"] = {"n": len(sub), "agreement": round(agree / len(sub), 4)}
            print(f"{domain}/{model}: n={len(sub)}, agreement={agree}/{len(sub)}={agree/len(sub):.1%}")

    summary = {
        "n_sampled": n,
        "exact_agreement": round(n_agree / n, 4),
        "cohens_kappa": round(float(kappa), 4),
        "confusion_matrix": {"labels_order": labels_order, "matrix": cm.tolist()},
        "by_domain_model": by_cell,
    }
    with open("classifier_corpus_validation_summary_reproduced.json", "w") as f:
        json.dump(summary, f, indent=2)
    print("\nSaved: classifier_corpus_validation_summary_reproduced.json")


if __name__ == "__main__":
    main()