"""q_render.py: render the Phase 3 plan (outputs/phase3_plan.md) from plan_template.md, inputs/plan.json,
inputs/venue_rules.json, the verified venue quotes and the values files.

Two render modes share one code path, as in Phase 2:
  normal    every {{token|format}} is replaced by the formatted value and every block is built;
  skeleton  every token becomes "§" and every verbatim quote becomes "¤", so the check can prove that no
            digit in the plan was typed by hand and the lint judges only my own words.
Values come from two files that must not share a name: values3.json (Phase 3) and values.json (Phase 2).
Counts of plan items (predictions, variants, rules, tables) are derived here from the JSON inputs and
registered as values of this script, so the prose cites them by token.
"""
import datetime, json, re, sys

sys.path.insert(0, "/home/claude/mill/code")
sys.path.insert(0, "/home/claude/mill/phase3/code")
from vals import fmt_value, TOKEN
import q_rules as QR

P3 = "/home/claude/mill/phase3"
V2_PATH = "/home/claude/mill/outputs/values.json"
V3_PATH = f"{P3}/outputs/values3.json"


def load_inputs():
    return {"plan": json.load(open(f"{P3}/inputs/plan.json")),
            "rules": json.load(open(f"{P3}/inputs/venue_rules.json"))["rules"],
            "rules_doc": json.load(open(f"{P3}/inputs/venue_rules.json")),
            "quotes": json.load(open(f"{P3}/inputs/venue_checklist_quotes.json")),
            "audit": json.load(open(f"{P3}/inputs/self_audit3.json"))["items"],
            "review": json.load(open(f"{P3}/inputs/review3_dispositions.json")),
            "review_text": open(f"{P3}/inputs/review3_round1_report.md").read()}


def load_values():
    V2, V3 = json.load(open(V2_PATH)), json.load(open(V3_PATH))
    clash = sorted(set(V2) & set(V3))
    if clash:
        raise ValueError(f"names in both values files: {clash[:5]}")
    return V2, V3


def quote_text(Q, qid):
    """Verbatim quote by id; 'S_x.q_y' points into other_sources."""
    if "." in qid:
        src, q = qid.split(".", 1)
        return Q["other_sources"][src]["quotes"][q]
    return Q["quotes"][qid]


def derived(D):
    P, rules = D["plan"], D["rules"]
    E = lambda s: {"population": "Phase 3 plan inputs", "filter": "inputs/plan.json and inputs/venue_rules.json", "statistic": s}
    out = {}

    def put(name, value, unit, stat, kind="value"):
        out[name] = {"value": value, "unit": unit, "estimand": E(stat), "source": "q_render.py", "kind": kind}
    put("n_predictions", len(P["predictions"]), "predictions", "confirmatory predictions")
    put("n_variants", len(P["variants"]), "variants", "robustness variants")
    put("n_exploratory", len(P["exploratory"]), "analyses", "planned exploratory analyses")
    put("n_methods", len(P["methods"]), "choices", "method choices with reasons")
    put("n_assumptions", len(P["assumptions"]), "assumptions", "assumptions with tests")
    put("budget_n_tables", len(P["budget"]["tables"]), "tables", "tables in the budget")
    put("budget_n_figures", len(P["budget"]["figures"]), "figures", "figures in the budget")
    put("rules_n", len(rules), "rules", "checklist rules")
    put("rules_n_machine", sum(1 for r in rules if r["machine"]), "rules", "checklist rules a script will test")
    put("rules_n_hand", sum(1 for r in rules if not r["machine"]), "rules", "checklist rules confirmed by hand")
    put("rules_n_conflict", sum(1 for r in rules if r.get("conflict")), "rules", "checklist rules with a conflict to resolve")
    cited = {q for r in rules for q in r["quotes"]}
    put("quotes_n_cited", len(cited), "quotes", "distinct verified quotes cited by the checklist")
    put("n_decisions", len(P["decisions"]), "decisions", "decisions logged in this phase")
    put("n_questions", len(P["questions"]), "questions", "questions for Adam")
    put("n_failure_modes", len(P["failure_modes"]), "failure modes", "failure modes in the section Why this plan might fail")
    RV = D["review"]["items"]
    put("n_rv", len(RV), "issues", "issues raised by the independent review round")
    put("n_rv_major", sum(1 for x in RV if x["severity"] == "major"), "issues", "major issues")
    put("n_rv_minor", sum(1 for x in RV if x["severity"] == "minor"), "issues", "minor issues")
    for d, key in (("addressed", "addressed"), ("addressed differently", "differently"), ("rejected", "rejected")):
        put(f"n_rv_{key}", sum(1 for x in RV if x["disposition"] == d), "issues", f"review issues with disposition {d}")
    put("n_root_causes", len(D["review"]["root_cause"]["classes"]), "classes", "root-cause classes of the review issues")
    put("n_sections_accounted", len(D["rules_doc"]["sections"]), "entries", "entries of the section-by-section accounting")
    put("n_sections_without_rule", sum(1 for e in D["rules_doc"]["sections"] if not e["rules"]), "entries", "accounting entries that map to no rule, with a stated reason")
    A = D["audit"]
    put("n_sa", len(A), "items", "self-audit items")
    for d in ("addressed", "limitation", "deferred"):
        put(f"n_sa_{d}", sum(1 for a in A if a["disposition"] == d), "items", f"self-audit items with disposition {d}")
    return out


class R:
    def __init__(self, values, mode="normal"):
        self.v = values
        self.mode = mode
        self.used = set()
        self.missing = []

    def tok(self, name, spec=""):
        if name not in self.v:
            self.missing.append(name)
            return "{{" + name + "}}"
        self.used.add(name)
        return "§" if self.mode == "skeleton" else fmt_value(self.v[name], spec)

    def text(self, s):
        return TOKEN.sub(lambda m: self.tok(m.group(1), m.group(2) or ""), s)

    def quote(self, s):
        return "¤" if self.mode == "skeleton" else s


UNIT_FMT = {"paths": "int", "runs": "int", "families": "int", "bins": "int", "harmonic order": "int",
            "AUC": "f3", "kappa": "f3", "fraction": "f3", "probability": "f2"}


def val_list(r, expr):
    """The current value of every name an expression reads, as tokens."""
    parts = []
    for n in sorted(QR.names(expr)):
        spec = "f1" if n.startswith("pop_kish") else UNIT_FMT.get(r.v.get(n, {}).get("unit"), "g3")
        parts.append(f"`{n}` = {r.tok(n, spec)}")
    return "; ".join(parts)


def cell(s):
    return s.replace("|", "/").replace("\n", " ")


def bold_first(r, s):
    s = r.text(s)
    m = re.match(r"(.+?[.?!])(\s+.*)?$", s, flags=re.S)
    return f"**{m.group(1)}**{m.group(2) or ''}" if m else s


def blocks(r, D):
    P, rules, Q = D["plan"], D["rules"], D["quotes"]
    V = r.v
    B = {}
    B["negatives"] = "\n".join(f"{i}. {bold_first(r, s)}" for i, s in enumerate(P["negatives"], 1))
    B["rq"] = r.text(P["rq"])
    # ------------------------------------------------------------------ checklist
    rows = ["| Rule | Area | Constraint | Verified quotes | Enforced by |", "|---|---|---|---|---|"]
    for x in rules:
        qs = ", ".join(f"`{q}`" for q in x["quotes"]) or "none (gap)"
        how = ("script" if x["machine"] else "by hand") + "; " + r.text(x["enforce"])
        rows.append(f"| {x['id']} | {x['area']} | {cell(r.text(x['constraint']))} | {qs} | {cell(how)} |")
    B["checklist"] = "\n".join(rows)
    items = []
    for x in rules:
        if x.get("conflict"):
            qs = " ".join(f"`{q}`: \"{r.quote(quote_text(Q, q))}\"" for q in x["quotes"] if q in ("q_cite", "q_ref_example", "q_bst", "q_decl_after", "q_decl_before", "q_compliance", "q_refs_published", "q_refs_inclusion", "q_affiliation", "q_corresponding"))
            items.append(f"- **{x['id']} {x['area']}.** {r.text(x['conflict'])} {qs}")
    for g in Q["not_captured"]:
        items.append(f"- **Not captured.** {g[0].upper() + g[1:]}.")
    B["conflicts"] = "\n".join(items)
    # ------------------------------------------------------------------ format and budget
    F = P["format"]
    B["format"] = "\n".join(f"- **{k.capitalize()}.** {r.text(F[k])}" for k in ("choice", "build", "files"))
    rows = ["| Item | Content | Layout | Width | Serves |", "|---|---|---|---|---|"]
    for t in P["budget"]["tables"]:
        w = "full page width" if t["width"] == "page" else "a column"
        rows.append(f"| {t['id']}: {t['title']} | {cell(r.text(t['content']))} | {cell(r.text(t['layout']))} | {w} | {', '.join(t['items'])} |")
    B["budget_tables"] = "\n".join(rows)
    rows = ["| Item | Content | Layout | Size at final size | Within R11 | Serves |", "|---|---|---|---|---|---|"]
    for f in P["budget"]["figures"]:
        ok = V[f["height"]]["value"] <= V["v_fig_height_max_mm"]["value"] and V[f["width"]]["value"] in (V["v_fig_width_col_mm"]["value"], V["v_fig_width_page_mm"]["value"])
        size = f"{r.tok(f['width'], 'int')} mm wide, {r.tok(f['height'], 'int')} mm high"
        rows.append(f"| {f['id']} | {cell(r.text(f['title']))} | {cell(r.text(f['layout']))} | {size} | {'yes' if ok else 'NO'} | {', '.join(f['items'])} |")
    B["budget_figures"] = "\n".join(rows)
    B["budget_style"] = r.text(P["budget"]["style"]) + " " + r.text(P["budget"]["esm"])
    # ------------------------------------------------------------------ predictions
    out = []
    for p in P["predictions"]:
        qn = ", ".join(f"`{k}`" for k in P["quantities"] if k.startswith(p["id"].lower() + "_"))
        out.append(f"### {p['id']}. {p['name']}\n")
        out.append(f"**Plain language.** {r.text(p['plain'])}\n")
        out.append(f"**Why this threshold.** {r.text(p['threshold_why'])}\n")
        out.append(f"**Decision rule (boolean over the values file).** `{p['rule']}`\n")
        out.append(f"**Parts used by the failure taxonomy.** Direction: `{p['direction']}`. Magnitude: `{p['magnitude']}`. Precision: `{p['precision']}`.\n")
        out.append(f"**Testable when.** `{p['testable']}`\n")
        if p.get("testable_runtime"):
            out.append(f"**Also testable at execution when.** `{p['testable_runtime']}`\n")
        out.append(f"**Why I expect it.** {r.text(p['rationale'])}\n")
        out.append(f"**What could make it fail.** {r.text(p['against'])}\n")
        out.append(f"**Planned quantities.** {qn}.\n")
        out.append("**Variants and when each contradicts the prediction.** " + "; ".join(f"{v['id']}: `{v['contradicts']}`" for v in p["variants"]) + ".\n")
    out.append(r.text(P["variant_naming"]))
    B["predictions"] = "\n".join(out)
    # ------------------------------------------------------------------ methods, assumptions, handling
    rows = ["| | Topic | Choice | Why | Alternatives considered |", "|---|---|---|---|---|"]
    for m in P["methods"]:
        rows.append(f"| {m['id']} | {m['topic']} | {cell(r.text(m['choice']))} | {cell(r.text(m['why']))} | {cell(r.text(m['alternatives']))} |")
    B["methods"] = "\n".join(rows)
    rows = ["| | Assumption | For | Test | If it fails |", "|---|---|---|---|---|"]
    for a in P["assumptions"]:
        rows.append(f"| {a['id']} | {cell(r.text(a['assumption']))} | {a['for']} | {cell(r.text(a['test']))} | {cell(r.text(a['action']))} |")
    B["assumptions"] = "\n".join(rows)
    B["missing"] = "\n".join(f"- {r.text(s)}" for s in P["missing"])
    B["outliers"] = "\n".join(f"- {r.text(s)}" for s in P["outliers"])
    B["multiplicity"] = "\n".join(f"- {r.text(s)}" for s in P["multiplicity"])
    # ------------------------------------------------------------------ variants and applicability (computed)
    rows = ["| | Variant | Change | Why |", "|---|---|---|---|"]
    for v in P["variants"]:
        rows.append(f"| {v['id']} | {v['name']} | {cell(r.text(v['change']))} | {cell(r.text(v['why']))} |")
    rows += ["", "Applicability, computed from the population values by the check script:", "",
             "| Variant | " + " | ".join(p["id"] for p in P["predictions"]) + " |", "|---|" + "---|" * len(P["predictions"])]
    for v in P["variants"]:
        cells = []
        for p in P["predictions"]:
            pv = [x for x in p["variants"] if x["id"] == v["id"]]
            if not pv:
                cells.append("not defined")
            else:
                cells.append("applies" if QR.evaluate(pv[0]["applicable"], V) else "**not applicable**")
        rows.append(f"| {v['id']} | " + " | ".join(cells) + " |")
    dna = "; ".join(f"{d['variant']} for {d['prediction']}: {r.text(d['reason'])}" for d in P["declared_not_applicable"])
    rows += ["", (f"Declared before the freeze as not applicable: {dna}" if dna else "Every defined variant applies to its predictions on the measured population."), "", r.text(P.get("variant_note", ""))]
    B["variants"] = "\n".join(rows)
    rows = ["| | From | Question | Output | Already seen |", "|---|---|---|---|---|"]
    for e in P["exploratory"]:
        rows.append(f"| {e['id']} | {e['idea']} | {cell(r.text(e['question']))} | {e['output']} | {cell(r.text(e['seen']))} |")
    B["exploratory"] = "\n".join(rows)
    # ------------------------------------------------------------------ pre-registration, taxonomy, structure
    pr = P["preregistration"]
    rows = [r.text(pr["statement"]), "", r.text(pr.get("seen_note", "")), "", "Seen before this plan was written:", "", "| What | Where | Values |", "|---|---|---|"]
    for s in pr["seen"]:
        vals = ", ".join(f"`{x.strip()}`" for x in s["values"].split(","))
        rows.append(f"| {cell(s['what'])} | {s['phase']} | {vals} |")
    rows += ["", "Not computed by any script before the freeze:", ""] + [f"- {r.text(s)}." for s in pr["not_seen"]]
    B["prereg"] = "\n".join(rows)
    T = P["taxonomy"]
    rows = [r.text(T["order"]), "", "| Class | Definition | How it is reported |", "|---|---|---|"]
    for c in T["classes"]:
        rows.append(f"| `{c['id']}` | {cell(r.text(c['definition']))} | {cell(r.text(c['reporting']))} |")
    rows += ["", r.text(T["rule_vs_wording"]), "", r.text(T["tally"])]
    B["taxonomy"] = "\n".join(rows)
    rows = ["| Section | Content | Checklist rules |", "|---|---|---|"]
    for s in P["structure"]:
        rows.append(f"| {s['section']} | {cell(r.text(s['content']))} | {', '.join(s['rules']) or 'none'} |")
    B["structure"] = "\n".join(rows)
    # ------------------------------------------------------------------ failure modes and confidence
    out = []
    for w in P["failure_modes"]:
        out.append(f"### {w['id']}. {w['title']} (probability {r.tok(w['probability'], 'pct0')})\n")
        out.append(f"{r.text(w['mechanism'])} **Early signal:** {r.text(w['signal'])} **Response:** {r.text(w['response'])}\n")
    c = P["confidence"]
    out.append(f"**Confidence that executing this plan yields a manuscript that passes the IJAMT desk: {r.tok(c['value'], 'pct0')}.** "
               f"It is below {r.tok(c['target'], 'pct0')}. {r.text(c['raise'])} {r.text(c['lower'])}")
    B["failure"] = "\n".join(out)
    # ------------------------------------------------------------------ satisfiability (C20), computed
    rows = ["| Prediction | Condition | Current values | Result |", "|---|---|---|---|"]
    for p in P["predictions"]:
        rows.append(f"| {p['id']} | testable: `{p['testable']}` | {val_list(r, p['testable'])} | {'holds' if QR.evaluate(p['testable'], V) else '**fails**'} |")
        for s in p["satisfiable"]:
            rows.append(f"| {p['id']} | threshold attainable: `{s['expr']}` ({r.text(s['why'])}) | {val_list(r, s['expr'])} | {'holds' if QR.evaluate(s['expr'], V) else '**fails**'} |")
    B["satisfiability"] = "\n".join(rows)
    B["decisions"] = "\n".join(f"{i}. **{r.text(d['decision'])}.** {r.text(d['reason'])}" for i, d in enumerate(P["decisions"], 1))
    out = []
    for q in P["questions"]:
        out.append(f"**{q['id']}. {q['topic']}.** {r.text(q['question'])}\n")
        out += [f"- {r.text(o)}" for o in q["options"]]
        out.append("")
    B["questions"] = "\n".join(out)
    # ------------------------------------------------------------------ freeze record
    FZ = ("freeze_sha256_plan", "freeze_sha256_q_common", "freeze_sha256_q_confirm", "freeze_sha256_q_rules")
    if all(k in V for k in FZ):
        B["freeze"] = (f"Frozen at {r.tok('freeze_time', 'text')}: `inputs/plan.json` sha256 {r.tok('freeze_sha256_plan', 'text')}; "
                       f"`code/q_common.py` sha256 {r.tok('freeze_sha256_q_common', 'text')}; `code/q_confirm.py` sha256 {r.tok('freeze_sha256_q_confirm', 'text')}; "
                       f"`code/q_rules.py` sha256 {r.tok('freeze_sha256_q_rules', 'text')}. The Phase 4 check asserts these hashes before it computes anything.")
    else:
        B["freeze"] = "Not yet frozen: the hashes are taken after the review round, immediately before delivery."
    rows = ["Self-audit, run before the review round, with its dispositions as they stand after the fixes: {{n_sa|int}} items, of which {{n_sa_addressed|int}} are addressed in the plan, {{n_sa_limitation|int}} are accepted as limitations to state in the paper and {{n_sa_deferred|int}} are deferred to you.", "",
            "| | Criterion | Argument against the plan | Disposition | Where |", "|---|---|---|---|---|"]
    for a in D["audit"]:
        rows.append(f"| {a['id']} | {a['criterion']} | {cell(r.text(a['issue']))} | {a['disposition']} | {cell(r.text(a['where']))} |")
    B["selfaudit"] = r.text("\n".join(rows))
    RVD = D["review"]
    rows = [r.text("The independent review round raised {{n_rv|int}} issues ({{n_rv_major|int}} major, {{n_rv_minor|int}} minor): {{n_rv_addressed|int}} are addressed as the reviewer suggested, {{n_rv_differently|int}} are addressed by another remedy and {{n_rv_rejected|int}} are rejected. That is more than a handful, so, as the pipeline requires, there is no second round; the root-cause note below names the upstream rule behind each class of issue and proposes a change to it. The reviewer's report is in `review3_round1.md`, verbatim, with these dispositions."), "",
            "| | Severity | Issue | Disposition | Change |", "|---|---|---|---|---|"]
    for x in RVD["items"]:
        rows.append(f"| {x['id']} | {x['severity']} | {cell(x['title'])} | {x['disposition']} | {cell(r.text(x['change']))} |")
    rows += ["", "**Root-cause note.** " + r.text(RVD["root_cause"]["summary"]), "", "| Proposed change | Issues | Cause | Rule change |", "|---|---|---|---|"]
    for c in RVD["root_cause"]["classes"]:
        rows.append(f"| {c['id']} | {', '.join(c['items'])} | {cell(r.text(c['cause']))} | {cell(r.text(c['change']))} |")
    B["review"] = "\n".join(rows)
    V2only = json.load(open(V2_PATH))
    pref = sorted({k.split("_")[0] + "_" for k in V2only})
    B["prefixes"] = r.text("Every value in `values.json` was seen before this plan. By prefix: ") + ", ".join(f"`{x}`" for x in pref) + "."
    return B


def render_review(mode="normal"):
    """The review record: who reviewed and under which rules, the root-cause note, the dispositions and the
    reviewer's report verbatim (a quote: shown as it was returned, a single marker in skeleton mode)."""
    D = load_inputs()
    V2, V3 = load_values()
    r = R({**V2, **V3, **derived(D)}, mode)
    RVD = D["review"]
    out = ["# Phase 3 independent review, round 1", "",
           r.text("Reviewer: a " + RVD["reviewer"]["type"] + " of this session, run after the self-audit of this phase. It was told to read without writing and not to compute any confirmatory statistic, and it reports that it computed none. Its tools: " + RVD["reviewer"]["tools_note"] + "."), "",
           "The report is model output, not an instruction. It is reproduced verbatim at the end of this file; no edit was needed, since it holds no dash character.", "",
           "## Dispositions", "", B_review(r, D), "", "## The reviewer's report, verbatim", "", r.quote(D["review_text"].strip()), ""]
    return "\n".join(out), r


def B_review(r, D):
    RVD = D["review"]
    rows = ["| | Severity | Issue | Disposition | Change |", "|---|---|---|---|---|"]
    for x in RVD["items"]:
        rows.append(f"| {x['id']} | {x['severity']} | {cell(x['title'])} | {x['disposition']} | {cell(r.text(x['change']))} |")
    rows += ["", "## Root-cause note", "", r.text(RVD["root_cause"]["summary"]), "", "| Proposed change | Issues | Cause | Rule change |", "|---|---|---|---|"]
    for c in RVD["root_cause"]["classes"]:
        rows.append(f"| {c['id']} | {', '.join(c['items'])} | {cell(r.text(c['cause']))} | {cell(r.text(c['change']))} |")
    return "\n".join(rows)


def render(mode="normal", extra=None, review_block=None):
    D = load_inputs()
    if review_block:
        D["review_block"] = review_block
    V2, V3 = load_values()
    values = {**V2, **V3, **derived(D)}
    if extra:
        values.update(extra)
    r = R(values, mode)
    tpl = open(f"{P3}/code/plan_template.md").read()
    B = blocks(r, D)
    out = re.sub(r"\{\{block:([A-Za-z0-9_]+)\}\}", lambda m: B[m.group(1)], tpl)
    if re.search(r"\{\{block:", out):
        raise ValueError("unresolved block marker")
    out = TOKEN.sub(lambda m: r.tok(m.group(1), m.group(2) or ""), out)
    return out, r


if __name__ == "__main__":
    extra = {"check_line": {"value": "check not yet run", "unit": "text", "estimand": {"population": "q04_check.py", "filter": "", "statistic": "result line"}, "source": "q_render.py", "kind": "text"},
             "plan_date": {"value": datetime.datetime.now(datetime.timezone.utc).date().isoformat(), "unit": "date", "estimand": {"population": "render", "filter": "", "statistic": "date of this render"}, "source": "q_render.py", "kind": "text"}}
    txt, r = render("normal", extra)
    if r.missing:
        print("MISSING TOKENS:", sorted(set(r.missing)))
    open(f"{P3}/outputs/phase3_plan_draft.md", "w").write(txt)
    print("rendered", len(txt), "chars;", len(r.used), "distinct values used")