"""r06_lint.py: language lint for the Phase 4 manuscript (the language constraints of the pipeline prompt).

Works on the LaTeX source rendered by r07_build.py in two forms:
  normal    what the reader sees (tokens substituted); checked for dashes, which are banned everywhere except
            in the CRediT role names (copied verbatim from the pipeline prompt) and in the page ranges that
            the journal's style file sets in the reference list;
  skeleton  every value token replaced by the sign § and every URL by ¤, so that the language rules judge
            only the words written here.
The LaTeX markup is stripped before the prose checks (detex below). Errors fail the Phase 4 check; warnings
are listed for a human pass.
"""
import json, re, sys

sys.path.insert(0, "/home/claude/mill/phase4/code")
ALLOW = "/home/claude/mill/phase4/inputs/lint_allow4.json"
MACROS = {"vmA": "imi_vm20i", "vmB": "imi_vmx30ui", "vmC": "tmf_vf10", "hurco": "Hurco VM20i", "axesthree": "3-axis",
          "axesfive": "5-axis", "sA": "Sound0", "sB": "Sound1", "sC": "Sound2", "aA": "Acc0", "aB": "Acc1",
          "alu": "AA 6061", "stA": "AISI 1018", "stB": "AISI 4140", "ccby": "CC BY 4.0", "shaname": "SHA-256"}
DASHES = {"\u2014": "em dash", "\u2013": "en dash", "\u2015": "horizontal bar", "\u2012": "figure dash", "\u2010": "hyphen character U+2010"}
FILLER = ["it is important to note", "it is worth noting", "it should be noted", "notably", "furthermore", "moreover",
          "additionally", "interestingly", "importantly", "crucially", "needless to say", "in conclusion", "in summary",
          "it goes without saying", "at the end of the day", "in order to", "overall,"]
GRANDIOSE = ["paradigm", "unprecedented", "sheds light", "shed light", "groundbreaking", "ground-breaking", "revolutioniz",
             "revolutionis", "game-chang", "game chang", "cutting-edge", "unlock", "unveil", "illuminat", "delve", "tapestry",
             "pivotal", "transformative", "landscape", "holistic", "leverag", "harness", "seamless", "robustly",
             "towards a new", "comprehensive", "unified", "remarkabl", "striking"]
NUMBER_WORDS = ["two", "three", "four", "five", "six", "seven", "eight", "nine", "ten", "eleven", "twelve", "thirteen",
                "fourteen", "fifteen", "sixteen", "seventeen", "eighteen", "nineteen", "twenty", "hundred", "thousand",
                "million", "dozen", "half", "twice", "thrice", "double", "triple"]
TITLE_BANNED = ["novel", "unprecedented", "comprehensive", "unified", "paradigm", "revolutioniz", "unlock", "illuminat", "towards a new"]


def body(tex):
    a = tex.index("\\begin{document}")
    return tex[a:]


def detex(tex):
    """Plain prose from the LaTeX body: markup removed, macros expanded, math and floats' commands dropped."""
    t = body(tex)
    t = re.sub(r"(?m)^%.*$", "", t)
    t = re.sub(r"(?<!\\)%.*", "", t)
    t = re.sub(r"\\begin\{equation\}.*?\\end\{equation\}", " ", t, flags=re.S)
    t = re.sub(r"\$[^$]*\$", " ", t)
    t = re.sub(r"\\estci\{([^{}]*)\}\{([^{}]*)\}", r"\1 (\2)", t)
    t = re.sub(r"\\estcim\{([^{}]*)\}\{([^{}]*)\}\{([^{}]*)\}", r"\1 (\2)", t)
    t = re.sub(r"\\(label|ref|cite|includegraphics|url|bibliography|typeout)(\[[^\]]*\])?\{[^}]*\}", " ", t)
    t = re.sub(r"\\(begin|end)\{[^}]*\}(\{[^}]*\})*(\[[^\]]*\])?", " ", t)
    t = re.sub(r"\\(footnotetext|footnotemark)(\[[^\]]*\])?", " ", t)
    t = re.sub(r"\\texorpdfstring\{(?:[^{}]|\{[^{}]*\})*\}\{([^{}]*)\}", r"\1", t)   # a heading: keep its plain text
    t = re.sub(r"\\textcolor\{[^}]*\}", " ", t)                                       # the colour argument is not prose
    t = re.sub(r"\\(author\*?|affil\*?)\[[^\]]*\]", " ", t)
    for k, v in MACROS.items():
        t = re.sub(r"\\" + k + r"(\{\})?(?![A-Za-z])", v, t)
    t = t.replace("\\textminus{}", "-").replace("\\&", "&").replace("~", " ").replace("\\\\", "\n")
    t = re.sub(r"\\[a-zA-Z]+\*?", " ", t)
    t = re.sub(r"[{}]", "", t)
    t = re.sub(r"[ \t]+", " ", t)
    return t


def load_allow():
    try:
        return json.load(open(ALLOW))
    except FileNotFoundError:
        return []


def title_of(tex):
    m = re.search(r"\\title\[[^\]]*\]\{([^}]*)\}", tex)
    return m.group(1) if m else ""


def lint(normal, skeleton):
    errors, warnings = [], []
    b = body(normal)
    credit = re.search(r"\\bmhead\{Author contributions\}(.*?)\\bmhead", b, flags=re.S)
    credit_txt = credit.group(1) if credit else ""
    b_no_credit = b.replace(credit_txt, "") if credit_txt else b
    # 1. dashes: em dashes nowhere; en dashes and LaTeX -- only in the CRediT roles
    for i, line in enumerate(b_no_credit.split("\n"), 1):
        for ch, name in DASHES.items():
            if ch in line:
                errors.append(("dash", i, f"{name}: {line.strip()[:90]}"))
        if "---" in line or re.search(r"(?<!-)--(?!-)", line):
            errors.append(("dash", i, f"LaTeX dash: {line.strip()[:90]}"))
    for i, line in enumerate(detex(normal).split("\n"), 1):
        if re.search(r"(?<=\w) - (?=\w)", line):
            errors.append(("dash", i, f"hyphen used as a dash: {line.strip()[:90]}"))
    if "---" in credit_txt or "\u2014" in credit_txt:
        errors.append(("dash", 0, "em dash in the CRediT block"))
    # 2. prose checks on the skeleton
    sk = detex(skeleton)
    low = sk.lower()
    line_of = lambda pos: low.count("\n", 0, pos) + 1
    for ph in FILLER:
        for m in re.finditer(r"\b" + re.escape(ph), low):
            errors.append(("filler", line_of(m.start()), ph))
    for ph in GRANDIOSE:
        for m in re.finditer(r"\b" + re.escape(ph), low):
            errors.append(("grandiose", line_of(m.start()), ph))
    for m in re.finditer(r"\bnovel\b", low):
        errors.append(("grandiose", line_of(m.start()), "novel"))
    for m in re.finditer(r"\bnot only\b|\bnot just\b|\bnot merely\b", low):
        errors.append(("not only", line_of(m.start()), m.group(0)))
    for m in re.finditer(r"\bnot\b[^.;:!?\n]{1,80}?,\s*but\b", low):
        errors.append(("not X, but Y", line_of(m.start()), m.group(0)[:80]))
    for m in re.finditer(r",\s+not\s+(?:a|an|the)?\s*\w+", low):
        warnings.append(("X, not Y", line_of(m.start()), sk[m.start():m.start() + 70].replace("\n", " ")))
    allow = load_allow()
    for i, line in enumerate(sk.split("\n"), 1):
        for m in re.finditer(r"\b(" + "|".join(NUMBER_WORDS) + r")\b", line, flags=re.I):
            ctx = line[max(0, m.start() - 40):m.end() + 40]
            if any(re.search(a["pattern"], ctx, flags=re.I) for a in allow):
                continue
            errors.append(("number word", i, ctx.strip()))
        for m in re.finditer(r"\bone\b", line, flags=re.I):
            ctx = line[max(0, m.start() - 40):m.end() + 40]
            if not any(re.search(a["pattern"], ctx, flags=re.I) for a in allow):
                warnings.append(("the word one", i, ctx.strip()))
    for m in re.finditer(r"\b(\w+(?: \w+){0,2}), (\w+(?: \w+){0,2}),? and (\w+(?: \w+){0,2})[.;]", sk):
        warnings.append(("possible tricolon", line_of(m.start()), m.group(0)[:80]))
    # 3. title rules (pipeline prompt)
    title = title_of(normal)
    tl = title.lower()
    for w in TITLE_BANNED:
        if w in tl:
            errors.append(("title claim word", 0, w))
    if "?" in title:
        errors.append(("title question", 0, title))
    if re.match(r"^\w+ing\b", title):
        errors.append(("title gerund opener", 0, title.split()[0]))
    if any(ch in title for ch in DASHES) or "--" in title:
        errors.append(("title dash", 0, title))
    # 4. sentence-length variation
    paras = [l for l in sk.split("\n") if len(l.split()) > 12]
    sents = [s for p in paras for s in re.split(r"(?<=[.?!])\s+(?=[A-Z§])", p) if len(s.split()) >= 3]
    lens = [len(s.split()) for s in sents]
    stats = {}
    if lens:
        mu = sum(lens) / len(lens)
        sd = (sum((x - mu) ** 2 for x in lens) / len(lens)) ** 0.5
        stats = {"sentences": len(lens), "mean_words": round(mu, 1), "sd_words": round(sd, 1), "cv": round(sd / mu, 2),
                 "shortest": min(lens), "longest": max(lens)}
        if sd / mu < 0.35:
            warnings.append(("uniform sentence length", 0, f"coefficient of variation {sd / mu:.2f}"))
    return errors, warnings, stats


COMPARATIVE_BELOW = r"\b(?:below|under|less than|lower than|short of)"
COMPARATIVE_ABOVE = r"\b(?:above|over|more than|greater than|higher than|beyond)"


def comparatives(template, values):
    """A comparison with the rounded extreme of the same quantity is false by construction ('below the maximum',
    'above the minimum'): the first draft printed 'below 6 percent' for a largest share of 6.2 percent (review
    item R11). Flags a comparative word followed by a token whose name or statistic marks a largest or a smallest
    value of the kind the comparison would contradict."""
    errors = []
    tok = r"\s+(?:the\s+|a\s+|an\s+)?(?:[a-z-]+\s+){0,3}?\{\{([a-z0-9_]+)\|"
    for kind, pat in (("max", COMPARATIVE_BELOW), ("min", COMPARATIVE_ABOVE)):
        for m in re.finditer(pat + tok, template, flags=re.I):
            name = m.group(1)
            rec = values.get(name, {})
            stat = str(rec.get("estimand", {}).get("statistic", "")).lower() if isinstance(rec.get("estimand"), dict) else ""
            extreme = (name.endswith(f"_{kind}") or f"_{kind}_" in name
                       or stat.startswith("largest" if kind == "max" else "smallest"))
            if extreme:
                errors.append(("comparison with an extreme", 0, template[max(0, m.start() - 30):m.end() + 30].replace("\n", " ")))
    return errors


def main():
    import r07_build as RB
    values, _ = RB.merged_values()
    normal, skeleton, missing, _ = RB.render_all(values)
    errors, warnings, stats = lint(normal, skeleton)
    errors += comparatives(RB.template()[0], values)
    for e in errors:
        print("ERROR", e)
    for w in warnings:
        print("warn ", w)
    print("sentence stats:", stats, "missing tokens:", sorted(set(missing)))
    print(f"lint: {len(errors)} errors, {len(warnings)} warnings")
    return 0 if not errors else 1


if __name__ == "__main__":
    sys.exit(main())