"""
Verify every reference in the manuscript against the issuing registry.

For each DOI: query api.crossref.org/works/{doi} and, if that 404s, api.datacite.org.
For each arXiv id: query export.arxiv.org/api/query.
Then compare the registry's title and first-author surname and year against what the
manuscript claims, and report any mismatch. Exit code is non-zero if anything fails to
resolve, so this can gate submission.

Titles are compared on a normalised form (lowercase, alphanumeric only) using a token
overlap ratio, because registry titles differ in punctuation, casing and subtitle style.
"""

import os
import json
import re
import sys
import time
import urllib.parse
import urllib.request
import xml.etree.ElementTree as ET

UA = "reference-verifier/1.0"

# (citation key, expected first-author surname, expected year, identifier, expected title)
REFS = [
    ("Bar-Ilan & Halevi 2017", "Bar-Ilan", 2017, "10.1007/s11192-017-2242-0",
     "Post retraction citations in context: a case study"),
    ("Cabanac & Labbe 2021", "Cabanac", 2021, "10.1002/asi.24495",
     "Prevalence of nonsensical algorithmically generated papers in the scientific literature"),
    ("Cabanac et al 2021", "Cabanac", 2021, "arXiv:2107.06751",
     "Tortured phrases: A dubious writing style emerging in science"),
    ("Candal-Pedreira 2024", "Candal-Pedreira", 2024, "10.1016/j.jclinepi.2024.111397",
     "Retracted papers originating from paper mills: a cross-sectional analysis of references and citations"),
    ("Candal-Pedreira 2022", "Candal-Pedreira", 2022, "10.1136/bmj-2022-071517",
     "Retracted papers originating from paper mills: cross sectional study"),
    ("Garcia-Romero 2026", "García-Romero", 2026, "10.1007/s11192-026-05699-7",
     "The enduring influence of retracted research"),
    ("Gray 2024", "Gray", 2024, "arXiv:2403.16887",
     "ChatGPT contamination: estimating the prevalence of LLMs in the scholarly literature"),
    ("Guo 2026", "Guo", 2026, "10.3390/publications14020027",
     "The attention mismatch: mapping the structural academic governance deficit in the age of generative AI"),
    ("Hsiao & Schneider 2021", "Hsiao", 2021, "10.1162/qss_a_00155",
     "Continued use of retracted papers"),
    ("Khademizadeh 2024", "Khademizadeh", 2024, "10.1080/08989621.2023.2223996",
     "Evolution of retracted publications in the medical sciences"),
    ("Kocyigit 2023", "Kocyigit", 2023, "10.3346/jkms.2023.38.e390",
     "Characteristics of Retracted Publications From Kazakhstan"),
    ("Kwee & Kwee 2023", "Kwee", 2023, "10.1016/j.acra.2022.06.025",
     "Retracted Publications in Medical Imaging Literature"),
    ("Labbe & Labbe 2013", "Labbé", 2013, "10.1007/s11192-012-0781-y",
     "Duplicate and fake publications in the scientific literature: how many SCIgen papers in computer science?"),
    ("Liang 2024", "Liang", 2024, "arXiv:2403.07183",
     "Monitoring AI-Modified Content at Scale"),
    ("Priem 2022", "Priem", 2022, "arXiv:2205.01833",
     "OpenAlex: A fully-open index of scholarly works, authors, venues, institutions, and concepts"),
    ("Samala 2025", "Samala", 2025, "10.1007/s11135-025-02453-8",
     "Retraction trends in Indonesian research"),
    ("Santos-d'Amorim 2025", "Santos-d'Amorim", 2025, "10.1007/s11192-025-05343-w",
     "Errors, questionable practices, or misconduct?"),
    ("Saqr 2026", "Saqr", 2026, "10.20944/preprints202601.0314.v1",
     "Dissecting AI-related paper retraction across countries and institutions"),
    ("Sridharan 2026", "Sridharan", 2026, "10.3389/frma.2025.1737168",
     "Artificial intelligence in the retraction spotlight"),
    ("Steen 2013", "Steen", 2013, "10.1371/journal.pone.0068397",
     "Why Has the Number of Scientific Retractions Increased?"),
    ("Van Noorden 2023", "Van Noorden", 2023, "10.1038/d41586-023-03974-8",
     "More than 10,000 research papers were retracted in 2023"),
]


def norm(s):
    return set(re.sub(r"[^a-z0-9 ]", " ", (s or "").lower()).split())


def overlap(a, b):
    ta, tb = norm(a), norm(b)
    if not ta or not tb:
        return 0.0
    stop = {"a", "an", "the", "of", "in", "on", "and", "for", "to", "how", "many"}
    ta, tb = ta - stop, tb - stop
    if not ta:
        return 0.0
    return len(ta & tb) / len(ta)


def get(url, timeout=20, attempts=2):
    last = None
    for i in range(attempts):
        try:
            with urllib.request.urlopen(req, timeout=timeout) as r:
                return r.read()
        except urllib.error.HTTPError as e:
            last = e
            if e.code in (429, 500, 502, 503, 504):
                time.sleep(4 * (i + 1))
                continue
            raise
        except Exception as e:
            last = e
            time.sleep(2 * (i + 1))
    raise last


def check_crossref(doi):
    url = "https://api.crossref.org/works/" + urllib.parse.quote(doi, safe="")
    try:
        d = json.loads(get(url))["message"]
    except Exception as e:
        return None, f"crossref: {type(e).__name__} {e}"
    title = (d.get("title") or [""])[0]
    auth = d.get("author") or []
    surname = auth[0].get("family", "") if auth else ""
    parts = (d.get("issued", {}).get("date-parts") or [[None]])[0]
    year = parts[0] if parts else None
    ctype = d.get("type", "")
    container = (d.get("container-title") or [""])[0]
    pub = d.get("publisher", "")
    return {"title": title, "surname": surname, "year": year, "type": ctype,
            "container": container, "publisher": pub, "registry": "Crossref"}, None


def check_datacite(doi):
    url = "https://api.datacite.org/dois/" + urllib.parse.quote(doi, safe="")
    try:
        a = json.loads(get(url))["data"]["attributes"]
    except Exception as e:
        return None, f"datacite: {type(e).__name__} {e}"
    titles = a.get("titles") or [{}]
    creators = a.get("creators") or [{}]
    return {"title": titles[0].get("title", ""),
            "surname": creators[0].get("familyName") or creators[0].get("name", ""),
            "year": a.get("publicationYear"),
            "type": (a.get("types") or {}).get("resourceTypeGeneral", ""),
            "container": (a.get("container") or {}).get("title", ""),
            "publisher": a.get("publisher", ""), "registry": "DataCite"}, None


def check_arxiv(arxiv_id):
    time.sleep(3.5)   # arXiv API asks for ~3s between requests
    ident = arxiv_id.split(":", 1)[1]
    url = f"http://export.arxiv.org/api/query?id_list={ident}&max_results=1"
    try:
        root = ET.fromstring(get(url))
    except Exception as e:
        return None, f"arxiv: {type(e).__name__} {e}"
    ns = {"a": "http://www.w3.org/2005/Atom"}
    entry = root.find("a:entry", ns)
    if entry is None:
        return None, "arxiv: no entry returned"
    title = (entry.findtext("a:title", "", ns) or "").strip()
    if "Error" in title:
        return None, f"arxiv: {title}"
    names = [e.findtext("a:name", "", ns) for e in entry.findall("a:author", ns)]
    pub = entry.findtext("a:published", "", ns)
    surname = names[0].split()[-1] if names else ""
    return {"title": title, "surname": surname,
            "year": int(pub[:4]) if pub[:4].isdigit() else None,
            "type": "preprint", "container": "arXiv", "publisher": "arXiv",
            "registry": "arXiv"}, None


def main():
    print(f"Verifying {len(REFS)} references against live registries\n")
    rows, failures, warnings = [], [], []
    for key, exp_sur, exp_year, ident, exp_title in REFS:
        if ident.startswith("arXiv:"):
            got, err = check_arxiv(ident)
        else:
            got, err = check_crossref(ident)
            if got is None:
                got2, err2 = check_datacite(ident)
                if got2 is not None:
                    got, err = got2, None
                else:
                    err = f"{err} | {err2}"
        if got is None:
            print(f"  FAIL      {key:24s} {ident}\n            {err}")
            failures.append((key, ident, err))
            rows.append({"Key": key, "Identifier": ident, "Status": "UNRESOLVED",
                         "Registry": "", "RegistryTitle": "", "RegistryFirstAuthor": "",
                         "RegistryYear": "", "Note": err})
            time.sleep(0.4)
            continue

        sim = overlap(exp_title, got["title"])
        sur_ok = exp_sur.lower().replace("'", "") in (got["surname"] or "").lower().replace("'", "") \
            or (got["surname"] or "").lower().replace("'", "") in exp_sur.lower().replace("'", "")
        yr_ok = got["year"] is not None and abs(int(got["year"]) - exp_year) <= 1
        flags = []
        if sim < 0.5:
            flags.append(f"title overlap {sim:.2f}")
        if not sur_ok:
            flags.append(f"author '{got['surname']}' vs '{exp_sur}'")
        if not yr_ok:
            flags.append(f"year {got['year']} vs {exp_year}")
        status = "OK" if not flags else "CHECK"
        if flags:
            warnings.append((key, flags))
        mark = "  ok      " if status == "OK" else "  CHECK   "
        print(f"{mark}{key:24s} {got['registry']:9s} {got['surname']:16s} "
              f"{got['year']}  {got['title'][:52]}")
        if flags:
            print(f"            -> {'; '.join(flags)}")
        rows.append({"Key": key, "Identifier": ident, "Status": status,
                     "Registry": got["registry"], "RegistryTitle": got["title"],
                     "RegistryFirstAuthor": got["surname"], "RegistryYear": got["year"],
                     "Note": "; ".join(flags)})
        time.sleep(0.4)

    import csv
    out = os.path.join(os.environ.get("REPRO_ROOT") or os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "papers", "ai-retractions", "refs", "references_verified_all.csv")
    with open(out, "w", newline="", encoding="utf-8-sig") as f:
        w = csv.DictWriter(f, fieldnames=list(rows[0].keys()), quoting=csv.QUOTE_ALL)
        w.writeheader()
        w.writerows(rows)

    print(f"\n{'='*70}")
    print(f"resolved      : {sum(1 for r in rows if r['Status'] != 'UNRESOLVED')}/{len(REFS)}")
    print(f"clean match   : {sum(1 for r in rows if r['Status'] == 'OK')}/{len(REFS)}")
    print(f"needs review  : {len(warnings)}")
    print(f"UNRESOLVED    : {len(failures)}")
    print(f"wrote {out}")
    return 1 if failures else 0


if __name__ == "__main__":
    sys.exit(main())
