"""p00_sources.py: register every number taken from an external source in the values file.

Numbers are parsed by regular expression from the verified quotes in inputs/sources.json
(never typed by hand), so the provenance chain is page text -> verified quote -> parsed value.
"""
import json, re, sys
sys.path.insert(0, "/home/claude/mill/code")
from vals import Values

S = json.load(open("/home/claude/mill/inputs/sources.json"))
QC = json.load(open("/home/claude/mill/inputs/quote_checks.json"))
src = {s["id"]: s for s in S["sources"]}
art = src["S_article"]
q = art["quotes"]
assert all(QC["results"][k] for k in q), "unverified quote present"

V = Values(script="p00_sources.py")


def ext(name, value, unit, sid, quote_key=None, note=None):
    est = {"population": "external source", "filter": f"{sid}" + (f", quote {quote_key}" if quote_key else ""),
           "statistic": "value as printed", "retrieved": src[sid]["retrieved"]}
    V.set(name, value, unit, est, note=note, kind="external")


def hms(h, m, s):
    return int(h) * 3600 + int(m) * 60 + int(s)


# durations (Data Records)
t = q["q_durations"]
m = re.search(r"total of (\d+) hours (\d+) minutes (\d+) seconds", t)
ext("doc_total_duration_s", hms(*m.groups()), "s", "S_article", "q_durations")
m = re.search(r"of which (\d+)% \((\d+) h (\d+) m (\d+) s\)", t)
ext("doc_cut_share", int(m.group(1)) / 100, "fraction", "S_article", "q_durations")
ext("doc_cut_time_s", hms(*m.groups()[1:]), "s", "S_article", "q_durations")
for mach in ("imi_vm20i", "imi_vmx30ui", "tmf_vf10"):
    m = re.search(r"(\d+) h (\d+) m (\d+) s for " + mach, t)
    ext(f"doc_duration_{mach}_s", hms(*m.groups()), "s", "S_article", "q_durations")

# sampling
m = re.search(r"sampled at ([\d.]+) kHz", q["q_acc_units"])
ext("doc_fs_acc_hz", float(m.group(1)) * 1000, "Hz", "S_article", "q_acc_units")
m = re.search(r"at a ([\d,]+) Hz sampling rate with (\d+)-bit", q["q_sound"])
ext("doc_fs_audio_hz", int(m.group(1).replace(",", "")), "Hz", "S_article", "q_sound")
ext("doc_audio_bits", int(m.group(2)), "bit", "S_article", "q_sound")
m = re.search(r"(\d+)-second sampling period", q["q_ct"])
ext("doc_ct_period_s", int(m.group(1)), "s", "S_article", "q_ct")
m = re.search(r"approximately (\d+) s intervals", q["q_adapter"])
ext("doc_lab_sensor_update_s", int(m.group(1)), "s", "S_article", "q_adapter")
m = re.search(r"approximately ([\d.]+) s\)", q["q_polling"])
ext("doc_lab_poll_s", float(m.group(1)), "s", "S_article", "q_polling")
m = re.search(r"at (\d+) Hz", q["q_vf10_mtc"])
ext("doc_vf10_mtc_hz", int(m.group(1)), "Hz", "S_article", "q_vf10_mtc")
m = re.search(r"in (\d+)-minute segments", q["q_vf10_sound"])
ext("doc_vf10_sound_segment_min", int(m.group(1)), "min", "S_article", "q_vf10_sound")

# article metadata as displayed on the retrieval date
m = re.search(r"(\d+) Citations", q["q_citations"])
ext("doc_article_citations", int(m.group(1)), "citations", "S_article", "q_citations",
    note="citation count displayed on the article page on the retrieval date")
m = re.search(r"(\d+) Accesses", q["q_accesses"])
ext("doc_article_accesses", int(m.group(1)), "accesses", "S_article", "q_accesses")
m = re.search(r"Published: (\d+ \w+ \d{4})", q["q_published"])
ext("doc_article_published", m.group(1), "date", "S_article", "q_published")

# Table 4 (percent of cutting time per label, basis not stated in the article)
t4 = src["S_table4"]["table"]
for row in t4["rows"]:
    mach = row[0]
    for lab, cell in zip((0, 1, 2), row[1:]):
        ext(f"doc_t4_{mach}_l{lab}", float(cell) / 100, "fraction", "S_table4",
            note="Table 4 cell as printed, divided by 100")

# Kaggle release metadata (raw JSON)
kf = src["S_kaggle_api"]["fields"]
ext("kaggle_total_bytes", kf["totalBytes"], "bytes", "S_kaggle_api")
ext("kaggle_version", kf["currentVersionNumber"], "version", "S_kaggle_api")
ext("kaggle_last_updated", kf["lastUpdated"][:10], "date", "S_kaggle_api")
ext("kaggle_downloads", kf["downloadCount"], "downloads", "S_kaggle_api")
ext("kaggle_views", kf["viewCount"], "views", "S_kaggle_api")
ext("kaggle_votes", kf["voteCount"], "votes", "S_kaggle_api")
ext("kaggle_notebooks", kf["kernelCount"], "notebooks", "S_kaggle_api")
ext("kaggle_topics", kf["topicCount"], "discussion topics", "S_kaggle_api")
ext("sources_retrieved", art["retrieved"], "date", "S_article")

# quote verification tally (computed)
V.set("n_quotes_verified", sum(1 for k in q if QC["results"].get(k)), "quotes",
      {"population": "quotes from S_article in inputs/sources.json", "filter": "substring of page text after space normalisation",
       "statistic": "count"})
V.set("n_quotes_total", len(q), "quotes",
      {"population": "quotes from S_article in inputs/sources.json", "filter": "none", "statistic": "count"})
V.save()
print("p00_sources: registered", len(V._touched), "values")