"""r12_package.py: build Online Resource 2 (esm/ESM_2.zip), the code and records behind the paper.

The archive holds the frozen plan with the SHA-256 digests registered at the freeze, the analysis code of
Phases 2 to 4 (the device pass that computed the per-path features included), the values files, the deviations
log, the check results, the per-path tables the analyses read or wrote, and a README that carries the article
title, journal, authors and the corresponding author's affiliation and e-mail address (R21). Files that hold
contact details of co-authors (inputs/authors.json) and the internal review records are left out; the README
says so. Run after a passing pre-packaging run (r10_check.py --prepackage); the final run of the check,
without the flag, verifies the archive against its manifest.
Every file sits at the top level of the archive, with no subfolders (Adam, at the Phase 5 gate: "no subfolders in any
zip"); the README says which part of the work each file belongs to.
"""
import sys, json, os, zipfile, hashlib, datetime, re, glob
sys.path.insert(0, "/home/claude/mill/phase4/code")
sys.path.insert(0, "/home/claude/mill/code")
import r_freeze
FROZEN = r_freeze.assert_freeze()

P2, P3, P4 = "/home/claude/mill", "/home/claude/mill/phase3", "/home/claude/mill/phase4"
OUT = f"{P4}/esm/ESM_2.zip"
sha = lambda p: hashlib.sha256(open(p, "rb").read()).hexdigest()

V3 = json.load(open(f"{P3}/outputs/values3.json"))
V4 = json.load(open(f"{P4}/outputs/values4.json"))
A = json.load(open(f"{P4}/inputs/authors.json"))
S4 = json.load(open(f"{P4}/inputs/sources4.json"))
front = open(f"{P4}/tex/src/01_front.tex").read()
title = re.search(r"\\title\[[^\]]*\]\{([^}]*)\}", front).group(1)
ca = next(a for a in A["authors"] if a.get("corresponding"))
inst = S4["institutions"][ca["inst"]]
check = json.load(open(f"{P4}/outputs/check4_result.json"))
assert check["status"] == "PASS" and check.get("prepackage") is True, "package only after a passing pre-packaging run of r10_check.py (--prepackage)"

GROUPS = []   # (group label, [(name in the archive, source path)]); the archive is flat, so the README names the groups
GROUPS.append(("the frozen plan and its rendered text", [("plan.json", f"{P3}/inputs/plan.json"), ("phase3_plan.md", f"{P3}/outputs/phase3_plan.md")]))
GROUPS.append(("Phase 2 code: exploratory analysis and the values-file library (vals.py)",
               [(os.path.basename(p), p) for p in sorted(glob.glob(f"{P2}/code/*.py")) + [f"{P2}/code/report_template.md"]]))
GROUPS.append(("Phase 2 device pass: the pass over the raw recordings that computed the per-path features and spectra",
               [(os.path.basename(p), p) for p in sorted(glob.glob(f"{P2}/raw/Phase2/code/*.py"))]))
GROUPS.append(("Phase 3 code, the plan's code; q_common.py, q_confirm.py and q_rules.py are the frozen estimators",
               [(os.path.basename(p), p) for p in sorted(glob.glob(f"{P3}/code/*.py")) + [f"{P3}/code/plan_template.md"]]))
GROUPS.append(("Phase 4 code: r01 confirmatory analyses, r02 assumption checks, r03 exploratory analyses, r04 figures, r05 facts, "
               "r11 analyses added after review (post hoc), r08 classification and tally, r09 data tables, r07 build, r06 language lint, "
               "r10 check, r12 this archive",
               [(os.path.basename(p), p) for p in sorted(glob.glob(f"{P4}/code/*.py"))]))
GROUPS.append(("Phase 4 inputs: reference records, sources and other inputs the scripts read",
               [(f, f"{P4}/inputs/{f}") for f in ("references.json", "sources4.json", "ijamt_recent_titles.json", "lint_allow4.json", "figure_inspection.json")]))
GROUPS.append(("manuscript template: the manuscript source with value tokens, as rendered by r07_build.py",
               [(os.path.basename(p), p) for p in sorted(glob.glob(f"{P4}/tex/src/*.tex"))]))
GROUPS.append(("values files: every number in the paper, by name, with its estimand and the script that wrote it",
               [("values.json", f"{P2}/outputs/values.json"), ("values3.json", f"{P3}/outputs/values3.json"), ("values4.json", f"{P4}/outputs/values4.json")]))
GROUPS.append(("logs: deviations from the plan and the check results",
               [("deviations4.json", f"{P4}/inputs/deviations4.json"), ("check3_result.json", f"{P3}/outputs/check3_result.json"), ("check4_result.json", f"{P4}/outputs/check4_result.json")]))
GROUPS.append(("data: per-path features and indicators, the leave-one-run-out refits and the saved confirmatory scores",
               [(f, f"{P2}/outputs/{f}") for f in ("segments.parquet",)] + [(f, f"{P4}/outputs/{f}") for f in ("a1_leave_one_run_out.csv", "per_path_indicators.csv", "line_frequencies.csv", "r01_arrays.npz", "r03_arrays.npz")]))
FILES = {}   # name in the archive (no folder): source path
for _, members in GROUPS:
    for arc, src in members:
        assert "/" not in arc and arc not in FILES and arc != "README.txt", f"archive name clash or folder: {arc}"
        FILES[arc] = src
PERSONAL = ("@" + "gm" + "ail." + "com").encode()   # built in pieces so that this file does not match itself
for p in FILES.values():
    assert os.path.exists(p), p
    body = open(p, "rb").read()
    assert PERSONAL not in body, p   # no personal e-mail address of the user in the archive

digests = {os.path.basename(path): FROZEN[name] for name, path in r_freeze.FROZEN.items()}
arc_of = {"plan.json": "plan.json", "q_common.py": "q_common.py", "q_confirm.py": "q_confirm.py", "q_rules.py": "q_rules.py"}
for fname, d in digests.items():
    assert sha(FILES[arc_of[fname]]) == d, fname
names = ", ".join(f"{a['given']} {a['family']}" for a in A["authors"])
lines = [f"Article: {title}",
         "Journal: The International Journal of Advanced Manufacturing Technology",
         f"Authors: {names}",
         f"Corresponding author: {ca['given']} {ca['family']}, {inst['name']}, {inst['city']}, {inst['state']}, {inst['country']}; {ca['email']}",
         "Online Resource 2 (ESM_2.zip): the analysis code, the frozen plan with its SHA-256 digests, the values files, the deviations log and the check results",
         "",
         "Frozen before any confirmatory statistic was computed (SHA-256):"]
lines += [f"  {arc_of[f]}  {d}" for f, d in digests.items()]
lines += ["",
          f"The digests were taken at {V4['facts_freeze_time_text']['value']}, and the confirmatory script (r01_confirm.py) wrote its results at {V4['facts_confirm_time_text']['value']}.",
          "Both times come from the authors' own files; no third party holds a time-stamped copy of the plan, so the digests show what was frozen but not when.",
          "Verify a digest with, for example: sha256sum plan.json",
          "",
          "Contents (every file is at the top level of the archive; this list says which part of the work each belongs to)",
          ]
for label, members in GROUPS:
    lines += [f"  {label}:", "    " + ", ".join(arc for arc, _ in members)]
lines += ["",
          "Run order in Phase 4: r01_confirm.py, r02_checks.py, r03_explore.py, r04_figures.py, r05_facts.py, r11_review.py,",
          "r08_tally.py, r09_esm.py, r10_check.py (builds the manuscript), r12_package.py.",
          "The scripts were run from a folder per phase and name their input and output paths near the top of each file.",
          "The raw recordings are the MSM dataset (CC BY 4.0), cited in the article; the device pass reads them, and its per-path",
          "spectra (about 100 MB) are not included. Left out on purpose: authors.json, which holds co-authors' contact",
          "details (the scripts that read it need a file of the same structure), and the internal review records.",
          "",
          f"Check at packaging: {check['check_line']}"]
readme = "\n".join(lines) + "\n"
assert "\u2014" not in readme and "\u2013" not in readme
assert len(FILES) == sum(len(m) for _, m in GROUPS)
os.makedirs(os.path.dirname(OUT), exist_ok=True)
with zipfile.ZipFile(OUT, "w", compression=zipfile.ZIP_DEFLATED) as z:
    z.writestr("README.txt", readme)
    for arc, src in sorted(FILES.items()):
        z.write(src, arc)
with zipfile.ZipFile(OUT) as z:
    assert not any("/" in n for n in z.namelist()), "the archive has a subfolder"
manifest = {arc: sha(src) for arc, src in FILES.items()}
json.dump({"archive": OUT, "files": len(FILES) + 1, "bytes": os.path.getsize(OUT), "sha256": sha(OUT), "manifest": manifest,
           "built_utc": datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")},
          open(f"{P4}/outputs/esm2_manifest.json", "w"), indent=1)
print("ESM_2.zip:", len(FILES) + 1, "files,", os.path.getsize(OUT), "bytes")