"""p07_lint.py: language lint for the Phase 2 report (the language constraints of the pipeline prompt). Runs on two renders of the same template: normal what the reader sees; checked for dashes, which are banned everywhere, verbatim quotes included; skeleton every value token replaced by "§" and every verbatim source string by "¤", so that the language rules judge only the words I wrote, never a publisher's or an author's sentence. Errors fail the phase check. Warnings are printed for a human pass and do not fail it. """ import json, re, sys sys.path.insert(0, "/home/claude/mill/code") ALLOW_PATH = "/home/claude/mill/inputs/lint_allow.json" DASHES = {"\u2014": "em dash", "\u2013": "en dash", "\u2015": "horizontal bar", "\u2012": "figure dash"} FILLER = ["it is important to note", "it is worth noting", "it should be noted", "notably", "furthermore", "moreover", "additionally", "interestingly", "importantly", "crucially", "needless to say", "in conclusion", "in summary", "it goes without saying", "at the end of the day", "in order to"] GRANDIOSE = ["paradigm", "unprecedented", "sheds light", "shed light", "groundbreaking", "ground-breaking", "revolutioniz", "revolutionis", "game-chang", "game chang", "cutting-edge", "unlock", "unveil", "illuminat", "delve", "tapestry", "pivotal", "transformative", "landscape", "holistic", "leverag", "harness", "seamless", "robustly", "towards a new"] NUMBER_WORDS = ["two", "three", "four", "five", "six", "seven", "eight", "nine", "ten", "eleven", "twelve", "thirteen", "fourteen", "fifteen", "sixteen", "seventeen", "eighteen", "nineteen", "twenty", "hundred", "thousand", "million", "dozen", "half", "twice", "thrice"] def _strip_code(text): """Drop inline code spans, which hold file names and identifiers rather than prose.""" return re.sub(r"`[^`]*`", " ", text) def _prose_lines(text): """Lines that are prose (tables and headings included; horizontal table rules excluded).""" for i, line in enumerate(text.split("\n"), 1): if re.fullmatch(r"\s*\|?(\s*:?-{3,}:?\s*\|)+\s*", line): continue yield i, line def load_allow(): try: return json.load(open(ALLOW_PATH)) except FileNotFoundError: return [] def lint(normal, skeleton): errors, warnings = [], [] # 1. dashes, everywhere, including verbatim quotes for i, line in _prose_lines(normal): for ch, name in DASHES.items(): if ch in line: errors.append(("dash", i, f"{name} in: {line.strip()[:90]}")) body = re.sub(r"^\s*(?:[-*]|\d+\.)\s+", "", line) # list markers are not dashes if re.search(r"(?<=\w) - (?=\w)", body) or re.search(r"\w--\w|\s--\s", body): errors.append(("dash", i, f"hyphen used as a dash in: {line.strip()[:90]}")) sk = _strip_code(skeleton) low = sk.lower() # 2. filler and grandiose phrases (my words only) for ph in FILLER: for m in re.finditer(r"\b" + re.escape(ph) + r"\b", low): errors.append(("filler", low.count("\n", 0, m.start()) + 1, ph)) for ph in GRANDIOSE: for m in re.finditer(r"\b" + re.escape(ph), low): errors.append(("grandiose", low.count("\n", 0, m.start()) + 1, ph)) for m in re.finditer(r"\bnovel\b", low): errors.append(("grandiose", low.count("\n", 0, m.start()) + 1, "novel")) # 3. scaffolding patterns for m in re.finditer(r"\bnot only\b|\bnot just\b|\bnot merely\b", low): errors.append(("not only", low.count("\n", 0, m.start()) + 1, m.group(0))) for m in re.finditer(r"\bnot\b[^.;:!?\n|]{1,80}?,\s*but\b", low): errors.append(("not X, but Y", low.count("\n", 0, m.start()) + 1, m.group(0)[:80])) for m in re.finditer(r",\s+not\s+(?:a|an|the)?\s*\w+", low): warnings.append(("X, not Y", low.count("\n", 0, m.start()) + 1, sk[m.start():m.start() + 70].replace("\n", " "))) # 4. number words in my own prose: errors unless allow-listed with a reason allow = load_allow() for i, line in _prose_lines(sk): for m in re.finditer(r"\b(" + "|".join(NUMBER_WORDS) + r")\b", line, flags=re.I): ctx = line[max(0, m.start() - 40):m.end() + 40] if any(re.search(a["pattern"], ctx, flags=re.I) for a in allow): continue errors.append(("number word", i, ctx.strip())) for m in re.finditer(r"\bone\b", line, flags=re.I): ctx = line[max(0, m.start() - 40):m.end() + 40] if any(re.search(a["pattern"], ctx, flags=re.I) for a in allow): continue warnings.append(("the word one", i, ctx.strip())) # 5. tricolon heuristic: three short parallel items closing a sentence for m in re.finditer(r"\b(\w+(?: \w+){0,2}), (\w+(?: \w+){0,2}),? and (\w+(?: \w+){0,2})[.;]", sk): warnings.append(("possible tricolon", sk.count("\n", 0, m.start()) + 1, m.group(0)[:80])) # 6. sentence-length variation (prose paragraphs only; tables and headings skipped) paras = [l for l in sk.split("\n") if l and not l.startswith(("|", "#"))] sents = [s for p in paras for s in re.split(r"(?<=[.?!])\s+", p) if len(s.split()) >= 3] lens = [len(s.split()) for s in sents] stats = {} if lens: mu = sum(lens) / len(lens) sd = (sum((x - mu) ** 2 for x in lens) / len(lens)) ** 0.5 stats = {"sentences": len(lens), "mean_words": round(mu, 1), "sd_words": round(sd, 1), "cv": round(sd / mu, 2), "shortest": min(lens), "longest": max(lens)} if sd / mu < 0.35: warnings.append(("uniform sentence length", 0, f"coefficient of variation {sd / mu:.2f}")) return errors, warnings, stats def main(): from p06_report import render, qlog_counts vals = json.load(open("/home/claude/mill/outputs/values.json")) extra = qlog_counts(vals) extra["check_line"] = {"value": "lint run", "unit": "text", "estimand": "p07_lint.py", "source": "p07_lint.py", "kind": "text"} normal, _ = render(vals, "normal", extra) skeleton, _ = render(vals, "skeleton", extra) errors, warnings, stats = lint(normal, skeleton) for e in errors: print("ERROR", e) for w in warnings: print("warn ", w) print("sentence stats:", stats) print(f"lint: {len(errors)} errors, {len(warnings)} warnings") return 0 if not errors else 1 if __name__ == "__main__": sys.exit(main())