# PROVENANCE SNAPSHOT: local paths anonymized; not a portable execution entry.
#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
Experiment D0 — Can a machine discover the future-bearing critical coordinate?
===============================================================================

Project 801: Machine-Native Representation and Predictive State

Controlled benchmark based on the 1-D Brusselator used in paper #21:

    dX/dt = 1 - (B+1)X + X^2 Y + d_xx X
    dY/dt = B X - X^2 Y + 16 d_xx Y

on x in [0, 2*pi] with no-flux boundaries.

Paper #21 gives:
    B_c = 25/16
    k_c = 1/2
    critical species direction = (1, -5/16)
    fast first-mode direction  = (1, -5)

D0 does NOT give the critical coordinate to learned representations. It is used
only after training as an external oracle for evaluation.

Matched representations at latent d = 1,2,4,8:
    PCA, RANDOM, RECON_AE, PRED_AE
Reference controls:
    ORACLE_CRITICAL (d=1), ORACLE_FAST (d=1), FULL_LINEAR (d=128)

Fairness rule:
After each representation is learned/frozen, the SAME ridge probe is fit from
latent Z to current/future fields and to the known critical/fast coordinates.
Thus the primary comparison is about what the representation exposes, not about
different downstream decoder strengths.

The initial field deliberately contains a small critical component and larger
fast nuisance components. A future-predictive bottleneck should therefore prefer
what survives into the future rather than what dominates present variance.

Outputs:
    config.json
    numerical_audit.json
    dataset_diagnostics.csv
    representation_results.csv
    representation_summary.csv
    D0_future_error_<regime>.png
    D0_critical_R2_<regime>.png
    D0_tradeoff_d1.png
    README_results.md
    datasets/*.npz

Smoke:
python -X utf8 experiment_D0_predictive_critical_representation.py ^
  --outdir "LOCAL_PATH_045/smoke" ^
  --mode smoke

Full pilot:
python -X utf8 experiment_D0_predictive_critical_representation.py ^
  --outdir "LOCAL_PATH_046/pilot_v1" ^
  --mode full
"""

from __future__ import annotations

import argparse
import hashlib
import json
import math
import random
from dataclasses import dataclass
from pathlib import Path

import numpy as np
import pandas as pd
from scipy.fft import dct, idct

import torch
import torch.nn as nn
from torch.utils.data import DataLoader, TensorDataset

import matplotlib
matplotlib.use("Agg")
import matplotlib.pyplot as plt

BC = 25.0 / 16.0
CRIT_Y_OVER_X = -5.0 / 16.0
FAST_Y_OVER_X = -5.0
B_REGIMES = {"below": 1.50, "critical": BC, "above": 1.625}
LATENT_DIMS = [1, 2, 4, 8]
MODEL_SEEDS = [20261004, 20261005, 20261006]


def set_seed(seed: int):
    random.seed(seed)
    np.random.seed(seed)
    torch.manual_seed(seed)
    if torch.cuda.is_available():
        torch.cuda.manual_seed_all(seed)


def sha256_file(path: Path):
    h = hashlib.sha256()
    with path.open("rb") as f:
        while True:
            b = f.read(1024 * 1024)
            if not b:
                break
            h.update(b)
    return h.hexdigest()


def reaction_rhs(X, Y, B):
    x2y = X * X * Y
    return 1.0 - (B + 1.0) * X + x2y, B * X - x2y


def diffuse_neumann(U, D, dt, L):
    # Raw physical endpoint samples use the unorthogonalized DCT-I pair.
    n = U.shape[1]
    k = np.arange(n, dtype=np.float64) * math.pi / L
    Uh = dct(U, type=1, axis=1, norm=None)
    Uh *= np.exp(-D * k * k * dt)[None, :]
    return idct(Uh, type=1, axis=1, norm=None)


def simulate(X0, Y0, B, horizon, dt, L):
    # D(dt/2) R_Heun(dt) D(dt/2); adjacent half diffusions combine exactly.
    # This is a finite second-order approximation, not an exact critical spectrum.
    steps = int(round(horizon / dt))
    if dt <= 0 or horizon < 0 or not np.isclose(steps * dt, horizon, atol=1e-10, rtol=0):
        raise ValueError("Nonnegative horizon must be an integer multiple of positive dt")
    X = np.asarray(X0, dtype=np.float64).copy()
    Y = np.asarray(Y0, dtype=np.float64).copy()
    if steps == 0:
        return X, Y
    X = diffuse_neumann(X, 1.0, dt / 2.0, L)
    Y = diffuse_neumann(Y, 16.0, dt / 2.0, L)
    for step in range(steps):
        f1, g1 = reaction_rhs(X, Y, B)
        f2, g2 = reaction_rhs(X + dt * f1, Y + dt * g1, B)
        X = X + 0.5 * dt * (f1 + f2)
        Y = Y + 0.5 * dt * (g1 + g2)
        diffusion_dt = dt / 2.0 if step == steps - 1 else dt
        X = diffuse_neumann(X, 1.0, diffusion_dt, L)
        Y = diffuse_neumann(Y, 16.0, diffusion_dt, L)
        if not np.isfinite(X).all() or not np.isfinite(Y).all():
            raise FloatingPointError("non-finite simulation value; reduce --dt")
    return X, Y


def cosine_coeff(Udev, x, n, L):
    phi = np.cos(n * math.pi * x / L)
    w = np.ones_like(x)
    w[0] = w[-1] = 0.5
    return np.sum(Udev * (w * phi)[None, :], axis=1) / np.sum(w * phi * phi)


def critical_fast_coords(X, Y, B, x, L):
    cX = cosine_coeff(X - 1.0, x, 1, L)
    cY = cosine_coeff(Y - B, x, 1, L)
    # [cX,cY] = a*(1,-5/16) + s*(1,-5)
    a = (16.0 / 75.0) * (5.0 * cX + cY)
    s = cX - a
    return a, s


def generate_initial_conditions(
    n_samples, B, n_grid, L, seed,
    sigma_critical, sigma_fast1, sigma_high, max_mode,
):
    rng = np.random.default_rng(seed)
    x = np.linspace(0.0, L, n_grid)
    Xd = np.zeros((n_samples, n_grid), dtype=np.float64)
    Yd = np.zeros((n_samples, n_grid), dtype=np.float64)

    phi1 = np.cos(math.pi * x / L)
    a = rng.normal(0.0, sigma_critical, n_samples)
    s = rng.normal(0.0, sigma_fast1, n_samples)

    Xd += (a + s)[:, None] * phi1[None, :]
    Yd += (CRIT_Y_OVER_X * a + FAST_Y_OVER_X * s)[:, None] * phi1[None, :]

    # Uniform nuisance.
    Xd += rng.normal(0.0, sigma_high, (n_samples, 1))
    Yd += rng.normal(0.0, sigma_high, (n_samples, 1))

    # Higher-frequency nuisance.
    for n in range(2, max_mode + 1):
        phi = np.cos(n * math.pi * x / L)
        sig = sigma_high / math.sqrt(float(n))
        Xd += rng.normal(0.0, sig, n_samples)[:, None] * phi[None, :]
        Yd += rng.normal(0.0, sig, n_samples)[:, None] * phi[None, :]

    # Preserve positivity by uniformly shrinking a sample only when necessary.
    min_x = Xd.min(axis=1)
    min_y = Yd.min(axis=1)
    sx = np.ones(n_samples)
    sy = np.ones(n_samples)
    bad = min_x < -0.80
    sx[bad] = 0.80 / (-min_x[bad])
    margin_y = max(B - 0.20, 0.20)
    bad = min_y < -margin_y
    sy[bad] = margin_y / (-min_y[bad])
    scale = np.minimum(1.0, np.minimum(sx, sy))
    Xd *= scale[:, None]
    Yd *= scale[:, None]

    return x, 1.0 + Xd, B + Yd


def flatten_dev(X, Y, B):
    return np.concatenate([X - 1.0, Y - B], axis=1).astype(np.float32)


def make_split(n, seed):
    rng = np.random.default_rng(seed)
    idx = rng.permutation(n)
    ntr = int(round(0.60 * n))
    nva = int(round(0.20 * n))
    return np.sort(idx[:ntr]), np.sort(idx[ntr:ntr+nva]), np.sort(idx[ntr+nva:])


@dataclass
class RegimeData:
    name: str
    B: float
    x: np.ndarray
    current: np.ndarray
    future: np.ndarray
    a0: np.ndarray
    s0: np.ndarray
    af: np.ndarray
    sf: np.ndarray
    train_idx: np.ndarray
    val_idx: np.ndarray
    test_idx: np.ndarray


def build_regime(name, B, args, n_samples, seed):
    x, X0, Y0 = generate_initial_conditions(
        n_samples, B, args.n_grid, args.L, seed,
        args.sigma_critical, args.sigma_fast1, args.sigma_high, args.max_mode,
    )
    Xf, Yf = simulate(X0, Y0, B, args.horizon, args.dt, args.L)
    a0, s0 = critical_fast_coords(X0, Y0, B, x, args.L)
    af, sf = critical_fast_coords(Xf, Yf, B, x, args.L)
    tr, va, te = make_split(n_samples, seed + 991)
    return RegimeData(
        name, B, x,
        flatten_dev(X0, Y0, B), flatten_dev(Xf, Yf, B),
        a0.astype(np.float32), s0.astype(np.float32),
        af.astype(np.float32), sf.astype(np.float32), tr, va, te,
    )


def fit_standardizer(X):
    m = X.mean(axis=0)
    s = np.maximum(X.std(axis=0), 1e-6)
    return m.astype(np.float32), s.astype(np.float32)


def standardize(X, m, s):
    return ((X - m) / s).astype(np.float32)


def standardize_z(ztr, *others):
    m = ztr.mean(axis=0)
    s = np.maximum(ztr.std(axis=0), 1e-8)
    out = [((ztr - m) / s).astype(np.float64)]
    out += [((z - m) / s).astype(np.float64) for z in others]
    return out


def ridge_fit(Z, Y, alpha):
    Z = np.asarray(Z, np.float64)
    Y = np.asarray(Y, np.float64)
    A = np.concatenate([Z, np.ones((len(Z), 1))], axis=1)
    G = A.T @ A
    R = np.eye(G.shape[0]) * alpha
    R[-1, -1] = 0.0
    return np.linalg.solve(G + R, A.T @ Y)


def ridge_predict(Z, W):
    Z = np.asarray(Z, np.float64)
    A = np.concatenate([Z, np.ones((len(Z), 1))], axis=1)
    return A @ W


def r2(y, yp):
    y = np.asarray(y).reshape(-1).astype(np.float64)
    yp = np.asarray(yp).reshape(-1).astype(np.float64)
    ss0 = np.sum((y - y.mean()) ** 2)
    if ss0 <= 1e-15:
        return float("nan")
    return float(1.0 - np.sum((y - yp) ** 2) / ss0)


def mean_rel_l2(Y, Yp):
    return float(np.mean(
        np.linalg.norm(Yp - Y, axis=1) /
        (np.linalg.norm(Y, axis=1) + 1e-10)
    ))


def branch_accuracy(y, yp, threshold):
    y = np.asarray(y).reshape(-1)
    yp = np.asarray(yp).reshape(-1)
    mask = np.abs(y) >= threshold
    if mask.sum() == 0:
        return float("nan"), 0
    return float(np.mean(np.sign(y[mask]) == np.sign(yp[mask]))), int(mask.sum())


def pca_projection(Xtr, Xv, Xte, d):
    m = Xtr.mean(axis=0, keepdims=True)
    _, _, Vt = np.linalg.svd(Xtr - m, full_matrices=False)
    P = Vt[:d].T
    return (Xtr - m) @ P, (Xv - m) @ P, (Xte - m) @ P


def random_projection(Xtr, Xv, Xte, d, seed):
    rng = np.random.default_rng(seed)
    Q, _ = np.linalg.qr(rng.normal(size=(Xtr.shape[1], d)))
    Q = Q[:, :d]
    return Xtr @ Q, Xv @ Q, Xte @ Q


class BottleneckNet(nn.Module):
    def __init__(self, input_dim, latent_dim, output_dim):
        super().__init__()
        self.encoder = nn.Sequential(
            nn.Linear(input_dim, 128), nn.SiLU(),
            nn.Linear(128, 64), nn.SiLU(),
            nn.Linear(64, latent_dim),
        )
        # Linear decoder intentionally keeps task-relevant content directly
        # accessible from the bottleneck.
        self.decoder = nn.Linear(latent_dim, output_dim)

    def encode(self, x):
        return self.encoder(x)

    def forward(self, x):
        return self.decoder(self.encoder(x))


def train_bottleneck(Xtr, Ytr, Xv, Yv, d, seed, args, epochs, device):
    set_seed(seed)
    model = BottleneckNet(Xtr.shape[1], d, Ytr.shape[1]).to(device)
    opt = torch.optim.AdamW(model.parameters(), lr=args.lr, weight_decay=1e-4)
    loss_fn = nn.MSELoss()
    ds = TensorDataset(torch.from_numpy(Xtr), torch.from_numpy(Ytr))
    g = torch.Generator().manual_seed(seed)
    dl = DataLoader(ds, batch_size=args.batch_size, shuffle=True, generator=g, num_workers=0)
    xv = torch.from_numpy(Xv).to(device)
    yv = torch.from_numpy(Yv).to(device)

    best, best_epoch, best_state, wait = float("inf"), 0, None, 0
    for epoch in range(1, epochs + 1):
        model.train()
        for xb, yb in dl:
            xb, yb = xb.to(device), yb.to(device)
            opt.zero_grad(set_to_none=True)
            loss = loss_fn(model(xb), yb)
            loss.backward()
            opt.step()

        model.eval()
        with torch.no_grad():
            val = float(loss_fn(model(xv), yv).item())
        if val < best - 1e-8:
            best, best_epoch = val, epoch
            best_state = {k: v.detach().cpu().clone() for k, v in model.state_dict().items()}
            wait = 0
        else:
            wait += 1
        if wait >= args.patience:
            break

    if best_state is None:
        raise RuntimeError("no valid checkpoint")
    model.load_state_dict(best_state)
    return model, best, best_epoch


def encode(model, X, device):
    model.eval()
    with torch.no_grad():
        z = model.encode(torch.from_numpy(X).to(device))
    return z.cpu().numpy()


def evaluate(method, data, d, seed_label, ztr, zv, zte, args, train_meta=None):
    tr, te = data.train_idx, data.test_idx
    ztr, _, zte = standardize_z(ztr, zv, zte)

    Wf = ridge_fit(ztr, data.future[tr], args.ridge_alpha)
    fut_pred = ridge_predict(zte, Wf)
    Wc = ridge_fit(ztr, data.current[tr], args.ridge_alpha)
    cur_pred = ridge_predict(zte, Wc)

    Wa0 = ridge_fit(ztr, data.a0[tr, None], args.ridge_alpha)
    Waf = ridge_fit(ztr, data.af[tr, None], args.ridge_alpha)
    Ws0 = ridge_fit(ztr, data.s0[tr, None], args.ridge_alpha)
    pa0 = ridge_predict(zte, Wa0).reshape(-1)
    paf = ridge_predict(zte, Waf).reshape(-1)
    ps0 = ridge_predict(zte, Ws0).reshape(-1)
    bacc, bn = branch_accuracy(data.af[te], paf, args.branch_threshold)

    row = {
        "regime": data.name,
        "B": data.B,
        "method": method,
        "latent_dim": int(d),
        "seed": str(seed_label),
        "future_rel_l2_linear_probe": mean_rel_l2(data.future[te], fut_pred),
        "current_reconstruction_rel_l2_linear_probe": mean_rel_l2(data.current[te], cur_pred),
        "critical_R2_current": r2(data.a0[te], pa0),
        "critical_R2_future": r2(data.af[te], paf),
        "fast_mode_R2_current": r2(data.s0[te], ps0),
        "future_branch_sign_accuracy": bacc,
        "future_branch_n": bn,
    }
    if train_meta:
        row.update(train_meta)
    return row


def numerical_audit(args):
    x, X0, Y0 = generate_initial_conditions(
        32, BC, args.n_grid, args.L, 8801,
        args.sigma_critical, args.sigma_fast1, args.sigma_high, args.max_mode,
    )
    Xa, Ya = simulate(X0, Y0, BC, args.horizon, args.dt, args.L)
    Xb, Yb = simulate(X0, Y0, BC, args.horizon, args.dt / 2.0, args.L)
    A = flatten_dev(Xa, Ya, BC).astype(np.float64)
    Bf = flatten_dev(Xb, Yb, BC).astype(np.float64)
    rel = np.linalg.norm(A - Bf, axis=1) / (np.linalg.norm(Bf, axis=1) + 1e-12)
    aa, _ = critical_fast_coords(Xa, Ya, BC, x, args.L)
    ab, _ = critical_fast_coords(Xb, Yb, BC, x, args.L)
    return {
        "B": BC,
        "n_samples": 32,
        "dt": args.dt,
        "dt_half": args.dt / 2.0,
        "horizon": args.horizon,
        "mean_relative_field_difference": float(rel.mean()),
        "max_relative_field_difference": float(rel.max()),
        "critical_amplitude_rmse": float(np.sqrt(np.mean((aa - ab) ** 2))),
    }


def make_plots(results, outdir):
    q0 = results[results.method.isin(["PCA", "RANDOM", "RECON_AE", "PRED_AE"])]
    for regime in q0.regime.unique():
        q = q0[q0.regime == regime]
        fig, ax = plt.subplots(figsize=(7.2, 4.8))
        for method in ["PCA", "RANDOM", "RECON_AE", "PRED_AE"]:
            d = q[q.method == method]
            if d.empty:
                continue
            a = d.groupby("latent_dim", as_index=False).future_rel_l2_linear_probe.mean()
            ax.plot(a.latent_dim, a.future_rel_l2_linear_probe, marker="o", label=method)
        ax.set_xlabel("Latent dimension")
        ax.set_ylabel("Future relative L2 error (same ridge probe)")
        ax.set_title(f"D0 future prediction — {regime}")
        ax.legend()
        fig.tight_layout()
        fig.savefig(outdir / f"D0_future_error_{regime}.png", dpi=180)
        plt.close(fig)

        fig, ax = plt.subplots(figsize=(7.2, 4.8))
        for method in ["PCA", "RANDOM", "RECON_AE", "PRED_AE"]:
            d = q[q.method == method]
            if d.empty:
                continue
            a = d.groupby("latent_dim", as_index=False).critical_R2_future.mean()
            ax.plot(a.latent_dim, a.critical_R2_future, marker="o", label=method)
        ax.axhline(0.0, linewidth=1.0)
        ax.set_xlabel("Latent dimension")
        ax.set_ylabel("R2: latent -> future critical coordinate")
        ax.set_title(f"D0 critical-coordinate retention — {regime}")
        ax.legend()
        fig.tight_layout()
        fig.savefig(outdir / f"D0_critical_R2_{regime}.png", dpi=180)
        plt.close(fig)

    fig, ax = plt.subplots(figsize=(7.2, 5.2))
    q = q0[q0.latent_dim == 1]
    for method in ["PCA", "RANDOM", "RECON_AE", "PRED_AE"]:
        d = q[q.method == method]
        if d.empty:
            continue
        ax.scatter(d.current_reconstruction_rel_l2_linear_probe,
                   d.future_rel_l2_linear_probe, label=method)
    ax.set_xlabel("Current reconstruction relative L2 (d=1)")
    ax.set_ylabel("Future prediction relative L2 (d=1)")
    ax.set_title("D0 present-preservation vs future-prediction tradeoff")
    ax.legend()
    fig.tight_layout()
    fig.savefig(outdir / "D0_tradeoff_d1.png", dpi=180)
    plt.close(fig)


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("--outdir", type=Path, required=True)
    ap.add_argument("--mode", choices=["smoke", "full"], default="smoke")
    ap.add_argument("--device", default="auto")
    ap.add_argument("--n-grid", type=int, default=64)
    ap.add_argument("--L", type=float, default=2.0 * math.pi)
    ap.add_argument("--horizon", type=float, default=10.0)
    ap.add_argument("--dt", type=float, default=0.01)
    ap.add_argument("--n-per-regime", type=int, default=None)
    ap.add_argument("--epochs", type=int, default=None)
    ap.add_argument("--batch-size", type=int, default=128)
    ap.add_argument("--lr", type=float, default=1e-3)
    ap.add_argument("--patience", type=int, default=8)
    ap.add_argument("--sigma-critical", type=float, default=0.018)
    ap.add_argument("--sigma-fast1", type=float, default=0.055)
    ap.add_argument("--sigma-high", type=float, default=0.035)
    ap.add_argument("--max-mode", type=int, default=10)
    ap.add_argument("--ridge-alpha", type=float, default=1e-3)
    ap.add_argument("--branch-threshold", type=float, default=0.005)
    ap.add_argument("--save-checkpoints", action="store_true")
    args = ap.parse_args()

    if args.mode == "smoke":
        n_per_regime = args.n_per_regime or 180
        epochs = args.epochs or 8
        latent_dims = [1, 2]
        seeds = [MODEL_SEEDS[0]]
    else:
        n_per_regime = args.n_per_regime or 1800
        epochs = args.epochs or 50
        latent_dims = LATENT_DIMS
        seeds = MODEL_SEEDS

    outdir = args.outdir.resolve()
    outdir.mkdir(parents=True, exist_ok=True)
    (outdir / "datasets").mkdir(exist_ok=True)
    if args.save_checkpoints:
        (outdir / "checkpoints").mkdir(exist_ok=True)

    device = torch.device("cuda" if args.device == "auto" and torch.cuda.is_available() else
                          "cpu" if args.device == "auto" else args.device)

    print("=" * 78)
    print("Experiment D0 — predictive critical representation")
    print("=" * 78)
    print("mode:", args.mode)
    print("device:", device)
    if torch.cuda.is_available():
        print("GPU:", torch.cuda.get_device_name(0))
    print("B_c:", BC)
    print("latent dims:", latent_dims)
    print("seeds:", seeds)
    print("samples/regime:", n_per_regime)
    print("horizon:", args.horizon, "dt:", args.dt)

    script_path = Path(__file__).resolve()
    config = {
        "project": "801_machine_native_representation",
        "experiment": "D0_predictive_critical_representation",
        "mode": args.mode,
        "script": str(script_path),
        "script_sha256": sha256_file(script_path),
        "B_c": BC,
        "critical_k": 0.5,
        "critical_species_direction": [1.0, CRIT_Y_OVER_X],
        "fast_first_mode_direction": [1.0, FAST_Y_OVER_X],
        "B_regimes": B_REGIMES,
        "n_grid": args.n_grid,
        "L": args.L,
        "horizon": args.horizon,
        "dt": args.dt,
        "n_per_regime": n_per_regime,
        "latent_dims": latent_dims,
        "model_seeds": seeds,
        "epochs_max": epochs,
        "batch_size": args.batch_size,
        "learning_rate": args.lr,
        "patience": args.patience,
        "sigma_critical": args.sigma_critical,
        "sigma_fast1": args.sigma_fast1,
        "sigma_high": args.sigma_high,
        "max_mode": args.max_mode,
        "ridge_alpha": args.ridge_alpha,
        "branch_threshold": args.branch_threshold,
        "oracle_used_only_for_evaluation": True,
        "primary_comparison": "PRED_AE vs RECON_AE/PCA/RANDOM at matched d",
    }
    (outdir / "config.json").write_text(json.dumps(config, indent=2), encoding="utf-8")

    print("Running dt-halving audit...")
    audit = numerical_audit(args)
    (outdir / "numerical_audit.json").write_text(json.dumps(audit, indent=2), encoding="utf-8")
    print(audit)

    regimes = {}
    diagnostics = []
    for i, (name, B) in enumerate(B_REGIMES.items()):
        print(f"\nGenerating {name}, B={B:.6f}")
        data = build_regime(name, B, args, n_per_regime, 81000 + 1000 * i)
        regimes[name] = data
        diagnostics.append({
            "regime": name,
            "B": B,
            "n_samples": n_per_regime,
            "n_train": len(data.train_idx),
            "n_val": len(data.val_idx),
            "n_test": len(data.test_idx),
            "current_critical_sd": float(np.std(data.a0)),
            "current_fast_sd": float(np.std(data.s0)),
            "future_critical_sd": float(np.std(data.af)),
            "future_fast_sd": float(np.std(data.sf)),
            "critical_to_fast_variance_ratio": float(np.var(data.a0)/(np.var(data.s0)+1e-15)),
            "critical_corr_current_future": float(np.corrcoef(data.a0, data.af)[0,1]),
            "mean_current_state_energy": float(np.mean(data.current**2)),
        })
        np.savez_compressed(
            outdir / "datasets" / f"D0_{name}.npz",
            B=np.array([B]), x=data.x.astype(np.float32),
            current=data.current, future=data.future,
            a0=data.a0, s0=data.s0, af=data.af, sf=data.sf,
            train_idx=data.train_idx, val_idx=data.val_idx, test_idx=data.test_idx,
        )

    diag_df = pd.DataFrame(diagnostics)
    diag_df.to_csv(outdir / "dataset_diagnostics.csv", index=False, encoding="utf-8-sig")
    print("\nDataset diagnostics:")
    print(diag_df.to_string(index=False))

    rows = []
    for regime_name, data in regimes.items():
        print("\n" + "="*78)
        print("REGIME", regime_name, "B=", data.B)
        print("="*78)
        tr, va, te = data.train_idx, data.val_idx, data.test_idx
        xm, xs = fit_standardizer(data.current[tr])
        ym, ys = fit_standardizer(data.future[tr])
        Xtr = standardize(data.current[tr], xm, xs)
        Xv  = standardize(data.current[va], xm, xs)
        Xte = standardize(data.current[te], xm, xs)
        Ytr = standardize(data.future[tr], ym, ys)
        Yv  = standardize(data.future[va], ym, ys)

        rows.append(evaluate("FULL_LINEAR", data, Xtr.shape[1], "deterministic",
                             Xtr, Xv, Xte, args))
        rows.append(evaluate("ORACLE_CRITICAL", data, 1, "oracle",
                             data.a0[tr,None], data.a0[va,None], data.a0[te,None], args))
        rows.append(evaluate("ORACLE_FAST", data, 1, "oracle",
                             data.s0[tr,None], data.s0[va,None], data.s0[te,None], args))

        for d in latent_dims:
            print("\nlatent d=", d)
            ztr, zv, zte = pca_projection(Xtr, Xv, Xte, d)
            rows.append(evaluate("PCA", data, d, "deterministic", ztr, zv, zte, args))

            for seed in seeds:
                ztr, zv, zte = random_projection(Xtr, Xv, Xte, d, seed)
                rows.append(evaluate("RANDOM", data, d, seed, ztr, zv, zte, args))

            for seed in seeds:
                print(" RECON_AE seed", seed)
                m, val, ep = train_bottleneck(Xtr, Xtr, Xv, Xv, d, seed, args, epochs, device)
                rows.append(evaluate(
                    "RECON_AE", data, d, seed,
                    encode(m, Xtr, device), encode(m, Xv, device), encode(m, Xte, device), args,
                    {"best_val_objective": val, "best_epoch": ep},
                ))
                if args.save_checkpoints:
                    torch.save({"state_dict":m.state_dict(),"method":"RECON_AE","regime":regime_name,
                                "B":data.B,"latent_dim":d,"seed":seed,"best_val":val,"best_epoch":ep},
                               outdir/"checkpoints"/f"{regime_name}_RECON_d{d}_seed{seed}.pt")
                del m
                if torch.cuda.is_available(): torch.cuda.empty_cache()

                print(" PRED_AE seed", seed)
                m, val, ep = train_bottleneck(Xtr, Ytr, Xv, Yv, d, seed, args, epochs, device)
                rows.append(evaluate(
                    "PRED_AE", data, d, seed,
                    encode(m, Xtr, device), encode(m, Xv, device), encode(m, Xte, device), args,
                    {"best_val_objective": val, "best_epoch": ep},
                ))
                if args.save_checkpoints:
                    torch.save({"state_dict":m.state_dict(),"method":"PRED_AE","regime":regime_name,
                                "B":data.B,"latent_dim":d,"seed":seed,"best_val":val,"best_epoch":ep},
                               outdir/"checkpoints"/f"{regime_name}_PRED_d{d}_seed{seed}.pt")
                del m
                if torch.cuda.is_available(): torch.cuda.empty_cache()

            pd.DataFrame(rows).to_csv(outdir/"representation_results.csv", index=False, encoding="utf-8-sig")

    results = pd.DataFrame(rows)
    results.to_csv(outdir/"representation_results.csv", index=False, encoding="utf-8-sig")

    metrics = [
        "future_rel_l2_linear_probe",
        "current_reconstruction_rel_l2_linear_probe",
        "critical_R2_current",
        "critical_R2_future",
        "fast_mode_R2_current",
        "future_branch_sign_accuracy",
    ]
    named = {}
    for m in metrics:
        named[m+"_mean"] = (m, "mean")
        named[m+"_sd"] = (m, "std")
    named["n_runs"] = ("seed", "count")
    summary = results.groupby(["regime","B","method","latent_dim"], as_index=False).agg(**named)
    summary.to_csv(outdir/"representation_summary.csv", index=False, encoding="utf-8-sig")
    make_plots(results, outdir)

    lines = [
        "# Experiment D0 result summary", "",
        "Automatically generated descriptive summary; not a manuscript claim.", "",
        "## Numerical audit", "",
        f"- mean dt-vs-dt/2 field difference: {audit['mean_relative_field_difference']:.6g}",
        f"- max dt-vs-dt/2 field difference: {audit['max_relative_field_difference']:.6g}",
        f"- critical-amplitude RMSE: {audit['critical_amplitude_rmse']:.6g}", "",
    ]
    for regime in B_REGIMES:
        lines += [f"## {regime}", ""]
        q = summary[(summary.regime==regime) & summary.method.isin(["PCA","RANDOM","RECON_AE","PRED_AE"])]
        for d in latent_dims:
            qd = q[q.latent_dim==d].sort_values("future_rel_l2_linear_probe_mean")
            if not qd.empty:
                b = qd.iloc[0]
                lines.append(f"- d={d}: lowest mean future error = {b.method} "
                             f"({b.future_rel_l2_linear_probe_mean:.6f}); "
                             f"future critical R2={b.critical_R2_future_mean:.4f}")
        lines.append("")
    (outdir/"README_results.md").write_text("\n".join(lines), encoding="utf-8")

    key = summary[
        summary.method.isin(["PCA","RANDOM","RECON_AE","PRED_AE"])
    ][["regime","method","latent_dim","future_rel_l2_linear_probe_mean",
       "critical_R2_future_mean","current_reconstruction_rel_l2_linear_probe_mean",
       "fast_mode_R2_current_mean","n_runs"]].sort_values(
        ["regime","latent_dim","future_rel_l2_linear_probe_mean"]
    )

    print("\n" + "="*78)
    print("D0 COMPLETE")
    print("="*78)
    print("Results:", outdir/"representation_results.csv")
    print("Summary:", outdir/"representation_summary.csv")
    print("\nKEY TABLE")
    print(key.to_string(index=False))


if __name__ == "__main__":
    main()
