# PROVENANCE SNAPSHOT: local paths anonymized; not a portable execution entry.
"""Prospective readout validation and common-output POD controls.

Stages are append-only. No neural training, test-based selection, or old-file writes.
"""
import os
for _key in ('OMP_NUM_THREADS','OPENBLAS_NUM_THREADS','MKL_NUM_THREADS','NUMEXPR_NUM_THREADS'):
    os.environ[_key]='1'
import sys, ast, csv, json, math, time, hashlib, datetime, itertools, ctypes, threading, traceback
from pathlib import Path
sys.dont_write_bytecode=True
sys.stdout.reconfigure(encoding='utf-8')
HERE=Path(__file__).resolve().parent
PAPER=HERE.parents[2]
BASE=Path(r'LOCAL_PATH_047/801_machine_native_representation')
OUT=BASE/'_fresh_validation/20261029_readout_prospective_rank_control_v1'
DATA=BASE/'_revalidation/20261007_solver_v2/D0/datasets/D0_critical.npz'
REC=BASE/'_representation_budget/20261008_equalbit_dimension_v1'
OLD=BASE/'_analysis/20261025_frozen_readout45_v2'
FRESH=BASE/'_fresh_validation/20261011_D0_frozen_PRED_RECON_B16_v1'
SOLVER=BASE/'_representation_budget/20261008_D0_bitcode_pilot_v1/source_snapshot/experiment_D0_solver_corrected_v2.py'
PROTOCOL=HERE/'execution_protocol.json'
SEEDS=[20261004,20261005,20261006]
ALPHAS=[0.,1e-6,1e-4,1e-2,1.]
N=720
GEN_SEED=202610290801
BOOT_SEED=202610290802
TZ=datetime.timezone(datetime.timedelta(hours=8))
def now():return datetime.datetime.now(TZ).isoformat()
def sha(p):
    with Path(p).open('rb') as f:return hashlib.file_digest(f,'sha256').hexdigest()
def dump(p,o):Path(p).write_text(json.dumps(o,ensure_ascii=False,indent=2,allow_nan=False)+'\n',encoding='utf-8')
def load(p):return json.loads(Path(p).read_text(encoding='utf-8'))
def npz(p):
    with np.load(p,allow_pickle=False) as z:return {k:z[k].copy() for k in z.files}
def memory():
    from ctypes import wintypes as W
    class PMC(ctypes.Structure):
        _fields_=[('cb',W.DWORD),('faults',W.DWORD)]+[(k,ctypes.c_size_t) for k in
          ['peak_ws','ws','peak_paged','paged','peak_nonpaged','nonpaged','pagefile','peak_pagefile','private']]
    k=ctypes.WinDLL('kernel32',use_last_error=True);k.GetCurrentProcess.restype=W.HANDLE
    p=ctypes.WinDLL('psapi',use_last_error=True);p.GetProcessMemoryInfo.argtypes=[W.HANDLE,ctypes.POINTER(PMC),W.DWORD]
    m=PMC();m.cb=ctypes.sizeof(m)
    if not p.GetProcessMemoryInfo(k.GetCurrentProcess(),ctypes.byref(m),m.cb):raise ctypes.WinError()
    return {'working_set_bytes':m.ws,'peak_working_set_bytes':m.peak_ws,'private_bytes':m.private}
def affinity():
    from ctypes import wintypes as W
    k=ctypes.WinDLL('kernel32',use_last_error=True);k.GetCurrentProcess.restype=W.HANDLE
    k.GetProcessAffinityMask.argtypes=[W.HANDLE,ctypes.POINTER(ctypes.c_size_t),ctypes.POINTER(ctypes.c_size_t)]
    k.SetProcessAffinityMask.argtypes=[W.HANDLE,ctypes.c_size_t]
    a,b=ctypes.c_size_t(),ctypes.c_size_t();h=k.GetCurrentProcess()
    if not k.GetProcessAffinityMask(h,ctypes.byref(a),ctypes.byref(b)):raise ctypes.WinError()
    single=a.value & -a.value
    if not k.SetProcessAffinityMask(h,single):raise ctypes.WinError()
    return {'original_mask':a.value,'single_cpu_mask':single}
def outbytes():return sum(p.stat().st_size for p in OUT.rglob('*') if p.is_file())
def check_sources(pr):
    changes=[p for p,h in pr['protected_sha256'].items() if not Path(p).is_file() or sha(p)!=h]
    assert not changes,changes
def stage_start(stage):
    pr=load(PROTOCOL);check_sources(pr)
    assert not (OUT/f'{stage}_started.json').exists(),'Refuse automatic repeat of stage'
    started=time.perf_counter();stop=threading.Event();aff=affinity()
    dump(OUT/f'{stage}_started.json',{'at':now(),'protocol_sha256':sha(PROTOCOL),'affinity':aff})
    def guard():
        while not stop.wait(.2):
            m=memory();elapsed=time.perf_counter()-started;size=outbytes()
            reason='wall_seconds' if elapsed>900 else 'peak_working_set' if m['peak_working_set_bytes']>2**30 else 'output_bytes' if size>100000000 else None
            if reason:
                dump(OUT/f'{stage}_budget_aborted.json',{'at':now(),'reason':reason,'seconds':elapsed,'memory':m,'output_bytes':size})
                os._exit(124)
    threading.Thread(target=guard,daemon=True).start()
    return pr,started,stop
def finish(stage,pr,t,stop,more=None):
    check_sources(pr);m=memory();size=outbytes();elapsed=time.perf_counter()-t
    assert elapsed<=900 and m['peak_working_set_bytes']<=2**30 and size<100000000
    receipt={'status':'PASS','at':now(),'seconds':elapsed,'memory':m,'output_bytes':size,'sources_unchanged':True}
    receipt.update(more or {});dump(OUT/f'{stage}_receipt.json',receipt);stop.set();print(json.dumps(receipt,ensure_ascii=False),flush=True)
def summary(a):
    return {'mean':float(a.mean()),'q25':float(np.quantile(a,.25)),'median':float(np.median(a)),
            'q75':float(np.quantile(a,.75)),'q90':float(np.quantile(a,.9)),'max':float(a.max())}
def physical(F):
    F=np.asarray(F,np.float64);grid=np.linspace(0,2*np.pi,64);w=np.ones(64);w[[0,-1]]=.5
    uv=[]
    for n in range(3):
        p=np.cos(n*grid/2);den=np.sum(w*p*p)
        uv.append(np.c_[(F[:,:64]*(w*p)).sum(1)/den,(F[:,64:]*(w*p)).sum(1)/den])
    a=(16/75)*(5*uv[1][:,0]+uv[1][:,1]);s=uv[1][:,0]-a
    return np.c_[a,s,uv[0],uv[2]]
def expand(F):
    pairs=np.array(list(itertools.combinations_with_replacement(range(F.shape[1]),2)),dtype=np.int64)
    return np.column_stack([F]+[F[:,i]*F[:,j] for i,j in pairs]),pairs
def make_features(X,latents):
    fs={};pairs={}
    for seed,z in latents.items():
        fs[f'PRED_AFFINE_{seed}']=z.astype(np.float64)
        fs[f'PRED_QUADRATIC_{seed}'],pairs[f'PRED_QUADRATIC_{seed}']=expand(z.astype(np.float64))
    fs['PHYSICAL_AFFINE_6']=physical(X)
    fs['PHYSICAL_QUADRATIC_27'],pairs['PHYSICAL_QUADRATIC_27']=expand(fs['PHYSICAL_AFFINE_6'])
    fs['FULL_CURRENT_AFFINE_128']=X.astype(np.float64)
    return fs,pairs
def parts(F):
    f=physical(F);p=np.cos(np.linspace(0,2*np.pi,64)/2)
    c=np.c_[np.outer(f[:,0],p),np.outer(-5/16*f[:,0],p)]
    s=np.c_[np.outer(f[:,1],p),np.outer(-5*f[:,1],p)]
    u=np.c_[np.repeat(f[:,2,None],64,1),np.repeat(f[:,3,None],64,1)]
    return {'critical':c,'fast':s,'uniform':u,'remaining':F-c-s-u}
def r2(y,p):return float(1-np.sum((p-y)**2)/np.sum((y-y.mean(0))**2))
def metrics(Y,p):
    den=np.linalg.norm(Y,axis=1)+1e-10;e=p-Y;ab=np.linalg.norm(e,axis=1);rel=ab/den
    f,fp=physical(Y),physical(p);decomp=parts(e)
    en={k:np.sum(v*v,axis=1)/den**2 for k,v in decomp.items()}
    cr={a+'__'+b:2*np.sum(decomp[a]*decomp[b],axis=1)/den**2 for a,b in itertools.combinations(decomp,2)}
    closure=float(np.max(np.abs(sum(en.values())+sum(cr.values())-rel**2)));assert closure<3e-13
    return {'relative_L2':summary(rel),'absolute_L2':summary(ab),'critical_R2':r2(f[:,0],fp[:,0]),
            'uniform_R2':r2(f[:,2:4],fp[:,2:4]),'mean_squared_relative_L2':float(np.mean(rel**2)),
            'squared_components':{k:float(v.mean()) for k,v in en.items()},'cross_terms':{k:float(v.mean()) for k,v in cr.items()},
            'max_squared_closure':closure},rel,ab
def numerical_rank(A):
    s=np.linalg.svd(A,compute_uv=False);tol=1e-10*float(s[0]) if len(s) and s[0]>0 else 0.
    return {'rank':int(np.sum(s>tol)),'relative_threshold':1e-10,'absolute_threshold':tol,'singular_values':s.tolist()}
def predict(w,name,F,basis=None):
    H=np.c_[(F-w[name+'__feature_mean'])/w[name+'__feature_std'],np.ones(len(F))]
    q=H@w[name+'__coef_scaled_target']
    if basis is not None:q=q@basis.T
    return q*float(w['target_global_rms'][0])+w['target_mean']
def prepare():
    assert not PROTOCOL.exists() and not OUT.exists(),'New protocol/run only'
    protected=load(PAPER/'过程文件夹/_internal/venue_selection_v026_20261029/source_hashes_before.json')
    plan=PAPER/'过程文件夹/_internal/strengthening_plan_v026_20261029/实质补强最小方案_待执行确认.md'
    assert sha(plan)=='aaf5905d232cbc646eb857806e5dccbfe38e5aaf4ca749ee88c425ac4f10c329'
    known={DATA:'8f142e3e40c73344bab6009030b348b13282aeb3c1f4f71cdf5de7fc18cb7f03',
           SOLVER:'a0993f2b7bfceeac56516059314c1bec7adf1501812b21e5a3158527d3cb0995',
           OLD/'selected_readouts.npz':'b41e175c25a39242fe3b6fddcbe36b1ea3ec66844c5f4acabd1244decde03c5d'}
    paths=[plan,Path(__file__),DATA,SOLVER,OLD/'selected_readouts.npz',OLD/'selection_frozen.json',
           REC/'arrays/D0_input_scalers_and_indices.npz',FRESH/'numerical_futures.npz']
    cp_hash=['2f50c4567540da12d4bbd74ade259472b38c4471ac34703d4c95b7838930b989',
             '34c48b45b4d35a94a22ab3ade437e0860c653a87a6d10194c3232e8ab13dfd7f',
             '3a0f769475376a40cbb9f83aba7f603f53c83e91509cb3144d50147511676aae']
    for seed,h in zip(SEEDS,cp_hash):
        cp=REC/f'checkpoints/D0_PRED_AE_d4_{seed}.pt';known[cp]=h;paths.extend([cp,REC/f'arrays/D0_PRED_AE_d4_{seed}_float_artifacts.npz'])
    for p,h in known.items():assert sha(p)==h,str(p)
    paths.extend([PAPER/'过程文件夹/RESULTS_LEDGER.md',PAPER/'过程文件夹/CLAIM_BOUNDARIES.md'])
    protected.update({str(p):sha(p) for p in paths})
    d=npz(DATA);sc=npz(REC/'arrays/D0_input_scalers_and_indices.npz')
    assert d['current'].shape==d['future'].shape==(1800,128)
    indices=[d[k] for k in ['train_idx','val_idx','test_idx']]
    assert list(map(len,indices))==[1080,360,360] and np.array_equal(np.sort(np.concatenate(indices)),np.arange(1800))
    for k in ['train_idx','val_idx','test_idx']:assert np.array_equal(d[k],sc[k])
    assert all(np.isfinite(v).all() for v in d.values())
    pr={'identity':'FIELD_PROSPECTIVE720_COMMON_OUTPUT_V1','frozen_at':now(),'author_authorization':'Administrative authorization text omitted from anonymous provenance',
        'output':str(OUT),'data_seed':GEN_SEED,'n_new_independent_initial_states':N,'model_seeds':SEEDS,'alphas':ALPHAS,
        'model':{'n_grid':64,'L':2*math.pi,'B':25/16,'horizon':10.,'dt':.0025,'sigma_critical':.018,'sigma_fast1':.055,'sigma_high':.035,'max_mode':10},
        'fit':{'new_fit_calls':90,'selected_readers':18,'output_ranks':[4,14],'basis':'ONE training-centered globally scaled target SVD; sign largest-absolute entry positive',
               'target_scale':'original train-only mean and global centered RMS','features':'raw frozen PRED latent or 6 physical amplitudes or 128 input values; quadratic all i<=j then train-only standardization floor1e-12',
               'ridge':'SVD augmented least squares; unpenalized intercept; select old validation mean per-state relative L2; exact ties larger alpha',
               'numerical_rank_relative_threshold':1e-10},
        'primary':{'identity':'UNRESTRICTED PRED affine minus quadratic, averaged seed losses within initial state then over720',
                   'bootstrap_seed':BOOT_SEED,'replicates':10000,'quantiles':[.025,.975],'method':'paired initial-state percentile bootstrap; linear quantiles; approximate95%; no pvalue',
                   'criterion':'lower approximate95% percentile endpoint >0; otherwise NOT_CONFIRMED'},
        'secondary':'all27 readers; ranks4/14; physical-vs-PRED comparisons; critical/uniform R2; four squared physical components and all six doubled cross terms; absolute errors and q25/median/q75/q90/max; no secondary confirmatory tests',
        'evaluation_target':'float64 corrected integration endpoints; float32 deviation future castfloat64 for evaluation; denominator norm+1e-10',
        'step_sensitivity':{'first_fixed_states':36,'dt':.00125,'rule':'descriptive endpoint/primary direction; no full trueODE claim or automated reintegration'},
        'historical_encoder_check':{'rows':'original training and validation only','raw_latent_max_abs_tolerance':1e-4,'all_differences_recorded':True},
        'hard_caps':{'per_stage_seconds':900,'peak_working_set_bytes':2**30,'new_E_output_bytes':100000000,'cpu_cores':1,'blas_threads':1},
        'prohibited':['new_neural_training','new_hyperparameter_search_after_seal','test_or_fresh_based_selection','repeat_generation','original_results_overwrite','publication','submission'],
        'E3':'SOLVER_CORRECTED_PRIMARY_NOT_CONFIRMED_UNCHANGED','protected_sha256':protected}
    check_sources(pr);OUT.mkdir(parents=True,exist_ok=False)
    dump(PROTOCOL,pr);dump(OUT/'execution_protocol.json',pr)
    dump(HERE/'preflight.json',{'at':now(),'passed':True,'sources_verified':len(protected),'splits_disjoint_complete':True,'no_fit_or_generation_yet':True})
    print(json.dumps({'prepared':True,'protocol_sha256':sha(PROTOCOL),'protected_files':len(protected)}))
def fit():
    pr,t,stop=stage_start('fit')
    from threadpoolctl import threadpool_limits
    with threadpool_limits(limits=1):
        d=npz(DATA);tr,va=d['train_idx'],d['val_idx'];X=d['current'].astype(np.float64)
        # No old test target or latent is used in fitting/selection.
        Ytr=d['future'][tr].astype(np.float64);Yva=d['future'][va].astype(np.float64)
        muY=Ytr.mean(0);rms=float(np.sqrt(np.mean((Ytr-muY)**2)));Ys=(Ytr-muY)/rms
        _,sing,Vt=np.linalg.svd(Ys,full_matrices=False)
        for row in Vt:
            if row[np.argmax(np.abs(row))]<0:row*=-1
        old=npz(OLD/'selected_readouts.npz');assert np.array_equal(muY,old['target_mean']) and rms==old['target_global_rms'][0]
        lattr={};latva={}
        for seed in SEEDS:
            z=npz(REC/f'arrays/D0_PRED_AE_d4_{seed}_float_artifacts.npz');lattr[seed]=z['latent_train'];latva[seed]=z['latent_val']
        ft,pairs=make_features(X[tr],lattr);fv,_=make_features(X[va],latva)
        selected={'target_mean':muY,'target_global_rms':np.array([rms]),'train_idx':tr,'val_idx':va,'pod_singular_values':sing}
        allfits={};rows=[];choices={}
        for rank in [4,14]:
            basis=Vt[:rank].T.copy();selected[f'POD{rank}_basis']=basis;target=Ys@basis
            for name,F in ft.items():
                key=f'POD{rank}__{name}';mu=F.mean(0);sd=np.maximum(F.std(0),1e-12)
                H=np.c_[(F-mu)/sd,np.ones(len(F))];Hv=np.c_[(fv[name]-mu)/sd,np.ones(len(Yva))]
                pen=np.eye(H.shape[1]);pen[-1,-1]=0;candidates=[]
                for alpha in ALPHAS:
                    A=H if alpha==0 else np.vstack([H,np.sqrt(alpha)*pen])
                    B=target if alpha==0 else np.vstack([target,np.zeros((H.shape[1],rank))])
                    coef=np.linalg.lstsq(A,B,rcond=None)[0]
                    residual=H@coef-target;grad=H.T@residual+alpha*pen@coef
                    kkt=float(np.linalg.norm(grad)/max(np.linalg.norm(H)*np.linalg.norm(residual)+alpha*np.linalg.norm(coef),1e-30));assert kkt<5e-8
                    pred=muY+rms*(Hv@coef@basis.T)
                    loss=float(np.mean(np.linalg.norm(pred-Yva,axis=1)/(np.linalg.norm(Yva,axis=1)+1e-10)))
                    fid=f'fit_{len(rows)+1:03d}';allfits[fid]=coef
                    rows.append({'fit_number':len(rows)+1,'key':key,'rank':rank,'features':F.shape[1],'alpha':alpha,'validation_mean_relative_L2':loss,'stationarity_backward_residual':kkt})
                    candidates.append((loss,-alpha,coef.copy(),alpha,fid))
                best=min(candidates,key=lambda z:(z[0],z[1]));choices[key]={'alpha':best[3],'validation_mean_relative_L2':best[0],'fit_id':best[4],
                     'features':F.shape[1],'coefficient_count':int(best[2].size),'numerical_map_rank':numerical_rank(best[2][:-1]@basis.T)}
                selected[key+'__coef_scaled_target']=best[2];selected[key+'__feature_mean']=mu;selected[key+'__feature_std']=sd
                if name in pairs:selected[key+'__quadratic_pairs']=pairs[name]
                dump(OUT/'fit_progress.json',{'at':now(),'actual_fit_calls':len(rows),'selected':len(choices)})
        assert len(rows)==90 and len(choices)==18
        np.savez_compressed(OUT/'all90_coefficients.npz',**allfits)
        np.savez_compressed(OUT/'selected18_readouts.npz',**selected)
        with (OUT/'fit_grid.csv').open('w',encoding='utf-8',newline='') as f:
            w=csv.DictWriter(f,fieldnames=list(rows[0]));w.writeheader();w.writerows(rows)
        dump(OUT/'selection_sealed.json',{'at':now(),'choices':choices,'new_fit_calls':len(rows),'old_readers':9,'new_selected_readers':18,
             'new_initials_or_futures_generated':False,'protocol_sha256':sha(PROTOCOL),
             'frozen_files_sha256':{str(p):sha(p) for p in [OUT/'all90_coefficients.npz',OUT/'selected18_readouts.npz',OUT/'fit_grid.csv',OLD/'selected_readouts.npz']}})
    finish('fit',pr,t,stop,{'actual_fit_calls':90,'selected':18})
def solver_functions():
    # Execute only the exact AST of the whitelisted frozen source definitions.
    import torch
    from scipy.fft import dct,idct
    names={'reaction_rhs','diffuse_neumann','simulate','generate_initial_conditions','flatten_dev','BottleneckNet'}
    tree=ast.parse(SOLVER.read_text(encoding='utf-8-sig'));nodes=[n for n in tree.body if isinstance(n,(ast.FunctionDef,ast.ClassDef)) and n.name in names]
    assert {n.name for n in nodes}==names
    ns={'np':np,'math':math,'nn':torch.nn,'torch':torch,'dct':dct,'idct':idct,'CRIT_Y_OVER_X':-5/16,'FAST_Y_OVER_X':-5.}
    exec(compile(ast.Module(body=nodes,type_ignores=[]),str(SOLVER),'exec'),ns)
    return ns
def generate():
    pr,t,stop=stage_start('generate');seal=load(OUT/'selection_sealed.json')
    assert sha(PROTOCOL)==seal['protocol_sha256']
    for p,h in seal['frozen_files_sha256'].items():assert sha(p)==h
    import torch
    from threadpoolctl import threadpool_limits
    torch.set_num_threads(1);torch.set_num_interop_threads(1)
    with threadpool_limits(limits=1):
        sol=solver_functions();sc=npz(REC/'arrays/D0_input_scalers_and_indices.npz');d=npz(DATA)
        encoders={};checks=[]
        for seed in SEEDS:
            state=torch.load(REC/f'checkpoints/D0_PRED_AE_d4_{seed}.pt',map_location='cpu',weights_only=True)
            net=sol['BottleneckNet'](128,4,128);net.load_state_dict(state['state_dict']);net.eval();encoders[seed]=net
            art=npz(REC/f'arrays/D0_PRED_AE_d4_{seed}_float_artifacts.npz')
            for split in ['train','val']:
                xx=((d['current'][d[split+'_idx']]-sc['input_mean'])/sc['input_std']).astype(np.float32)
                with torch.inference_mode():zz=net.encode(torch.from_numpy(xx)).numpy()
                err=float(np.max(np.abs(zz-art['latent_'+split])));assert err<=1e-4
                checks.append({'seed':seed,'split':split,'max_abs_raw_latent_difference':err,'bitwise_equal':bool(np.array_equal(zz,art['latent_'+split]))})
        dump(OUT/'historical_forward_checks.json',{'at':now(),'checks':checks,'no_new_data_yet':True})
        cfg=pr['model'];gt=time.perf_counter()
        x,X0,Y0=sol['generate_initial_conditions'](N,cfg['B'],cfg['n_grid'],cfg['L'],GEN_SEED,cfg['sigma_critical'],cfg['sigma_fast1'],cfg['sigma_high'],cfg['max_mode'])
        X=sol['flatten_dev'](X0,Y0,cfg['B']);assert np.isfinite(X).all()
        newrows=[hashlib.sha256(row.tobytes()).hexdigest() for row in X]
        oldx=npz(FRESH/'numerical_futures.npz')['current']
        oldrows={hashlib.sha256(row.astype(np.float32).tobytes()).hexdigest() for row in np.vstack([d['current'],oldx])}
        assert len(set(newrows))==N and not (set(newrows)&oldrows)
        dump(OUT/'generation_before_futures.json',{'at':now(),'seal_sha256':sha(OUT/'selection_sealed.json'),'data_seed':GEN_SEED,'all_new_rows_unique_and_no_historical_overlap':True,'row_sha256':newrows})
        np.savez_compressed(OUT/'physical_initials.npz',x=x,X0=X0,Y0=Y0,current=X,sample_id=np.arange(N))
        Xf,Yf=sol['simulate'](X0,Y0,cfg['B'],cfg['horizon'],cfg['dt'],cfg['L'])
        main_seconds=time.perf_counter()-gt
        np.savez_compressed(OUT/'numerical_futures.npz',future=sol['flatten_dev'](Xf,Yf,cfg['B']),Xf=Xf,Yf=Yf,sample_id=np.arange(N))
        ht=time.perf_counter();Xh,Yh=sol['simulate'](X0[:36],Y0[:36],cfg['B'],cfg['horizon'],cfg['dt']/2,cfg['L'])
        half_seconds=time.perf_counter()-ht
        np.savez_compressed(OUT/'fixed36_halfstep.npz',Xf=Xh,Yf=Yh,future=sol['flatten_dev'](Xh,Yh,cfg['B']),sample_id=np.arange(36))
        latents={};xx=((X-sc['input_mean'])/sc['input_std']).astype(np.float32)
        for seed,net in encoders.items():
            with torch.inference_mode():latents[f'latent_{seed}']=net.encode(torch.from_numpy(xx)).numpy().copy()
        assert all(np.isfinite(a).all() for a in [Xf,Yf,Xh,Yh,*latents.values()])
        np.savez_compressed(OUT/'frozen_latents.npz',**latents)
        dump(OUT/'generation_complete.json',{'at':now(),'samples_completed':N,'main_steps_per_state':4000,'halfstep_samples':36,
              'main_generation_integration_seconds':main_seconds,'halfstep_integration_seconds':half_seconds,
              'no_method_errors_viewed_yet':True,'data_seed':GEN_SEED,
              'frozen_sha256':{p.name:sha(p) for p in [OUT/n for n in ['physical_initials.npz','numerical_futures.npz','fixed36_halfstep.npz','frozen_latents.npz']]}})
    finish('generate',pr,t,stop,{'samples_completed':N,'halfstep_samples':36})
def evaluate():
    pr,t,stop=stage_start('evaluate')
    assert (OUT/'generate_receipt.json').exists()
    from threadpoolctl import threadpool_limits
    with threadpool_limits(limits=1):
        seal=load(OUT/'selection_sealed.json');gen=load(OUT/'generation_complete.json')
        for p,h in seal['frozen_files_sha256'].items():assert sha(p)==h
        for n,h in gen['frozen_sha256'].items():assert sha(OUT/n)==h
        ini=npz(OUT/'physical_initials.npz');fut=npz(OUT/'numerical_futures.npz');Y=fut['future'].astype(np.float64)
        zz=npz(OUT/'frozen_latents.npz');fs,_=make_features(ini['current'].astype(np.float64),{s:zz[f'latent_{s}'] for s in SEEDS})
        old=npz(OLD/'selected_readouts.npz');new=npz(OUT/'selected18_readouts.npz')
        pred={};res={};loss={};absolute={};aggregates={};comparisons={};allarrays={}
        for group in ['UNRESTRICTED','POD4','POD14']:
            bank=old if group=='UNRESTRICTED' else new
            basis=None if group=='UNRESTRICTED' else new[group+'_basis']
            for name,F in fs.items():
                reader=name if basis is None else group+'__'+name;key=group+'__'+name
                pp=predict(bank,reader,F,basis);assert np.isfinite(pp).all();pred[key]=pp
                m,rel,ab=metrics(Y,pp);res[key]=m;loss[key]=rel;absolute[key]=ab
                w=bank[reader+'__coef_scaled_target'];mapcoef=w[:-1] if basis is None else w[:-1]@basis.T
                res[key]['numerical_coefficient_map_rank']=numerical_rank(mapcoef)
                res[key]['feature_columns']=F.shape[1];res[key]['reader_coefficient_count']=int(w.size)
                if basis is not None:assert res[key]['numerical_coefficient_map_rank']['rank']<=int(group[3:])
            for family in ['PRED_AFFINE','PRED_QUADRATIC']:
                keys=[group+'__'+family+'_'+str(s) for s in SEEDS];key=group+'__'+family
                loss[key]=np.mean([loss[k] for k in keys],axis=0);absolute[key]=np.mean([absolute[k] for k in keys],axis=0)
                aggregates[key]={'relative_L2':summary(loss[key]),'absolute_L2':summary(absolute[key]),
                    'critical_R2_mean_across_seed_models':float(np.mean([res[k]['critical_R2'] for k in keys])),
                    'uniform_R2_mean_across_seed_models':float(np.mean([res[k]['uniform_R2'] for k in keys]))}
            for label,a,b in [('PRED_affine_minus_quadratic',group+'__PRED_AFFINE',group+'__PRED_QUADRATIC'),
                              ('PRED_quadratic_minus_physical_quadratic',group+'__PRED_QUADRATIC',group+'__PHYSICAL_QUADRATIC_27')]:
                delta=loss[a]-loss[b];comparisons[group+'__'+label]={'mean_baseline_minus_candidate':float(delta.mean()),
                    'relative_mean_reduction':float(delta.mean()/loss[a].mean()),'candidate_better':int(np.sum(delta>0)),
                    'candidate_worse':int(np.sum(delta<0)),'ties':int(np.sum(delta==0)),
                    'mean_absolute_difference':float((absolute[a]-absolute[b]).mean()),'evidence':'PREPLANNED_SECONDARY_DESCRIPTIVE'}
            if basis is not None:
                pp=old['target_mean']+((Y-old['target_mean'])@basis)@basis.T
                aggregates[group+'__projection_reference']=metrics(Y,pp)[0]
        assert len(pred)==27
        delta=loss['UNRESTRICTED__PRED_AFFINE']-loss['UNRESTRICTED__PRED_QUADRATIC']
        boot=np.empty(10000);rng=np.random.default_rng(BOOT_SEED)
        for first in range(0,10000,200):
            ix=rng.integers(0,N,size=(min(200,10000-first),N));boot[first:first+len(ix)]=delta[ix].mean(1)
        ci=np.quantile(boot,[.025,.975],method='linear')
        primary=comparisons['UNRESTRICTED__PRED_affine_minus_quadratic'].copy()
        primary.update({'evidence':'SINGLE_PRESPECIFIED_PROSPECTIVE_COMPARISON','approximate95_percentile_interval':ci.tolist(),
             'status':'SUPPORTED_ON_NEW_SAME_DISTRIBUTION_STATES' if ci[0]>0 else 'NOT_CONFIRMED','n_independent_initial_states':N,
             'bootstrap_replicates':10000,'bootstrap_seed':BOOT_SEED,'p_value':None})
        half=npz(OUT/'fixed36_halfstep.npz');Yh=half['future'].astype(np.float64);Y64=np.c_[fut['Xf']-1,fut['Yf']-25/16]
        Yh64=np.c_[half['Xf']-1,half['Yf']-25/16];endpoint=np.linalg.norm(Yh64-Y64[:36],axis=1)/(np.linalg.norm(Yh64,axis=1)+1e-10)
        def primary_against(target):
            ls={family:np.mean([np.linalg.norm(pred[f'UNRESTRICTED__{family}_{s}'][:len(target)]-target,axis=1)/(np.linalg.norm(target,axis=1)+1e-10) for s in SEEDS],axis=0) for family in ['PRED_AFFINE','PRED_QUADRATIC']}
            return ls['PRED_AFFINE']-ls['PRED_QUADRATIC']
        halfdelta=primary_against(Yh);base36=delta[:36];float64delta=primary_against(Y64)
        sensitivity={'fixed_first36_endpoint_relative_change':summary(endpoint),'mean_primary_delta_dt_36':float(base36.mean()),
              'mean_primary_delta_halfstep36':float(halfdelta.mean()),'primary_difference_change_36':float((halfdelta-base36).mean()),
              'dt_and_halfstep_mean_directions_same':bool(base36.mean()*halfdelta.mean()>0),
              'primary_delta_using_float64_deviations_all720':float(float64delta.mean()),
              'float32_target_primary_change':float(float64delta.mean()-delta.mean()),
              'scope':'numerical sensitivity only, not trueODE or continuousPDE certification'}
        seedresults={str(s):{g:{'affine':float(loss[f'{g}__PRED_AFFINE_{s}'].mean()),'quadratic':float(loss[f'{g}__PRED_QUADRATIC_{s}'].mean()),
                 'difference':float((loss[f'{g}__PRED_AFFINE_{s}']-loss[f'{g}__PRED_QUADRATIC_{s}']).mean())} for g in ['UNRESTRICTED','POD4','POD14']} for s in SEEDS}
        np.savez_compressed(OUT/'all27_predictions.npz',**pred)
        np.savez_compressed(OUT/'point_losses.npz',**{k+'__relative':v for k,v in loss.items()},**{k+'__absolute':v for k,v in absolute.items()})
        np.savez_compressed(OUT/'primary_bootstrap.npz',per_state_delta=delta,bootstrap_means=boot)
        result={'at':now(),'primary':primary,'comparisons':comparisons,'aggregates':aggregates,'reader_metrics':res,'seed_results':seedresults,
                'sensitivity':sensitivity,'new_fits':90,'selected_new_readers':18,'evaluated_pipelines':27,
                'E3':pr['E3'],'limitations':['fixed3 pre-existing encoders','same controlled initial-state distribution','approximate bootstrap not exact coverage',
                'common output space not equal parameter/prior/compute budget','secondary controls descriptive','no new trueODE certificate','no generalization beyond this benchmark']}
        dump(OUT/'result.json',result);dump(HERE/'result_summary.json',{k:v for k,v in result.items() if k!='reader_metrics'})
    finish('evaluate',pr,t,stop,{'primary_status':primary['status']})
def replay():
    # Independent evaluation expressions: no fitting, encoder forward, generator or ODE.
    pr,t,stop=stage_start('replay')
    from scipy.fft import dct
    from threadpoolctl import threadpool_limits
    with threadpool_limits(limits=1):
        ini=npz(OUT/'physical_initials.npz');fut=npz(OUT/'numerical_futures.npz');Y=fut['future'].astype(np.float64)
        X=ini['current'].astype(np.float64);lat=npz(OUT/'frozen_latents.npz');old=npz(OLD/'selected_readouts.npz');new=npz(OUT/'selected18_readouts.npz')
        saved=npz(OUT/'all27_predictions.npz');L=npz(OUT/'point_losses.npz');result=load(OUT/'result.json')
        cx=dct(X[:,:64],type=1,axis=1);cy=dct(X[:,64:],type=1,axis=1)
        a=16/75*(5*cx[:,1]/63+cy[:,1]/63)
        phy=np.c_[a,cx[:,1]/63-a,cx[:,0]/126,cy[:,0]/126,cx[:,2]/63,cy[:,2]/63]
        maxpred=maxloss=maxsummary=0.;indloss={};checks=[]
        for key,p in saved.items():
            group,name=key.split('__',1);bank=old if group=='UNRESTRICTED' else new;reader=name if group=='UNRESTRICTED' else key
            F=lat['latent_'+name.rsplit('_',1)[1]].astype(np.float64) if name.startswith('PRED') else phy.copy() if name.startswith('PHYSICAL') else X.copy()
            if 'QUADRATIC' in name:
                pairs=bank[reader+'__quadratic_pairs'];F=np.column_stack([F]+[F[:,i]*F[:,j] for i,j in pairs])
            W=bank[reader+'__coef_scaled_target'];W=W if group=='UNRESTRICTED' else W@bank[group+'_basis'].T
            mu=bank[reader+'__feature_mean'];sd=bank[reader+'__feature_std'];rms=float(bank['target_global_rms'][0])
            raw=W[:-1]*rms/sd[:,None];intercept=W[-1]*rms+bank['target_mean']-mu@raw
            pp=F@raw+intercept;pd=float(np.max(np.abs(pp-p)));maxpred=max(maxpred,pd);assert pd<1e-10
            rel=np.array([math.sqrt(math.fsum(float(v)*float(v) for v in (row-yy)))/(math.sqrt(math.fsum(float(v)*float(v) for v in yy))+1e-10) for row,yy in zip(pp,Y)])
            ld=float(np.max(np.abs(rel-L[key+'__relative'])));maxloss=max(maxloss,ld);assert ld<1e-10
            indloss[key]=rel;md=abs(float(rel.mean())-result['reader_metrics'][key]['relative_L2']['mean']);maxsummary=max(maxsummary,md)
            checks.append({'reader':key,'prediction_max_abs':pd,'loss_max_abs':ld})
        # Recheck training-only POD and all90 validation selections without new fitting.
        d=npz(DATA);tr,va=d['train_idx'],d['val_idx'];Ytr=d['future'][tr].astype(np.float64);Yva=d['future'][va].astype(np.float64)
        mean=Ytr.mean(0);scale=float(np.sqrt(np.mean((Ytr-mean)**2)));assert np.array_equal(mean,new['target_mean']) and scale==new['target_global_rms'][0]
        cov=((Ytr-mean)/scale).T@((Ytr-mean)/scale)
        for rank in [4,14]:
            Q=new[f'POD{rank}_basis'];assert np.max(np.abs(Q.T@Q-np.eye(rank)))<1e-12
            assert np.max(np.abs(cov@Q-Q*(new['pod_singular_values'][:rank]**2)))<1e-7
        assert np.array_equal(new['POD4_basis'],new['POD14_basis'][:,:4])
        ftr={};fva={}
        for split,ids,dest in [('train',tr,ftr),('val',va,fva)]:
            z={s:npz(REC/f'arrays/D0_PRED_AE_d4_{s}_float_artifacts.npz')['latent_'+split] for s in SEEDS}
            dest.update(make_features(d['current'][ids].astype(np.float64),z)[0])
        allfits=npz(OUT/'all90_coefficients.npz');seal=load(OUT/'selection_sealed.json')
        with (OUT/'fit_grid.csv').open(encoding='utf-8',newline='') as f:rows=list(csv.DictReader(f))
        choices={};maxval=0.;maxkkt=0.
        for row in rows:
            key=row['key'];rank=int(row['rank']);name=key.split('__',1)[1];F=ftr[name];mu=F.mean(0);sd=np.maximum(F.std(0),1e-12)
            assert np.array_equal(mu,new[key+'__feature_mean']) and np.array_equal(sd,new[key+'__feature_std'])
            W=allfits[f"fit_{int(row['fit_number']):03d}"];Q=new[f'POD{rank}_basis'];H=np.c_[(F-mu)/sd,np.ones(len(F))]
            target=(Ytr-mean)/scale@Q;residual=H@W-target;pen=W.copy();pen[-1]=0;alpha=float(row['alpha'])
            grad=H.T@residual+alpha*pen;kkt=float(np.linalg.norm(grad)/max(np.linalg.norm(H)*np.linalg.norm(residual)+alpha*np.linalg.norm(W),1e-30));assert kkt<5e-8;maxkkt=max(maxkkt,kkt)
            pp=(np.c_[(fva[name]-mu)/sd,np.ones(len(va))]@W)@Q.T*scale+mean
            vl=float(np.mean(np.linalg.norm(pp-Yva,axis=1)/(np.linalg.norm(Yva,axis=1)+1e-10)));maxval=max(maxval,abs(vl-float(row['validation_mean_relative_L2'])))
            choices.setdefault(key,[]).append((vl,-alpha,int(row['fit_number'])))
        assert len(rows)==90 and len(choices)==18 and maxval<1e-12
        for key,candidates in choices.items():
            best=min(candidates,key=lambda z:(z[0],z[1]));assert best[1]==-seal['choices'][key]['alpha']
            assert np.array_equal(new[key+'__coef_scaled_target'],allfits[f'fit_{best[2]:03d}'])
        av=np.mean([indloss[f'UNRESTRICTED__PRED_AFFINE_{s}'] for s in SEEDS],axis=0)
        qu=np.mean([indloss[f'UNRESTRICTED__PRED_QUADRATIC_{s}'] for s in SEEDS],axis=0);delta=av-qu
        bs=npz(OUT/'primary_bootstrap.npz');assert np.max(np.abs(delta-bs['per_state_delta']))<1e-10
        boot=np.empty(10000);rng=np.random.default_rng(BOOT_SEED)
        for j in range(10000):
            counts=np.bincount(rng.integers(0,N,size=N),minlength=N);boot[j]=np.dot(counts,delta)/N
        db=float(np.max(np.abs(boot-bs['bootstrap_means'])));assert db<1e-10
        ci=np.quantile(boot,[.025,.975],method='linear');assert np.max(np.abs(ci-result['primary']['approximate95_percentile_interval']))<1e-10
        oldrows={row.astype(np.float32).tobytes() for row in np.vstack([d['current'],npz(FRESH/'numerical_futures.npz')['current']])}
        newrows={row.tobytes() for row in ini['current']};assert len(newrows)==N and not(newrows&oldrows)
        chronology={n:load(OUT/n)['at'] for n in ['selection_sealed.json','generation_before_futures.json','generation_complete.json']}
        assert datetime.datetime.fromisoformat(chronology['selection_sealed.json'])<datetime.datetime.fromisoformat(chronology['generation_before_futures.json'])<datetime.datetime.fromisoformat(chronology['generation_complete.json'])
        report={'at':now(),'status':'PASS','independent_evaluation_only':True,'new_fit_calls':0,'new_ODE_calls':0,
                'n_initial_states':N,'readers_replayed':len(saved),'validation_configs_checked':len(rows),'selections_checked':len(choices),
                'max_prediction_abs_difference':maxpred,'max_per_state_loss_difference':maxloss,'max_mean_summary_difference':maxsummary,
                'max_bootstrap_difference':db,'max_validation_difference':maxval,'max_stationarity_residual':maxkkt,
                'train_POD_eigen_equations_and_nested_basis':True,'chronology':chronology,'independent_input_nonoverlap':True,
                'primary_mean_difference':float(delta.mean()),'primary_approximate95_interval':ci.tolist(),'readers':checks}
        dump(OUT/'independent_replay.json',report);dump(HERE/'independent_replay.json',report)
    finish('replay',pr,t,stop,{'readers_replayed':27,'validation_configs_checked':90})
if __name__=='__main__':
    stage=sys.argv[1]
    try:
        import numpy as np
        if stage=='prepare':prepare()
        elif stage=='fit':fit()
        elif stage=='generate':generate()
        elif stage=='evaluate':evaluate()
        elif stage=='replay':replay()
        else:raise ValueError(stage)
    except BaseException:
        if OUT.is_dir():dump(OUT/f'{stage}_failed.json',{'at':now(),'traceback':traceback.format_exc()})
        raise
