| """Validation-only inventory; no EEG, checkpoint loading or evaluation calls.""" |
| import json,datetime,collections,csv,hashlib |
| from pathlib import Path |
| import numpy as np |
| from monitor_benchmarks import snapshot |
| s=snapshot();groups=collections.defaultdict(list);detail=[] |
| for p in sorted(Path('experiments').glob('strict_*.json')): |
| try:r=json.loads(p.read_text()) |
| except json.JSONDecodeError:continue |
| if 'metrics' not in r:continue |
| subjects=r.get('subjects',[r.get('subject')]);scope='pooled' if len(subjects)>1 else 'subject_specific' |
| k=(r['model'],r.get('montage','unknown'),r['channels'],r.get('seed','deterministic'),scope) |
| groups[k].append(r) |
| history=r.get('history',[]);valid=[h for h in history if 'balanced_accuracy' in h] |
| best=max(valid,key=lambda h:h['balanced_accuracy']) if valid else None |
| detail.append({'file':str(p),'model':k[0],'channels':k[2],'scope':scope,'subject':r.get('subject'),'validation_balanced_accuracy':r['metrics']['balanced_accuracy'],'macro_f1':r['metrics']['macro_f1'],'best_epoch':best.get('epoch') if best else None,'history_available':bool(valid)}) |
| rows=[] |
| for k,rs in sorted(groups.items(),key=lambda q:str(q[0])): |
| vals=[r['metrics']['balanced_accuracy'] for r in rs];subjects={r.get('subject',r.get('subjects',[None])[0]) for r in rs} if k[4]=='subject_specific' else set(rs[0]['subjects']) |
| ci=np.quantile(np.random.default_rng(2026).choice(vals,(10000,len(vals)),replace=True).mean(1),[.025,.975]).tolist() if k[4]=='subject_specific' and len(subjects)==15 else None |
| rows.append(dict(model=k[0],montage=k[1],channels=k[2],seed=k[3],scope=k[4],participants=len(subjects),balanced_accuracy=float(np.mean(vals)),macro_f1=float(np.mean([r['metrics']['macro_f1'] for r in rs])),subject_bootstrap_ci95=ci)) |
| watch=json.load(open('reports/active_run_source_watch.json'));changes=[p for p,h in watch['sha256'].items() if hashlib.sha256(Path(p).read_bytes()).hexdigest()!=h] |
| report={'utc':s['utc'],'strict_families':rows,'individual_completed_experiments':detail,'reference_completed':s['reference_validation'],'reference_active':s['active_reference_jobs'],'source_changes_since_monitor_baseline':changes,'intentional_change':'scripts/evaluate.py future freeze/test entry guards enforce latest user seal; no active training/runner files changed','test_sealed':Path('experiments/coordination/FINAL_TEST_SEALED_BY_USER').exists(),'frozen_selection_present':Path('reports/frozen_selection.json').exists(),'test_raw_caches_present':bool(list(Path('preprocessing').glob('*_test_raw.npz')))} |
| assert set(changes)<={'scripts/evaluate.py'},changes |
| Path('reports/audit/validation_inventory.json').write_text(json.dumps(report,indent=2)) |
| text=f'# SETU model comparison\n\nSnapshot {s["utc"]}. All numbers below are measured validation metrics, not final-test results. Chance balanced accuracy is 20%. Official test remains sealed.\n\n## Pooled reference window (not imagination-only)\n\n| Model | Channels | Best validation BA | Macro-F1 | Best epoch (zero-based) | State |\n|---|---:|---:|---:|---:|---|\n' |
| for r in s['reference_validation']:text+=f'| {r["model"]} | {r["channels"]} | {r["balanced_accuracy"]:.2%} | {r["macro_f1"]:.4f} | {r["best_epoch_zero_based"]} | completed |\n' |
| for r in s['active_reference_jobs']:text+=f'| {r["job"]} | 64 | {r["best_validation_balanced_accuracy_so_far"]:.2%} | pending | pending | active; provisional maximum |\n' |
| text+='\n## Imagination-only validation\n\nSubject-specific rows average one separately trained model per participant. Pooled rows are separately labelled. Incomplete families must not be ranked against 15-participant families. Confidence intervals resample participants; they do not account for adaptive validation model selection or establish cue-group independence. Macro-F1 is the mean participant value for subject-specific families.\n\n| Model | Montage | Channels | Scope | Participants | BA | Macro-F1 | 95% participant CI |\n|---|---|---:|---|---:|---:|---:|---|\n' |
| for r in rows: |
| ci=r['subject_bootstrap_ci95'];ci=f'{ci[0]:.2%}–{ci[1]:.2%}' if ci else 'not estimated' |
| text+=f'| {r["model"]} | {r["montage"]} | {r["channels"]} | {r["scope"]} | {r["participants"]}/15 | {r["balanced_accuracy"]:.2%} | {r["macro_f1"]:.4f} | {ci} |\n' |
| text+='\nPer-experiment saved metrics/best-epoch metadata: [validation inventory](reports/audit/validation_inventory.json). Pending REVE/CSBrain/CBraMod 64/32/16/8 mode matrix: [live status](reports/BENCHMARK_STATUS.md). Missing results are pending, never zero accuracy.\n\nPublished reference BA: REVE 56.35%, CSBrain 60.04%, CBraMod 53.73%. These are not reproduced SETU scores. Reference-window and strict-window scores cannot be used in a common winner ranking. No new-person accuracy is inferred from pooled or within-person models.\n\nRegenerate after queue completion: `.venv/bin/python scripts/accuracy_report.py`. This reads validation logs/results only and cannot launch training or open final-test data.\n' |
| loso=Path('reports/audit/loso_target_validation.json') |
| if loso.exists(): |
| r=json.load(open(loso));ci=r['subject_bootstrap_ci95'];text+=f'\n## Excluded-participant validation audit\n\nExisting 64-channel bandpower LOSO checkpoints: **{r["metrics"]["balanced_accuracy"]:.2%} BA**, macro-F1 {r["metrics"]["macro_f1"]:.4f}; participant-bootstrap 95% interval {ci[0]:.2%}–{ci[1]:.2%}. No target participant data used in fitting or C selection; official validation only, not final test. This near-chance result is separate from within-person and foundation-model performance.\n' |
| Path('MODEL_COMPARISON.md').write_text(text) |
| print(json.dumps({'utc':s['utc'],'strict_families':len(rows),'completed_experiments':len(detail),'active':s['active_reference_jobs'],'source_changes':changes})) |
|
|
