"""Reproduce paired held-out tables from score files, without reading test prompts.""" from __future__ import annotations import argparse import json from pathlib import Path import pandas as pd from uvw.stats import cluster_bootstrap_ci, paired_cluster_bootstrap_diff METRICS = { 'target rate': ('vangogh-test', 'target', 'target_hit'), 'neighbour correct': ('vangogh-test', 'neighbour', 'neighbour_correct'), 'neighbour CLIP': ('vangogh-test', 'neighbour', 'clip_score'), 'indirect target rate': ('vangogh-probes', 'target', 'target_hit'), 'COCO CLIP': ('coco-test', None, 'clip_score'), } def paired(a, b, metric): keys = ['prompt_id', 'seed'] if a.duplicated(keys).any() or b.duplicated(keys).any(): raise ValueError('duplicate prompt/seed') if metric == 'neighbour_correct': a, b = a[a[metric].notna()], b[b[metric].notna()] pair = a.merge(b, on=keys, suffixes=('_a', '_b'), how='outer', validate='one_to_one', indicator=True) if not pair['_merge'].eq('both').all(): raise ValueError('unpaired prompts/seeds') if not pair.group_id_a.eq(pair.group_id_b).all(): raise ValueError('mismatched clusters') if pair[[metric+'_a', metric+'_b']].isna().any().any(): raise ValueError('missing metric') return paired_cluster_bootstrap_diff(pair[metric+'_a'].astype(float), pair[metric+'_b'].astype(float), pair.group_id_a) def analyse(root, checkpoints): frames, absolute, differences = {}, {}, {} for name, checkpoint in checkpoints.items(): frames[name], absolute[name] = {}, {} for label, (suite, role, metric) in METRICS.items(): df = pd.read_json(root / checkpoint / suite / 'scores.jsonl', lines=True) if role: df = df[df.role.eq(role)] if metric == 'neighbour_correct': # None by design for neighbours without an artist label (e.g. famous non-Van Gogh works) df = df[df[metric].notna()] if df.empty or df[metric].isna().any(): raise ValueError(f'{name}: missing {label}') frames[name][label] = df absolute[name][label] = dict(zip(['mean', 'lo', 'hi'], cluster_bootstrap_ci(df[metric].astype(float), df.group_id))) | {'n': len(df), 'clusters': df.group_id.nunique()} comparisons = [(c, 'C1') for c in checkpoints if c != 'C1'] comparisons += [('C6', c) for c in checkpoints if c not in ('C1', 'C6')] for a, b in comparisons: differences[f'{a} − {b}'] = {} for label, (_, _, metric) in METRICS.items(): differences[f'{a} − {b}'][label] = dict(zip(['mean', 'lo', 'hi', 'p'], paired(frames[a][label], frames[b][label], metric))) return {'checkpoints': checkpoints, 'absolute': absolute, 'differences': differences} def markdown(result): labels = list(METRICS) lines = ['All intervals are 95% percentile CIs from 10,000 template-group bootstrap resamples. Differences pair exactly by `prompt_id` + `seed`; positive target-rate differences mean more target detections. CLIP here is neighbour CLIP, matching the selection utility metric. These pilot r1 comparisons are exploratory; no six-cell Holm-adjusted confirmatory claim is made.', '', '| condition | ' + ' | '.join(labels) + ' |', '|---|' + '---|'*len(labels)] def cell(v): return f'{v["mean"]:.4f} [{v["lo"]:.4f}, {v["hi"]:.4f}]' for name, values in result['absolute'].items(): lines.append('| '+name+' | '+' | '.join(cell(values[l]) for l in labels)+' |') lines += ['', '| paired difference | '+' | '.join(labels)+' |', '|---|'+'---|'*len(labels)] for name, values in result['differences'].items(): lines.append('| '+name+' | '+' | '.join(cell(values[l]) for l in labels)+' |') lines += ['', 'Checkpoint paths:', ''] + [f'- {k}: `{v}`' for k,v in result['checkpoints'].items()] return '\n'.join(lines)+'\n' def main(): p=argparse.ArgumentParser() p.add_argument('root',type=Path) p.add_argument('mapping',type=Path,help='JSON condition-to-checkpoint path mapping') p.add_argument('--out',type=Path,required=True) a=p.parse_args() r=analyse(a.root,json.loads(a.mapping.read_text())) a.out.with_suffix('.json').write_text(json.dumps(r,indent=2,default=int)) a.out.with_suffix('.md').write_text(markdown(r)) if __name__ == '__main__': main()