javadtaghia's picture
Code release: unlearning-via-weight (history squashed before publication)
788274d
Raw History Blame Contribute Delete
4.39 kB
"""Reproduce paired held-out tables from score files, without reading test prompts."""
from __future__ import annotations
import argparse
import json
from pathlib import Path
import pandas as pd
from uvw.stats import cluster_bootstrap_ci, paired_cluster_bootstrap_diff
METRICS = {
'target rate': ('vangogh-test', 'target', 'target_hit'),
'neighbour correct': ('vangogh-test', 'neighbour', 'neighbour_correct'),
'neighbour CLIP': ('vangogh-test', 'neighbour', 'clip_score'),
'indirect target rate': ('vangogh-probes', 'target', 'target_hit'),
'COCO CLIP': ('coco-test', None, 'clip_score'),
}
def paired(a, b, metric):
keys = ['prompt_id', 'seed']
if a.duplicated(keys).any() or b.duplicated(keys).any():
raise ValueError('duplicate prompt/seed')
if metric == 'neighbour_correct':
a, b = a[a[metric].notna()], b[b[metric].notna()]
pair = a.merge(b, on=keys, suffixes=('_a', '_b'), how='outer', validate='one_to_one', indicator=True)
if not pair['_merge'].eq('both').all():
raise ValueError('unpaired prompts/seeds')
if not pair.group_id_a.eq(pair.group_id_b).all():
raise ValueError('mismatched clusters')
if pair[[metric+'_a', metric+'_b']].isna().any().any():
raise ValueError('missing metric')
return paired_cluster_bootstrap_diff(pair[metric+'_a'].astype(float), pair[metric+'_b'].astype(float), pair.group_id_a)
def analyse(root, checkpoints):
frames, absolute, differences = {}, {}, {}
for name, checkpoint in checkpoints.items():
frames[name], absolute[name] = {}, {}
for label, (suite, role, metric) in METRICS.items():
df = pd.read_json(root / checkpoint / suite / 'scores.jsonl', lines=True)
if role:
df = df[df.role.eq(role)]
if metric == 'neighbour_correct':
# None by design for neighbours without an artist label (e.g. famous non-Van Gogh works)
df = df[df[metric].notna()]
if df.empty or df[metric].isna().any():
raise ValueError(f'{name}: missing {label}')
frames[name][label] = df
absolute[name][label] = dict(zip(['mean', 'lo', 'hi'], cluster_bootstrap_ci(df[metric].astype(float), df.group_id))) | {'n': len(df), 'clusters': df.group_id.nunique()}
comparisons = [(c, 'C1') for c in checkpoints if c != 'C1']
comparisons += [('C6', c) for c in checkpoints if c not in ('C1', 'C6')]
for a, b in comparisons:
differences[f'{a} − {b}'] = {}
for label, (_, _, metric) in METRICS.items():
differences[f'{a} − {b}'][label] = dict(zip(['mean', 'lo', 'hi', 'p'], paired(frames[a][label], frames[b][label], metric)))
return {'checkpoints': checkpoints, 'absolute': absolute, 'differences': differences}
def markdown(result):
labels = list(METRICS)
lines = ['All intervals are 95% percentile CIs from 10,000 template-group bootstrap resamples. Differences pair exactly by `prompt_id` + `seed`; positive target-rate differences mean more target detections. CLIP here is neighbour CLIP, matching the selection utility metric. These pilot r1 comparisons are exploratory; no six-cell Holm-adjusted confirmatory claim is made.', '', '| condition | ' + ' | '.join(labels) + ' |', '|---|' + '---|'*len(labels)]
def cell(v):
return f'{v["mean"]:.4f} [{v["lo"]:.4f}, {v["hi"]:.4f}]'
for name, values in result['absolute'].items():
lines.append('| '+name+' | '+' | '.join(cell(values[l]) for l in labels)+' |')
lines += ['', '| paired difference | '+' | '.join(labels)+' |', '|---|'+'---|'*len(labels)]
for name, values in result['differences'].items():
lines.append('| '+name+' | '+' | '.join(cell(values[l]) for l in labels)+' |')
lines += ['', 'Checkpoint paths:', ''] + [f'- {k}: `{v}`' for k,v in result['checkpoints'].items()]
return '\n'.join(lines)+'\n'
def main():
p=argparse.ArgumentParser()
p.add_argument('root',type=Path)
p.add_argument('mapping',type=Path,help='JSON condition-to-checkpoint path mapping')
p.add_argument('--out',type=Path,required=True)
a=p.parse_args()
r=analyse(a.root,json.loads(a.mapping.read_text()))
a.out.with_suffix('.json').write_text(json.dumps(r,indent=2,default=int))
a.out.with_suffix('.md').write_text(markdown(r))
if __name__ == '__main__':
main()