Download src/uvw/eval/analysis.py from javadtaghia/unlearning-via-weight-code: direct link, hf CLI and curl.
- Browser
- Download file 4.39 kB
-
https://huggingface.co/javadtaghia/unlearning-via-weight-code/resolve/main/src/uvw/eval/analysis.py
- Command line
-
hf download hf://javadtaghia/unlearning-via-weight-code/src/uvw/eval/analysis.py
-
curl -L -o analysis.py https://huggingface.co/javadtaghia/unlearning-via-weight-code/resolve/main/src/uvw/eval/analysis.py
4.39 kB
| """Reproduce paired held-out tables from score files, without reading test prompts.""" | |
| from __future__ import annotations | |
| import argparse | |
| import json | |
| from pathlib import Path | |
| import pandas as pd | |
| from uvw.stats import cluster_bootstrap_ci, paired_cluster_bootstrap_diff | |
| METRICS = { | |
| 'target rate': ('vangogh-test', 'target', 'target_hit'), | |
| 'neighbour correct': ('vangogh-test', 'neighbour', 'neighbour_correct'), | |
| 'neighbour CLIP': ('vangogh-test', 'neighbour', 'clip_score'), | |
| 'indirect target rate': ('vangogh-probes', 'target', 'target_hit'), | |
| 'COCO CLIP': ('coco-test', None, 'clip_score'), | |
| } | |
| def paired(a, b, metric): | |
| keys = ['prompt_id', 'seed'] | |
| if a.duplicated(keys).any() or b.duplicated(keys).any(): | |
| raise ValueError('duplicate prompt/seed') | |
| if metric == 'neighbour_correct': | |
| a, b = a[a[metric].notna()], b[b[metric].notna()] | |
| pair = a.merge(b, on=keys, suffixes=('_a', '_b'), how='outer', validate='one_to_one', indicator=True) | |
| if not pair['_merge'].eq('both').all(): | |
| raise ValueError('unpaired prompts/seeds') | |
| if not pair.group_id_a.eq(pair.group_id_b).all(): | |
| raise ValueError('mismatched clusters') | |
| if pair[[metric+'_a', metric+'_b']].isna().any().any(): | |
| raise ValueError('missing metric') | |
| return paired_cluster_bootstrap_diff(pair[metric+'_a'].astype(float), pair[metric+'_b'].astype(float), pair.group_id_a) | |
| def analyse(root, checkpoints): | |
| frames, absolute, differences = {}, {}, {} | |
| for name, checkpoint in checkpoints.items(): | |
| frames[name], absolute[name] = {}, {} | |
| for label, (suite, role, metric) in METRICS.items(): | |
| df = pd.read_json(root / checkpoint / suite / 'scores.jsonl', lines=True) | |
| if role: | |
| df = df[df.role.eq(role)] | |
| if metric == 'neighbour_correct': | |
| # None by design for neighbours without an artist label (e.g. famous non-Van Gogh works) | |
| df = df[df[metric].notna()] | |
| if df.empty or df[metric].isna().any(): | |
| raise ValueError(f'{name}: missing {label}') | |
| frames[name][label] = df | |
| absolute[name][label] = dict(zip(['mean', 'lo', 'hi'], cluster_bootstrap_ci(df[metric].astype(float), df.group_id))) | {'n': len(df), 'clusters': df.group_id.nunique()} | |
| comparisons = [(c, 'C1') for c in checkpoints if c != 'C1'] | |
| comparisons += [('C6', c) for c in checkpoints if c not in ('C1', 'C6')] | |
| for a, b in comparisons: | |
| differences[f'{a} − {b}'] = {} | |
| for label, (_, _, metric) in METRICS.items(): | |
| differences[f'{a} − {b}'][label] = dict(zip(['mean', 'lo', 'hi', 'p'], paired(frames[a][label], frames[b][label], metric))) | |
| return {'checkpoints': checkpoints, 'absolute': absolute, 'differences': differences} | |
| def markdown(result): | |
| labels = list(METRICS) | |
| lines = ['All intervals are 95% percentile CIs from 10,000 template-group bootstrap resamples. Differences pair exactly by `prompt_id` + `seed`; positive target-rate differences mean more target detections. CLIP here is neighbour CLIP, matching the selection utility metric. These pilot r1 comparisons are exploratory; no six-cell Holm-adjusted confirmatory claim is made.', '', '| condition | ' + ' | '.join(labels) + ' |', '|---|' + '---|'*len(labels)] | |
| def cell(v): | |
| return f'{v["mean"]:.4f} [{v["lo"]:.4f}, {v["hi"]:.4f}]' | |
| for name, values in result['absolute'].items(): | |
| lines.append('| '+name+' | '+' | '.join(cell(values[l]) for l in labels)+' |') | |
| lines += ['', '| paired difference | '+' | '.join(labels)+' |', '|---|'+'---|'*len(labels)] | |
| for name, values in result['differences'].items(): | |
| lines.append('| '+name+' | '+' | '.join(cell(values[l]) for l in labels)+' |') | |
| lines += ['', 'Checkpoint paths:', ''] + [f'- {k}: `{v}`' for k,v in result['checkpoints'].items()] | |
| return '\n'.join(lines)+'\n' | |
| def main(): | |
| p=argparse.ArgumentParser() | |
| p.add_argument('root',type=Path) | |
| p.add_argument('mapping',type=Path,help='JSON condition-to-checkpoint path mapping') | |
| p.add_argument('--out',type=Path,required=True) | |
| a=p.parse_args() | |
| r=analyse(a.root,json.loads(a.mapping.read_text())) | |
| a.out.with_suffix('.json').write_text(json.dumps(r,indent=2,default=int)) | |
| a.out.with_suffix('.md').write_text(markdown(r)) | |
| if __name__ == '__main__': | |
| main() | |