Download tools/compare_runs.py from murabcd/fev-4b: direct link, hf CLI and curl.
- Browser
- Download file 3.93 kB
-
https://huggingface.co/murabcd/fev-4b/resolve/main/tools/compare_runs.py
- Command line
-
hf download hf://murabcd/fev-4b/tools/compare_runs.py
-
curl -L -o compare_runs.py https://huggingface.co/murabcd/fev-4b/resolve/main/tools/compare_runs.py
3.93 kB
| """Summarize saved native typed runs without requerying or tuning a model.""" | |
| import argparse | |
| import json | |
| from pathlib import Path | |
| from fev4b.metrics import summarize | |
| def compare(runs): | |
| indexed = {} | |
| for name, rows in runs.items(): | |
| values = {(r['id'], r['question']): r for r in rows} | |
| if len(values) != len(rows): | |
| raise ValueError('Duplicate question in saved run') | |
| indexed[name] = values | |
| keys = next(iter(indexed.values())).keys() | |
| if any(values.keys() != keys for values in indexed.values()): | |
| raise ValueError('Runs do not contain the same questions') | |
| for key in keys: | |
| labels = {(v[key]['target'], v[key]['type'], v[key]['source']) | |
| for v in indexed.values() if v[key]['error'] is None} | |
| if len(labels) > 1: | |
| raise ValueError('Runs disagree on target/type/source metadata') | |
| results = {} | |
| for name, rows in runs.items(): | |
| subsets = {'all': rows, 'publicTransfer': [r for r in rows if r['source'] != 'authored-russian-typed'], | |
| 'authoredRussian': [r for r in rows if r['source'] == 'authored-russian-typed']} | |
| results[name] = {} | |
| for label, values in subsets.items(): | |
| summary = summarize(values) | |
| ordinal = [r['ordinalAbsoluteError'] for r in values | |
| if r['error'] is None and 'ordinalAbsoluteError' in r] | |
| summary['ordinalMeanAbsoluteError'] = sum(ordinal)/len(ordinal) if ordinal else None | |
| results[name][label] = summary | |
| results[name]['byType'] = {t:summarize([r for r in rows if r['type']==t]) for t in sorted({r['type'] for r in rows})} | |
| results[name]['bySource'] = {s:summarize([r for r in rows if r['source']==s]) for s in sorted({r['source'] for r in rows})} | |
| paired = {} | |
| names = list(runs) | |
| for i, a in enumerate(names): | |
| for b in names[i+1:]: | |
| av, bv = indexed[a], indexed[b] | |
| paired[a+' vs '+b] = { | |
| 'aOnlyCorrect': sum(bool(av[k].get('correct')) and not bv[k].get('correct') for k in keys), | |
| 'bOnlyCorrect': sum(bool(bv[k].get('correct')) and not av[k].get('correct') for k in keys), | |
| 'bothCorrect': sum(bool(av[k].get('correct')) and bool(bv[k].get('correct')) for k in keys), | |
| 'neitherCorrect': sum(not av[k].get('correct') and not bv[k].get('correct') for k in keys)} | |
| return {'models': results, 'pairedOutcomes': paired, | |
| 'interpretation': 'Descriptive comparison on fixed public development and repetitive authored inputs; no significance or unseen customer accuracy claim'} | |
| def main(): | |
| parser = argparse.ArgumentParser(description=__doc__) | |
| parser.add_argument('--run', action='append', required=True, help='NAME=run-directory; repeat for each system') | |
| parser.add_argument('--out', type=Path, required=True) | |
| args = parser.parse_args() | |
| if args.out.exists(): | |
| raise ValueError('Output already exists') | |
| runs = {} | |
| provenance = {} | |
| hashes = set() | |
| for spec in args.run: | |
| name, directory = spec.split('=', 1) | |
| if name in runs: | |
| raise ValueError('Duplicate system name') | |
| directory = Path(directory) | |
| report = json.loads((directory/'report.json').read_text()) | |
| p = report['provenance'] | |
| digest = p.get('dataSha256', p.get('datasetSha256')) | |
| if not digest: | |
| raise ValueError('Missing input data provenance') | |
| hashes.add(digest) | |
| runs[name] = json.loads((directory/'rows.json').read_text()) | |
| provenance[name] = p | |
| if len(hashes) != 1: | |
| raise ValueError('Runs use different datasets') | |
| result = {**compare(runs), 'datasetSha256': hashes.pop(), 'provenance': provenance} | |
| args.out.parent.mkdir(parents=True, exist_ok=True) | |
| args.out.write_text(json.dumps(result, indent=2)+'\n') | |
| if __name__ == '__main__': | |
| main() | |