fev-4b / tools /compare_runs.py
murabcd's picture
feat: add compiled serving shared training and benchmarks
5adf069 verified
Raw History Blame Contribute Delete
3.93 kB
"""Summarize saved native typed runs without requerying or tuning a model."""
import argparse
import json
from pathlib import Path
from fev4b.metrics import summarize
def compare(runs):
indexed = {}
for name, rows in runs.items():
values = {(r['id'], r['question']): r for r in rows}
if len(values) != len(rows):
raise ValueError('Duplicate question in saved run')
indexed[name] = values
keys = next(iter(indexed.values())).keys()
if any(values.keys() != keys for values in indexed.values()):
raise ValueError('Runs do not contain the same questions')
for key in keys:
labels = {(v[key]['target'], v[key]['type'], v[key]['source'])
for v in indexed.values() if v[key]['error'] is None}
if len(labels) > 1:
raise ValueError('Runs disagree on target/type/source metadata')
results = {}
for name, rows in runs.items():
subsets = {'all': rows, 'publicTransfer': [r for r in rows if r['source'] != 'authored-russian-typed'],
'authoredRussian': [r for r in rows if r['source'] == 'authored-russian-typed']}
results[name] = {}
for label, values in subsets.items():
summary = summarize(values)
ordinal = [r['ordinalAbsoluteError'] for r in values
if r['error'] is None and 'ordinalAbsoluteError' in r]
summary['ordinalMeanAbsoluteError'] = sum(ordinal)/len(ordinal) if ordinal else None
results[name][label] = summary
results[name]['byType'] = {t:summarize([r for r in rows if r['type']==t]) for t in sorted({r['type'] for r in rows})}
results[name]['bySource'] = {s:summarize([r for r in rows if r['source']==s]) for s in sorted({r['source'] for r in rows})}
paired = {}
names = list(runs)
for i, a in enumerate(names):
for b in names[i+1:]:
av, bv = indexed[a], indexed[b]
paired[a+' vs '+b] = {
'aOnlyCorrect': sum(bool(av[k].get('correct')) and not bv[k].get('correct') for k in keys),
'bOnlyCorrect': sum(bool(bv[k].get('correct')) and not av[k].get('correct') for k in keys),
'bothCorrect': sum(bool(av[k].get('correct')) and bool(bv[k].get('correct')) for k in keys),
'neitherCorrect': sum(not av[k].get('correct') and not bv[k].get('correct') for k in keys)}
return {'models': results, 'pairedOutcomes': paired,
'interpretation': 'Descriptive comparison on fixed public development and repetitive authored inputs; no significance or unseen customer accuracy claim'}
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--run', action='append', required=True, help='NAME=run-directory; repeat for each system')
parser.add_argument('--out', type=Path, required=True)
args = parser.parse_args()
if args.out.exists():
raise ValueError('Output already exists')
runs = {}
provenance = {}
hashes = set()
for spec in args.run:
name, directory = spec.split('=', 1)
if name in runs:
raise ValueError('Duplicate system name')
directory = Path(directory)
report = json.loads((directory/'report.json').read_text())
p = report['provenance']
digest = p.get('dataSha256', p.get('datasetSha256'))
if not digest:
raise ValueError('Missing input data provenance')
hashes.add(digest)
runs[name] = json.loads((directory/'rows.json').read_text())
provenance[name] = p
if len(hashes) != 1:
raise ValueError('Runs use different datasets')
result = {**compare(runs), 'datasetSha256': hashes.pop(), 'provenance': provenance}
args.out.parent.mkdir(parents=True, exist_ok=True)
args.out.write_text(json.dumps(result, indent=2)+'\n')
if __name__ == '__main__':
main()