Spaces:
Running
Running
File size: 5,830 Bytes
089d8d6 b744758 089d8d6 7ee6f4e 089d8d6 b744758 7ee6f4e 089d8d6 5856d6a 089d8d6 5856d6a 089d8d6 b744758 089d8d6 b744758 089d8d6 b744758 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 | """Export the toolkit's audited results for the public static Space."""
import argparse
from datetime import datetime, timezone
import hashlib
import json
from pathlib import Path
import subprocess
import sys
def export(source, output, results_path=None):
sys.path.insert(0, str(source))
from jevalkit.config import MODELS
from jevalkit.datasets import BENCHMARKS, published_manifest
board_file = results_path or source / 'artifacts/leaderboard/leaderboard.json'
if board_file.is_dir():
board_file = board_file / 'leaderboard.json'
board = json.loads(board_file.read_text())
inventory = {'datasets': [published_manifest(name) for name in BENCHMARKS]}
pinned = {d['name']: d for d in inventory['datasets']}
assert len({(r['model'], r['benchmark']) for r in board['primary_results']}) == len(board['primary_results']), 'Multiple current protocols need explicit selection before publishing.'
for row in board['primary_results']:
assert row['dataset_sha256'] == pinned[row['benchmark']]['sha256']
assert row['protocol'] == pinned[row['benchmark']]['protocol']
commit = subprocess.check_output(['git', '-C', str(source), 'rev-parse', 'HEAD'], text=True).strip()
primary = {(r['model'], r['benchmark'], r['dataset_sha256'], r['protocol'], r['evidence'])
for r in board['primary_results']}
results = []
for row in board['results']:
record = {k: v for k, v in row.items() if k != 'evidence'}
record['primary'] = (row['model'], row['benchmark'], row['dataset_sha256'],
row['protocol'], row['evidence']) in primary
if record['eligible']:
assert row['complete'] and not row['smoke'] and row['failed'] == 0
assert row.get('input_conformant', True)
evidence = Path(row['evidence'])
if evidence.is_file():
raw = evidence.read_bytes()
metrics = json.loads(raw)
record['metrics_sha256'] = hashlib.sha256(raw).hexdigest()
record['predictions_sha256'] = metrics.get('predictions_sha256')
record['manifest_sha256'] = metrics.get('manifest_sha256')
record['requests'] = metrics.get('total_requests')
record['overall'] = metrics.get('metrics', {}).get('overall', {})
record['by_type'] = metrics.get('metrics', {}).get('groups', {}).get('type', {})
record['id'] = hashlib.sha256(json.dumps(record, sort_keys=True).encode()).hexdigest()[:16]
results.append(record)
present = {r['model'] for r in results}
size_file = Path(__file__).resolve().parents[1] / 'data/model-sizes.json'
sizes = json.loads(size_file.read_text())
models = []
for name, model in MODELS.items():
if model.family == 'baseline' or name == 'Clef' and name not in present:
continue
models.append({'id': name, 'family': model.family, 'checkpoint': model.model,
'source_url': model.source_url, 'transport': model.transport,
'evaluated': name in {r['model'] for r in board['primary_results']},
**sizes.get(name, {'parameters_b': None, 'parameter_label': 'Undisclosed',
'parameter_source': None})})
benchmarks = []
for item in inventory['datasets']:
if item['name'] == 'VisualDecisionBench':
continue
item = {k: v for k, v in item.items() if k not in {'path', 'source'}}
item.update(BENCHMARKS[item['name']])
upstream = next(d['source']['repository'] for d in inventory['datasets'] if d['name'] == item['name'])
item['source_url'] = upstream
benchmarks.append(item)
full = [r for r in results if r['primary'] and r['eligible']]
snapshot = {
'schema_version': 1,
'snapshot': {
'exported_at': datetime.now(timezone.utc).isoformat(),
'last_run_at': max(r.get('created_utc') or '' for r in results),
'source_commit': commit,
'source_url': f'https://github.com/Valen-Team/JevalKit/tree/{commit}',
'counts': {
'model_families': len({m.family for m in MODELS.values() if m.family != 'baseline'}),
'evaluated_families': len({MODELS[r['model']].family for r in full}),
'model_configurations': len({r['model'] for r in board['primary_results']}),
'benchmark_families': len({b['family'] for b in BENCHMARKS.values()}),
'eligible_scores': len(full), 'retained_runs': len(results),
'fully_unsupported': sum(r['primary'] and not r['eligible'] and r['coverage'] == 0 for r in results),
},
},
'models': models, 'benchmarks': benchmarks, 'results': results,
'policy': board['policy'],
}
text = json.dumps(snapshot, ensure_ascii=False, separators=(',', ':'), allow_nan=False) + '\n'
assert '/mnt/shared-storage' not in text
assert len({r['id'] for r in results}) == len(results)
assert len(full) == sum(r['eligible'] for r in board['primary_results'])
output.parent.mkdir(parents=True, exist_ok=True)
output.write_text(text)
print(json.dumps(snapshot['snapshot'], indent=2))
if __name__ == '__main__':
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--source', type=Path, required=True)
parser.add_argument('--output', type=Path, default=Path(__file__).resolve().parents[1] / 'data/leaderboard.json')
parser.add_argument('--results', type=Path, help='Generated leaderboard JSON or its directory; defaults to the source checkout artifact cache')
args = parser.parse_args()
export(args.source.resolve(), args.output.resolve(), args.results.resolve() if args.results else None)
|