Spaces:
Running
Running
| """Export the toolkit's audited results for the public static Space.""" | |
| import argparse | |
| from datetime import datetime, timezone | |
| import hashlib | |
| import json | |
| from pathlib import Path | |
| import subprocess | |
| import sys | |
| def export(source, output, results_path=None): | |
| sys.path.insert(0, str(source)) | |
| from jevalkit.config import MODELS | |
| from jevalkit.datasets import BENCHMARKS, published_manifest | |
| board_file = results_path or source / 'artifacts/leaderboard/leaderboard.json' | |
| if board_file.is_dir(): | |
| board_file = board_file / 'leaderboard.json' | |
| board = json.loads(board_file.read_text()) | |
| inventory = {'datasets': [published_manifest(name) for name in BENCHMARKS]} | |
| pinned = {d['name']: d for d in inventory['datasets']} | |
| assert len({(r['model'], r['benchmark']) for r in board['primary_results']}) == len(board['primary_results']), 'Multiple current protocols need explicit selection before publishing.' | |
| for row in board['primary_results']: | |
| assert row['dataset_sha256'] == pinned[row['benchmark']]['sha256'] | |
| assert row['protocol'] == pinned[row['benchmark']]['protocol'] | |
| commit = subprocess.check_output(['git', '-C', str(source), 'rev-parse', 'HEAD'], text=True).strip() | |
| primary = {(r['model'], r['benchmark'], r['dataset_sha256'], r['protocol'], r['evidence']) | |
| for r in board['primary_results']} | |
| results = [] | |
| for row in board['results']: | |
| record = {k: v for k, v in row.items() if k != 'evidence'} | |
| record['primary'] = (row['model'], row['benchmark'], row['dataset_sha256'], | |
| row['protocol'], row['evidence']) in primary | |
| if record['eligible']: | |
| assert row['complete'] and not row['smoke'] and row['failed'] == 0 | |
| assert row.get('input_conformant', True) | |
| evidence = Path(row['evidence']) | |
| if evidence.is_file(): | |
| raw = evidence.read_bytes() | |
| metrics = json.loads(raw) | |
| record['metrics_sha256'] = hashlib.sha256(raw).hexdigest() | |
| record['predictions_sha256'] = metrics.get('predictions_sha256') | |
| record['manifest_sha256'] = metrics.get('manifest_sha256') | |
| record['requests'] = metrics.get('total_requests') | |
| record['overall'] = metrics.get('metrics', {}).get('overall', {}) | |
| record['by_type'] = metrics.get('metrics', {}).get('groups', {}).get('type', {}) | |
| record['id'] = hashlib.sha256(json.dumps(record, sort_keys=True).encode()).hexdigest()[:16] | |
| results.append(record) | |
| present = {r['model'] for r in results} | |
| size_file = Path(__file__).resolve().parents[1] / 'data/model-sizes.json' | |
| sizes = json.loads(size_file.read_text()) | |
| models = [] | |
| for name, model in MODELS.items(): | |
| if model.family == 'baseline' or name == 'Clef' and name not in present: | |
| continue | |
| models.append({'id': name, 'family': model.family, 'checkpoint': model.model, | |
| 'source_url': model.source_url, 'transport': model.transport, | |
| 'evaluated': name in {r['model'] for r in board['primary_results']}, | |
| **sizes.get(name, {'parameters_b': None, 'parameter_label': 'Undisclosed', | |
| 'parameter_source': None})}) | |
| benchmarks = [] | |
| for item in inventory['datasets']: | |
| if item['name'] == 'VisualDecisionBench': | |
| continue | |
| item = {k: v for k, v in item.items() if k not in {'path', 'source'}} | |
| item.update(BENCHMARKS[item['name']]) | |
| upstream = next(d['source']['repository'] for d in inventory['datasets'] if d['name'] == item['name']) | |
| item['source_url'] = upstream | |
| benchmarks.append(item) | |
| full = [r for r in results if r['primary'] and r['eligible']] | |
| snapshot = { | |
| 'schema_version': 1, | |
| 'snapshot': { | |
| 'exported_at': datetime.now(timezone.utc).isoformat(), | |
| 'last_run_at': max(r.get('created_utc') or '' for r in results), | |
| 'source_commit': commit, | |
| 'source_url': f'https://github.com/Valen-Team/JevalKit/tree/{commit}', | |
| 'counts': { | |
| 'model_families': len({m.family for m in MODELS.values() if m.family != 'baseline'}), | |
| 'evaluated_families': len({MODELS[r['model']].family for r in full}), | |
| 'model_configurations': len({r['model'] for r in board['primary_results']}), | |
| 'benchmark_families': len({b['family'] for b in BENCHMARKS.values()}), | |
| 'eligible_scores': len(full), 'retained_runs': len(results), | |
| 'fully_unsupported': sum(r['primary'] and not r['eligible'] and r['coverage'] == 0 for r in results), | |
| }, | |
| }, | |
| 'models': models, 'benchmarks': benchmarks, 'results': results, | |
| 'policy': board['policy'], | |
| } | |
| text = json.dumps(snapshot, ensure_ascii=False, separators=(',', ':'), allow_nan=False) + '\n' | |
| assert '/mnt/shared-storage' not in text | |
| assert len({r['id'] for r in results}) == len(results) | |
| assert len(full) == sum(r['eligible'] for r in board['primary_results']) | |
| output.parent.mkdir(parents=True, exist_ok=True) | |
| output.write_text(text) | |
| print(json.dumps(snapshot['snapshot'], indent=2)) | |
| if __name__ == '__main__': | |
| parser = argparse.ArgumentParser(description=__doc__) | |
| parser.add_argument('--source', type=Path, required=True) | |
| parser.add_argument('--output', type=Path, default=Path(__file__).resolve().parents[1] / 'data/leaderboard.json') | |
| parser.add_argument('--results', type=Path, help='Generated leaderboard JSON or its directory; defaults to the source checkout artifact cache') | |
| args = parser.parse_args() | |
| export(args.source.resolve(), args.output.resolve(), args.results.resolve() if args.results else None) | |