Spaces:
Running
Running
Read bundled dataset catalog after documentation cleanup
Browse files- data/leaderboard.json +0 -0
- scripts/sync_results.py +2 -2
data/leaderboard.json
CHANGED
|
The diff for this file is too large to render.
See raw diff
|
|
|
scripts/sync_results.py
CHANGED
|
@@ -11,10 +11,10 @@ import sys
|
|
| 11 |
def export(source, output):
|
| 12 |
sys.path.insert(0, str(source))
|
| 13 |
from jevalkit.config import MODELS
|
| 14 |
-
from jevalkit.datasets import BENCHMARKS
|
| 15 |
|
| 16 |
board = json.loads((source / 'leaderboard/leaderboard.json').read_text())
|
| 17 |
-
inventory =
|
| 18 |
pinned = {d['name']: d for d in inventory['datasets']}
|
| 19 |
assert len({(r['model'], r['benchmark']) for r in board['primary_results']}) == len(board['primary_results']), 'Multiple current protocols need explicit selection before publishing.'
|
| 20 |
for row in board['primary_results']:
|
|
|
|
| 11 |
def export(source, output):
|
| 12 |
sys.path.insert(0, str(source))
|
| 13 |
from jevalkit.config import MODELS
|
| 14 |
+
from jevalkit.datasets import BENCHMARKS, published_manifest
|
| 15 |
|
| 16 |
board = json.loads((source / 'leaderboard/leaderboard.json').read_text())
|
| 17 |
+
inventory = {'datasets': [published_manifest(name) for name in BENCHMARKS]}
|
| 18 |
pinned = {d['name']: d for d in inventory['datasets']}
|
| 19 |
assert len({(r['model'], r['benchmark']) for r in board['primary_results']}) == len(board['primary_results']), 'Multiple current protocols need explicit selection before publishing.'
|
| 20 |
for row in board['primary_results']:
|