laolao77 commited on
Commit
7ee6f4e
·
1 Parent(s): c171772

Read bundled dataset catalog after documentation cleanup

Browse files
Files changed (2) hide show
  1. data/leaderboard.json +0 -0
  2. scripts/sync_results.py +2 -2
data/leaderboard.json CHANGED
The diff for this file is too large to render. See raw diff
 
scripts/sync_results.py CHANGED
@@ -11,10 +11,10 @@ import sys
11
  def export(source, output):
12
  sys.path.insert(0, str(source))
13
  from jevalkit.config import MODELS
14
- from jevalkit.datasets import BENCHMARKS
15
 
16
  board = json.loads((source / 'leaderboard/leaderboard.json').read_text())
17
- inventory = json.loads((source / 'docs/data-manifest.json').read_text())
18
  pinned = {d['name']: d for d in inventory['datasets']}
19
  assert len({(r['model'], r['benchmark']) for r in board['primary_results']}) == len(board['primary_results']), 'Multiple current protocols need explicit selection before publishing.'
20
  for row in board['primary_results']:
 
11
  def export(source, output):
12
  sys.path.insert(0, str(source))
13
  from jevalkit.config import MODELS
14
+ from jevalkit.datasets import BENCHMARKS, published_manifest
15
 
16
  board = json.loads((source / 'leaderboard/leaderboard.json').read_text())
17
+ inventory = {'datasets': [published_manifest(name) for name in BENCHMARKS]}
18
  pinned = {d['name']: d for d in inventory['datasets']}
19
  assert len({(r['model'], r['benchmark']) for r in board['primary_results']}) == len(board['primary_results']), 'Multiple current protocols need explicit selection before publishing.'
20
  for row in board['primary_results']: