Fix RUNTIME_ERROR: handle {name, category} dicts in models.benchmarks
Browse files
data.py
CHANGED
|
@@ -1,8 +1,10 @@
|
|
| 1 |
"""Load and flatten the SaylorTwift/llm-benchmark-usage dataset for exploration.
|
| 2 |
|
| 3 |
The dataset has two configs:
|
| 4 |
-
- `models`: one row per model, `benchmarks` is a
|
| 5 |
-
|
|
|
|
|
|
|
| 6 |
- `benchmarks`: reference metadata for a subset of those names — `categories`
|
| 7 |
(controlled vocabulary, can be more than one, e.g. a vision-math benchmark is
|
| 8 |
both `math` and `perception`), `modality`, `language`, `description`,
|
|
@@ -60,7 +62,8 @@ def load_data():
|
|
| 60 |
rows = []
|
| 61 |
for m in models_raw:
|
| 62 |
meta = model_meta[m["model_id"]]
|
| 63 |
-
for
|
|
|
|
| 64 |
info = bench_meta.get(name)
|
| 65 |
categories = info["categories"] if info and info["categories"] else ["uncategorized"]
|
| 66 |
for cat in categories:
|
|
|
|
| 1 |
"""Load and flatten the SaylorTwift/llm-benchmark-usage dataset for exploration.
|
| 2 |
|
| 3 |
The dataset has two configs:
|
| 4 |
+
- `models`: one row per model, `benchmarks` is a list of canonicalized names — either
|
| 5 |
+
plain strings or `{name, category}` dicts (the `category` there is a free-text,
|
| 6 |
+
per-source label and is not used here; the controlled-vocabulary category used
|
| 7 |
+
for aggregation always comes from the `benchmarks` reference table below).
|
| 8 |
- `benchmarks`: reference metadata for a subset of those names — `categories`
|
| 9 |
(controlled vocabulary, can be more than one, e.g. a vision-math benchmark is
|
| 10 |
both `math` and `perception`), `modality`, `language`, `description`,
|
|
|
|
| 62 |
rows = []
|
| 63 |
for m in models_raw:
|
| 64 |
meta = model_meta[m["model_id"]]
|
| 65 |
+
for b in m["benchmarks"]:
|
| 66 |
+
name = b["name"] if isinstance(b, dict) else b
|
| 67 |
info = bench_meta.get(name)
|
| 68 |
categories = info["categories"] if info and info["categories"] else ["uncategorized"]
|
| 69 |
for cat in categories:
|