Franok / source /tools /eval_inputs.py
yunusemrejr's picture
Publish source, film and evaluations from 4434b261773d
df6d041 verified
Raw History Blame Contribute Delete
1.72 kB
"""Regenerate the pinned multiple-choice comparisons without redistributing test passages."""
import json
from pathlib import Path
from huggingface_hub import hf_hub_download
from .reading import REV
def main():
root = Path("work/measurement")
root.mkdir(parents=True, exist_ok=True)
prior = json.loads(Path("docs/evaluation/franok-belebele.json").read_text())
samples = []
for lang, dialect in [("en", "eng_Latn"), ("tr", "tur_Latn")]:
path = hf_hub_download(
"facebook/belebele",
"data/" + dialect + ".jsonl",
revision=REV,
repo_type="dataset",
)
all_items = {
(r["link"], r["question_number"]): r
for r in map(json.loads, Path(path).read_text().splitlines())
}
for item in prior["items"]:
if item["language"] != lang:
continue
key = item["link"], item["question"]
r = all_items[key]
samples.append(
dict(
suite="reading",
lang=lang,
key=list(key),
q=r["flores_passage"] + "\n\n" + r["question"],
choices=[r["mc_answer" + str(j)] for j in range(1, 5)],
gold=int(r["correct_answer_num"]) - 1,
)
)
for i, (lang, q, correct, foil) in enumerate(
json.loads(Path("tests/dialogue.json").read_text())
):
samples.append(
dict(suite="short", lang=lang, key=i, q=q, choices=[correct, foil], gold=0)
)
(root / "cases.json").write_text(json.dumps(samples, ensure_ascii=False))
if __name__ == "__main__":
main()