Franok / source /tools /runtime_reading.py
yunusemrejr's picture
Publish source, film and evaluations from 8b3b199f3d72
eb373ad verified
Raw History Blame Contribute Delete
2.78 kB
"""Prepare and score the same bilingual reading items for the browser and native models."""
import argparse
from collections import defaultdict
import json
from pathlib import Path
from tools.transfer import cases, grade, SYSTEM, XQUAD
from tools.reading import REV
def prepare(output):
ids = json.loads(Path("tests/transfer_ids.json").read_text())
suites = {suite: list(cases(suite, keys)) for suite, keys in ids.items()}
output.parent.mkdir(parents=True, exist_ok=True)
output.write_text(
json.dumps(dict(system=SYSTEM, suites=suites), ensure_ascii=False)
)
def summarize(inputs, result, output):
suites = json.loads(inputs.read_text())["suites"]
lookup = {
(suite, row["lang"], json.dumps(row["key"])): row
for suite, items in suites.items()
for row in items
}
summaries = defaultdict(list)
rows = []
for line in result.read_text().splitlines():
row = json.loads(line)
suite = row["suite"]
ref = lookup[
"belebele" if suite == "belebele" else "xquad",
row["lang"],
json.dumps(row["key"]),
]
metrics = (
{"correct": int(ref["gold"] == row["prediction"])}
if suite == "belebele"
else grade(row["answer"], ref["answers"], row["lang"])
)
row.update(metrics)
summaries[suite, row["lang"]].append(metrics)
rows.append(row)
seen = {(r["suite"], r["lang"], json.dumps(r["key"])) for r in rows}
if len(seen) != len(rows) or not rows:
raise ValueError("Empty result or duplicate reading item")
summary = defaultdict(dict)
for (suite, lang), values in summaries.items():
expected = sum(
1
for item in suites["belebele" if suite == "belebele" else "xquad"]
if item["lang"] == lang
)
if len(values) != expected:
raise ValueError("Incomplete " + suite + "/" + lang)
summary[suite][lang] = {
"count": len(values),
**{k: sum(r[k] for r in values) / len(values) for k in values[0]},
}
data = dict(
dataset_revisions={"belebele": REV, "xquad": XQUAD}, summary=summary, items=rows
)
output.write_text(json.dumps(data, indent=2, ensure_ascii=False) + "\n")
print(json.dumps(summary))
if __name__ == "__main__":
p = argparse.ArgumentParser()
p.add_argument("inputs", type=Path)
p.add_argument("--results", type=Path)
p.add_argument("--out", type=Path)
args = p.parse_args()
if bool(args.results) != bool(args.out):
p.error("Supply --results and --out together.")
if args.results:
summarize(args.inputs, args.results, args.out)
else:
prepare(args.inputs)