File size: 2,723 Bytes
2bff5be | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 | """Score the extractive reading mode separately from free next-token generation."""
import argparse, hashlib, json
from pathlib import Path
import numpy as np
import torch
from safetensors.torch import load_file
from tokenizers import Tokenizer
from franok.model import Franok, Config
from franok.span import SpanReadout
from franok.reading import PassageReader
from .transfer import cases, grade, XQUAD
def main():
p = argparse.ArgumentParser()
p.add_argument("--run", type=Path, required=True)
p.add_argument("--head", type=Path, required=True)
p.add_argument("--out", type=Path, required=True)
p.add_argument("--tokenizer", type=Path, default=Path("web/tokenizer.json"))
p.add_argument("--threads", type=int, default=2)
args = p.parse_args()
torch.set_num_threads(args.threads)
cfg = Config(**json.loads((args.run / "config.json").read_text()))
config = json.loads((args.head / "config.json").read_text())
model = Franok(cfg).eval()
model.load_state_dict(load_file(str(args.run / "model.safetensors")))
head = SpanReadout(cfg.width, config["hidden"], config["max_span"]).eval()
head.load_state_dict(load_file(str(args.head / "head.safetensors")))
reader = PassageReader(
model, head, Tokenizer.from_file(str(args.tokenizer)), config["threshold"]
)
ids = json.loads(Path("tests/transfer_ids.json").read_text())["xquad"]
results = []
for item in cases("xquad", ids):
lang = item["lang"]
prediction = reader.read(item["question"], item["passage"], lang)
results.append(
dict(
lang=lang,
key=item["key"],
**prediction,
**grade(prediction["answer"], item["answers"], lang)
)
)
summary = {}
for lang in ("en", "tr"):
rows = [r for r in results if r["lang"] == lang]
summary[lang] = dict(
n=len(rows),
exact=sum(r["exact"] for r in rows),
f1=float(np.mean([r["f1"] for r in rows])),
answered=sum(r["accepted"] for r in rows),
)
def digest(path):
with path.open("rb") as f:
return hashlib.file_digest(f, "sha256").hexdigest()
info = dict(
suite="xquad-extractive-head",
dataset_revision=XQUAD,
threshold=reader.threshold,
base=digest(args.run / "model.safetensors"),
head=digest(args.head / "head.safetensors"),
summary=summary,
items=results,
)
args.out.parent.mkdir(parents=True, exist_ok=True)
args.out.write_text(json.dumps(info, indent=2, ensure_ascii=False) + "\n")
print(summary, flush=True)
if __name__ == "__main__":
main()
|