Download source/tools/span_transfer.py from yunusemrejr/Franok: direct link, hf CLI and curl.
- Browser
- Download file 2.72 kB
-
https://huggingface.co/yunusemrejr/Franok/resolve/main/source/tools/span_transfer.py
- Command line
-
hf download hf://yunusemrejr/Franok/source/tools/span_transfer.py
-
curl -L -o span_transfer.py https://huggingface.co/yunusemrejr/Franok/resolve/main/source/tools/span_transfer.py
2.72 kB
| """Score the extractive reading mode separately from free next-token generation.""" | |
| import argparse, hashlib, json | |
| from pathlib import Path | |
| import numpy as np | |
| import torch | |
| from safetensors.torch import load_file | |
| from tokenizers import Tokenizer | |
| from franok.model import Franok, Config | |
| from franok.span import SpanReadout | |
| from franok.reading import PassageReader | |
| from .transfer import cases, grade, XQUAD | |
| def main(): | |
| p = argparse.ArgumentParser() | |
| p.add_argument("--run", type=Path, required=True) | |
| p.add_argument("--head", type=Path, required=True) | |
| p.add_argument("--out", type=Path, required=True) | |
| p.add_argument("--tokenizer", type=Path, default=Path("web/tokenizer.json")) | |
| p.add_argument("--threads", type=int, default=2) | |
| args = p.parse_args() | |
| torch.set_num_threads(args.threads) | |
| cfg = Config(**json.loads((args.run / "config.json").read_text())) | |
| config = json.loads((args.head / "config.json").read_text()) | |
| model = Franok(cfg).eval() | |
| model.load_state_dict(load_file(str(args.run / "model.safetensors"))) | |
| head = SpanReadout(cfg.width, config["hidden"], config["max_span"]).eval() | |
| head.load_state_dict(load_file(str(args.head / "head.safetensors"))) | |
| reader = PassageReader( | |
| model, head, Tokenizer.from_file(str(args.tokenizer)), config["threshold"] | |
| ) | |
| ids = json.loads(Path("tests/transfer_ids.json").read_text())["xquad"] | |
| results = [] | |
| for item in cases("xquad", ids): | |
| lang = item["lang"] | |
| prediction = reader.read(item["question"], item["passage"], lang) | |
| results.append( | |
| dict( | |
| lang=lang, | |
| key=item["key"], | |
| **prediction, | |
| **grade(prediction["answer"], item["answers"], lang) | |
| ) | |
| ) | |
| summary = {} | |
| for lang in ("en", "tr"): | |
| rows = [r for r in results if r["lang"] == lang] | |
| summary[lang] = dict( | |
| n=len(rows), | |
| exact=sum(r["exact"] for r in rows), | |
| f1=float(np.mean([r["f1"] for r in rows])), | |
| answered=sum(r["accepted"] for r in rows), | |
| ) | |
| def digest(path): | |
| with path.open("rb") as f: | |
| return hashlib.file_digest(f, "sha256").hexdigest() | |
| info = dict( | |
| suite="xquad-extractive-head", | |
| dataset_revision=XQUAD, | |
| threshold=reader.threshold, | |
| base=digest(args.run / "model.safetensors"), | |
| head=digest(args.head / "head.safetensors"), | |
| summary=summary, | |
| items=results, | |
| ) | |
| args.out.parent.mkdir(parents=True, exist_ok=True) | |
| args.out.write_text(json.dumps(info, indent=2, ensure_ascii=False) + "\n") | |
| print(summary, flush=True) | |
| if __name__ == "__main__": | |
| main() | |