File size: 2,586 Bytes
2bff5be | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 | """Build bilingual validation prefixes and full-precision distributions for Q4/Q8 checks."""
import argparse
import hashlib
import json
from pathlib import Path
import numpy as np
import torch
from safetensors.torch import load_file
from franok.model import Config, Franok
from training.third.batches import Conversations
@torch.inference_mode()
def main():
p = argparse.ArgumentParser()
p.add_argument("--run", type=Path, required=True)
p.add_argument("--out", type=Path, required=True)
p.add_argument("--per-language", type=int, default=48)
p.add_argument("--threads", type=int, default=2)
a = p.parse_args()
torch.set_num_threads(a.threads)
config = Config(**json.loads((a.run / "config.json").read_text()))
model = Franok(config).eval()
model.load_state_dict(load_file(str(a.run / "model.safetensors")))
data = Conversations("val")
g = np.random.default_rng(97521)
cases = []
seen = set()
for lang in ("en", "tr"):
for i in range(a.per_language):
band = (64, 128, 256, 512)[i % 4]
for _ in range(2000):
pre, ans, _, name = data.sample(g, lang, band, False)
if len(pre) > 210 or len(ans) < 3:
continue
digest = hashlib.sha256(
np.asarray(pre, dtype="<u2").tobytes()
).hexdigest()
if digest not in seen:
seen.add(digest)
break
else:
raise RuntimeError("Could not draw an independent prefix")
ans = ans[:16]
ids = pre.astype(int).tolist() + ans.astype(int).tolist()
out = model(torch.tensor([ids[:-1]]))[0, len(pre) - 1 :].log_softmax(-1)
v, k = out.topk(64)
gold = out[torch.arange(len(ans)), torch.tensor(ans.astype(int))]
cases.append(
dict(
lang=lang,
source=name,
group=("dev" if (i // 4) % 2 == 0 else "confirm"),
ids=ids,
at=len(pre),
key=digest,
gold=gold.tolist(),
top=k.tolist(),
probs=v.exp().tolist(),
)
)
a.out.parent.mkdir(parents=True, exist_ok=True)
a.out.write_text(json.dumps(cases, separators=(",", ":")))
print(
len(cases),
"prefixes",
sum(len(r["gold"]) for r in cases),
"reference positions",
)
if __name__ == "__main__":
main()
|