no-FIM models are evaluated with prefix-only prompts
Browse files- v2/train.py +4 -3
v2/train.py
CHANGED
|
@@ -399,7 +399,7 @@ def train(model, rows: torch.Tensor, val_rows: torch.Tensor, args, device, track
|
|
| 399 |
# ---- Eval, export --------------------------------------------------------------------------------------------
|
| 400 |
|
| 401 |
|
| 402 |
-
def as_token_lm(ev, name: str, model: TinyNext, tokenizer, device):
|
| 403 |
texts = [tokenizer.decode([i], skip_special_tokens=False) if i >= len(SPECIAL) else "" for i in range(tokenizer.get_vocab_size())]
|
| 404 |
|
| 405 |
@torch.no_grad()
|
|
@@ -409,7 +409,8 @@ def as_token_lm(ev, name: str, model: TinyNext, tokenizer, device):
|
|
| 409 |
return logits[0, -1].float().cpu().numpy(), (k, v)
|
| 410 |
|
| 411 |
encode = lambda text: tokenizer.encode(text).ids or [EOT]
|
| 412 |
-
|
|
|
|
| 413 |
return ev.TokenLM(name, next_logits, encode, texts, set(range(len(SPECIAL))), prompt=prompt)
|
| 414 |
|
| 415 |
|
|
@@ -615,7 +616,7 @@ def main():
|
|
| 615 |
from safetensors.torch import save_file
|
| 616 |
|
| 617 |
save_file({k: v.contiguous() for k, v in model.state_dict().items()}, output / "model.safetensors")
|
| 618 |
-
scored = ev.score(as_token_lm(ev, args.name, model, tokenizer, device), ev.pauses(ev.load_eval_set(), args.eval_pauses))
|
| 619 |
(output / "metrics").mkdir(exist_ok=True)
|
| 620 |
(output / "metrics" / f"{args.name}.json").write_text(json.dumps(scored, indent=1))
|
| 621 |
tracker.log(headline(scored))
|
|
|
|
| 399 |
# ---- Eval, export --------------------------------------------------------------------------------------------
|
| 400 |
|
| 401 |
|
| 402 |
+
def as_token_lm(ev, name: str, model: TinyNext, tokenizer, device, fim: bool = True):
|
| 403 |
texts = [tokenizer.decode([i], skip_special_tokens=False) if i >= len(SPECIAL) else "" for i in range(tokenizer.get_vocab_size())]
|
| 404 |
|
| 405 |
@torch.no_grad()
|
|
|
|
| 409 |
return logits[0, -1].float().cpu().numpy(), (k, v)
|
| 410 |
|
| 411 |
encode = lambda text: tokenizer.encode(text).ids or [EOT]
|
| 412 |
+
# A model trained without fill-in-the-middle never saw the special tokens, so it only gets the text before.
|
| 413 |
+
prompt = (lambda prefix, suffix: prefix if not suffix else [SUF, *suffix, PRE, *prefix, MID]) if fim else None
|
| 414 |
return ev.TokenLM(name, next_logits, encode, texts, set(range(len(SPECIAL))), prompt=prompt)
|
| 415 |
|
| 416 |
|
|
|
|
| 616 |
from safetensors.torch import save_file
|
| 617 |
|
| 618 |
save_file({k: v.contiguous() for k, v in model.state_dict().items()}, output / "model.safetensors")
|
| 619 |
+
scored = ev.score(as_token_lm(ev, args.name, model, tokenizer, device, fim=args.fim_rate > 0), ev.pauses(ev.load_eval_set(), args.eval_pauses))
|
| 620 |
(output / "metrics").mkdir(exist_ok=True)
|
| 621 |
(output / "metrics" / f"{args.name}.json").write_text(json.dumps(scored, indent=1))
|
| 622 |
tracker.log(headline(scored))
|