cfahlgren1 HF Staff commited on
Commit
793c5c9
·
verified ·
1 Parent(s): 1cffd47

no-FIM models are evaluated with prefix-only prompts

Browse files
Files changed (1) hide show
  1. v2/train.py +4 -3
v2/train.py CHANGED
@@ -399,7 +399,7 @@ def train(model, rows: torch.Tensor, val_rows: torch.Tensor, args, device, track
399
  # ---- Eval, export --------------------------------------------------------------------------------------------
400
 
401
 
402
- def as_token_lm(ev, name: str, model: TinyNext, tokenizer, device):
403
  texts = [tokenizer.decode([i], skip_special_tokens=False) if i >= len(SPECIAL) else "" for i in range(tokenizer.get_vocab_size())]
404
 
405
  @torch.no_grad()
@@ -409,7 +409,8 @@ def as_token_lm(ev, name: str, model: TinyNext, tokenizer, device):
409
  return logits[0, -1].float().cpu().numpy(), (k, v)
410
 
411
  encode = lambda text: tokenizer.encode(text).ids or [EOT]
412
- prompt = lambda prefix, suffix: prefix if not suffix else [SUF, *suffix, PRE, *prefix, MID]
 
413
  return ev.TokenLM(name, next_logits, encode, texts, set(range(len(SPECIAL))), prompt=prompt)
414
 
415
 
@@ -615,7 +616,7 @@ def main():
615
  from safetensors.torch import save_file
616
 
617
  save_file({k: v.contiguous() for k, v in model.state_dict().items()}, output / "model.safetensors")
618
- scored = ev.score(as_token_lm(ev, args.name, model, tokenizer, device), ev.pauses(ev.load_eval_set(), args.eval_pauses))
619
  (output / "metrics").mkdir(exist_ok=True)
620
  (output / "metrics" / f"{args.name}.json").write_text(json.dumps(scored, indent=1))
621
  tracker.log(headline(scored))
 
399
  # ---- Eval, export --------------------------------------------------------------------------------------------
400
 
401
 
402
+ def as_token_lm(ev, name: str, model: TinyNext, tokenizer, device, fim: bool = True):
403
  texts = [tokenizer.decode([i], skip_special_tokens=False) if i >= len(SPECIAL) else "" for i in range(tokenizer.get_vocab_size())]
404
 
405
  @torch.no_grad()
 
409
  return logits[0, -1].float().cpu().numpy(), (k, v)
410
 
411
  encode = lambda text: tokenizer.encode(text).ids or [EOT]
412
+ # A model trained without fill-in-the-middle never saw the special tokens, so it only gets the text before.
413
+ prompt = (lambda prefix, suffix: prefix if not suffix else [SUF, *suffix, PRE, *prefix, MID]) if fim else None
414
  return ev.TokenLM(name, next_logits, encode, texts, set(range(len(SPECIAL))), prompt=prompt)
415
 
416
 
 
616
  from safetensors.torch import save_file
617
 
618
  save_file({k: v.contiguous() for k, v in model.state_dict().items()}, output / "model.safetensors")
619
+ scored = ev.score(as_token_lm(ev, args.name, model, tokenizer, device, fim=args.fim_rate > 0), ev.pauses(ev.load_eval_set(), args.eval_pauses))
620
  (output / "metrics").mkdir(exist_ok=True)
621
  (output / "metrics" / f"{args.name}.json").write_text(json.dumps(scored, indent=1))
622
  tracker.log(headline(scored))