File size: 1,489 Bytes
0624276
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
"""Genera una respuesta (greedy) con `MODEL_ID` y mide tokens por segundo.

Uso: cd engine && uv run python -m scripts.generate "¿Qué es una neurona?" --max-new-tokens 64
"""

import argparse
import time

import torch

from app.model import load_model, model_id_from_env


def main() -> None:
    parser = argparse.ArgumentParser()
    parser.add_argument("prompt", nargs="?", default="What is the capital of France?")
    parser.add_argument("--max-new-tokens", type=int, default=64)
    args = parser.parse_args()

    loaded = load_model(model_id_from_env())
    model, tokenizer = loaded.model, loaded.tokenizer

    messages = [{"role": "user", "content": args.prompt}]
    inputs = tokenizer.apply_chat_template(
        messages, add_generation_prompt=True, return_tensors="pt", return_dict=True
    ).to(model.device)
    prompt_len = inputs["input_ids"].shape[1]

    start = time.perf_counter()
    with torch.no_grad():
        output = model.generate(
            **inputs,
            max_new_tokens=args.max_new_tokens,
            do_sample=False,
            pad_token_id=tokenizer.eos_token_id,
        )
    elapsed = time.perf_counter() - start

    new_tokens = output[0, prompt_len:]
    print(tokenizer.decode(new_tokens, skip_special_tokens=True))
    print()
    print(f"{loaded.info.model_id} en {loaded.info.device}")
    print(f"{len(new_tokens)} tokens en {elapsed:.2f} s → {len(new_tokens) / elapsed:.1f} tokens/s")


if __name__ == "__main__":
    main()