Spaces:
Sleeping
Sleeping
File size: 1,489 Bytes
0624276 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 | """Genera una respuesta (greedy) con `MODEL_ID` y mide tokens por segundo.
Uso: cd engine && uv run python -m scripts.generate "¿Qué es una neurona?" --max-new-tokens 64
"""
import argparse
import time
import torch
from app.model import load_model, model_id_from_env
def main() -> None:
parser = argparse.ArgumentParser()
parser.add_argument("prompt", nargs="?", default="What is the capital of France?")
parser.add_argument("--max-new-tokens", type=int, default=64)
args = parser.parse_args()
loaded = load_model(model_id_from_env())
model, tokenizer = loaded.model, loaded.tokenizer
messages = [{"role": "user", "content": args.prompt}]
inputs = tokenizer.apply_chat_template(
messages, add_generation_prompt=True, return_tensors="pt", return_dict=True
).to(model.device)
prompt_len = inputs["input_ids"].shape[1]
start = time.perf_counter()
with torch.no_grad():
output = model.generate(
**inputs,
max_new_tokens=args.max_new_tokens,
do_sample=False,
pad_token_id=tokenizer.eos_token_id,
)
elapsed = time.perf_counter() - start
new_tokens = output[0, prompt_len:]
print(tokenizer.decode(new_tokens, skip_special_tokens=True))
print()
print(f"{loaded.info.model_id} en {loaded.info.device}")
print(f"{len(new_tokens)} tokens en {elapsed:.2f} s → {len(new_tokens) / elapsed:.1f} tokens/s")
if __name__ == "__main__":
main()
|