lorcapoul's picture
Smol Neuron Studio
0624276 verified
Raw History Blame Contribute Delete
1.49 kB
"""Genera una respuesta (greedy) con `MODEL_ID` y mide tokens por segundo.
Uso: cd engine && uv run python -m scripts.generate "¿Qué es una neurona?" --max-new-tokens 64
"""
import argparse
import time
import torch
from app.model import load_model, model_id_from_env
def main() -> None:
parser = argparse.ArgumentParser()
parser.add_argument("prompt", nargs="?", default="What is the capital of France?")
parser.add_argument("--max-new-tokens", type=int, default=64)
args = parser.parse_args()
loaded = load_model(model_id_from_env())
model, tokenizer = loaded.model, loaded.tokenizer
messages = [{"role": "user", "content": args.prompt}]
inputs = tokenizer.apply_chat_template(
messages, add_generation_prompt=True, return_tensors="pt", return_dict=True
).to(model.device)
prompt_len = inputs["input_ids"].shape[1]
start = time.perf_counter()
with torch.no_grad():
output = model.generate(
**inputs,
max_new_tokens=args.max_new_tokens,
do_sample=False,
pad_token_id=tokenizer.eos_token_id,
)
elapsed = time.perf_counter() - start
new_tokens = output[0, prompt_len:]
print(tokenizer.decode(new_tokens, skip_special_tokens=True))
print()
print(f"{loaded.info.model_id} en {loaded.info.device}")
print(f"{len(new_tokens)} tokens en {elapsed:.2f} s → {len(new_tokens) / elapsed:.1f} tokens/s")
if __name__ == "__main__":
main()