Franok / source /tools /decoder_speed.py
yunusemrejr's picture
Publish source, film and evaluations from 8b3b199f3d72
eb373ad verified
Raw History Blame Contribute Delete
4.89 kB
"""Compare single-request, warm CPU inference on identical bilingual text."""
import argparse
import hashlib
import json
from pathlib import Path
import platform
from statistics import median
from time import perf_counter
import torch
from safetensors.torch import load_file
from tokenizers import Tokenizer
from franok.model import Config, Franok
from franok.incremental import Decoder
from tools.reading import BASE
TEXT = {
"en": (
"We used the public library for our evening meeting. Deniz brought a notebook and Ayla wrote down what we agreed. ",
"We will meet again on Friday. Everyone can bring a book to discuss.",
),
"tr": (
"Akşam toplantımızı mahalle kütüphanesinde yaptık. Deniz bir defter getirdi ve Ayla aldığımız kararları yazdı. ",
"Cuma günü yeniden buluşacağız. Herkes tartışmak için bir kitap getirebilir.",
),
}
@torch.inference_mode()
def main():
p = argparse.ArgumentParser()
g = p.add_mutually_exclusive_group(required=True)
g.add_argument("--run", type=Path)
g.add_argument("--peer", action="store_true")
p.add_argument("--out", type=Path, required=True)
p.add_argument("--threads", type=int, default=3)
a = p.parse_args()
torch.set_num_threads(a.threads)
if a.peer:
from transformers import AutoModelForCausalLM, AutoTokenizer
repo = "HuggingFaceTB/SmolLM2-360M-Instruct"
model = AutoModelForCausalLM.from_pretrained(
repo, revision=BASE, dtype=torch.float32, local_files_only=True
).eval()
tok = AutoTokenizer.from_pretrained(repo, revision=BASE, local_files_only=True)
ident = BASE
def encode(s):
return tok.encode(s, add_special_tokens=False)
def pre(ids):
return model(
torch.tensor([ids]), use_cache=True, logits_to_keep=1
).past_key_values
def step(t, past):
return model(
torch.tensor([[t]]),
past_key_values=past,
use_cache=True,
logits_to_keep=1,
).past_key_values
else:
cfg = Config(**json.loads((a.run / "config.json").read_text()))
file = a.run / "model.safetensors"
with file.open("rb") as f:
ident = hashlib.file_digest(f, "sha256").hexdigest()
model = Franok(cfg).eval()
model.load_state_dict(load_file(str(file)))
dec = Decoder(model)
tok = Tokenizer.from_file("web/tokenizer.json")
def encode(s):
return tok.encode(s).ids
def pre(ids):
dec.prefill(ids)
def step(t, _):
dec.step(t)
rows = []
for lang, (base, cont) in TEXT.items():
tail = encode(cont)
for repeats in (2, 12):
text = (base * repeats).strip()
ids = encode(text)
assert len(ids) + len(tail) <= 1024
small = ids[: min(16, len(ids))]
# Warm both the matrix prefill and the single-token path.
seed = pre(small)
step(tail[0], seed)
pf, decoding = [], []
for _ in range(5):
begin = perf_counter()
cache = pre(ids)
split = perf_counter()
for t in tail:
cache = step(t, cache)
pf.append(split - begin)
decoding.append(perf_counter() - split)
rows.append(
dict(
lang=lang,
repeats=repeats,
prompt_tokens=len(ids),
prompt_bytes=len(text.encode()),
continuation_tokens=len(tail),
continuation_bytes=len(cont.encode()),
prefill_seconds=pf,
decode_seconds=decoding,
median_prefill_ms=1000 * median(pf),
median_decode_tokens_per_second=len(tail) / median(decoding),
median_decode_bytes_per_second=len(cont.encode())
/ median(decoding),
)
)
cpu = next(
(
l.split(":", 1)[1].strip()
for l in Path("/proc/cpuinfo").read_text().splitlines()
if l.startswith("model name")
),
platform.machine(),
)
data = dict(
model="SmolLM2-360M-Instruct" if a.peer else "Franok",
weights=ident,
cpu=cpu,
threads=a.threads,
dtype="float32",
torch=torch.__version__,
cases=rows,
)
a.out.parent.mkdir(exist_ok=True, parents=True)
a.out.write_text(json.dumps(data, indent=2) + "\n")
print(
[
(r["lang"], r["repeats"], round(r["median_decode_bytes_per_second"]))
for r in rows
]
)
if __name__ == "__main__":
main()