File size: 4,894 Bytes
eb373ad | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 | """Compare single-request, warm CPU inference on identical bilingual text."""
import argparse
import hashlib
import json
from pathlib import Path
import platform
from statistics import median
from time import perf_counter
import torch
from safetensors.torch import load_file
from tokenizers import Tokenizer
from franok.model import Config, Franok
from franok.incremental import Decoder
from tools.reading import BASE
TEXT = {
"en": (
"We used the public library for our evening meeting. Deniz brought a notebook and Ayla wrote down what we agreed. ",
"We will meet again on Friday. Everyone can bring a book to discuss.",
),
"tr": (
"Akşam toplantımızı mahalle kütüphanesinde yaptık. Deniz bir defter getirdi ve Ayla aldığımız kararları yazdı. ",
"Cuma günü yeniden buluşacağız. Herkes tartışmak için bir kitap getirebilir.",
),
}
@torch.inference_mode()
def main():
p = argparse.ArgumentParser()
g = p.add_mutually_exclusive_group(required=True)
g.add_argument("--run", type=Path)
g.add_argument("--peer", action="store_true")
p.add_argument("--out", type=Path, required=True)
p.add_argument("--threads", type=int, default=3)
a = p.parse_args()
torch.set_num_threads(a.threads)
if a.peer:
from transformers import AutoModelForCausalLM, AutoTokenizer
repo = "HuggingFaceTB/SmolLM2-360M-Instruct"
model = AutoModelForCausalLM.from_pretrained(
repo, revision=BASE, dtype=torch.float32, local_files_only=True
).eval()
tok = AutoTokenizer.from_pretrained(repo, revision=BASE, local_files_only=True)
ident = BASE
def encode(s):
return tok.encode(s, add_special_tokens=False)
def pre(ids):
return model(
torch.tensor([ids]), use_cache=True, logits_to_keep=1
).past_key_values
def step(t, past):
return model(
torch.tensor([[t]]),
past_key_values=past,
use_cache=True,
logits_to_keep=1,
).past_key_values
else:
cfg = Config(**json.loads((a.run / "config.json").read_text()))
file = a.run / "model.safetensors"
with file.open("rb") as f:
ident = hashlib.file_digest(f, "sha256").hexdigest()
model = Franok(cfg).eval()
model.load_state_dict(load_file(str(file)))
dec = Decoder(model)
tok = Tokenizer.from_file("web/tokenizer.json")
def encode(s):
return tok.encode(s).ids
def pre(ids):
dec.prefill(ids)
def step(t, _):
dec.step(t)
rows = []
for lang, (base, cont) in TEXT.items():
tail = encode(cont)
for repeats in (2, 12):
text = (base * repeats).strip()
ids = encode(text)
assert len(ids) + len(tail) <= 1024
small = ids[: min(16, len(ids))]
# Warm both the matrix prefill and the single-token path.
seed = pre(small)
step(tail[0], seed)
pf, decoding = [], []
for _ in range(5):
begin = perf_counter()
cache = pre(ids)
split = perf_counter()
for t in tail:
cache = step(t, cache)
pf.append(split - begin)
decoding.append(perf_counter() - split)
rows.append(
dict(
lang=lang,
repeats=repeats,
prompt_tokens=len(ids),
prompt_bytes=len(text.encode()),
continuation_tokens=len(tail),
continuation_bytes=len(cont.encode()),
prefill_seconds=pf,
decode_seconds=decoding,
median_prefill_ms=1000 * median(pf),
median_decode_tokens_per_second=len(tail) / median(decoding),
median_decode_bytes_per_second=len(cont.encode())
/ median(decoding),
)
)
cpu = next(
(
l.split(":", 1)[1].strip()
for l in Path("/proc/cpuinfo").read_text().splitlines()
if l.startswith("model name")
),
platform.machine(),
)
data = dict(
model="SmolLM2-360M-Instruct" if a.peer else "Franok",
weights=ident,
cpu=cpu,
threads=a.threads,
dtype="float32",
torch=torch.__version__,
cases=rows,
)
a.out.parent.mkdir(exist_ok=True, parents=True)
a.out.write_text(json.dumps(data, indent=2) + "\n")
print(
[
(r["lang"], r["repeats"], round(r["median_decode_bytes_per_second"]))
for r in rows
]
)
if __name__ == "__main__":
main()
|