TurkLM-scratch-0.5B / eval /eval_efficiency.py
eademir's picture
Model card, config, tokenizer and evaluation bundle
bfa0693 verified
Raw History Blame Contribute Delete
6.19 kB
"""
Measure two efficiency metrics for the paper's efficiency analysis section:
1. Tokenizer fertility β€” avg tokens per word on Turkish text (lower = better compression)
2. Inference speed β€” tokens/second for autoregressive generation
Usage:
python eval/eval_efficiency.py --model-path ./ckpt_scratch_small
python eval/eval_efficiency.py --model-path Qwen/Qwen3-1.7B
python eval/eval_efficiency.py --model-path TURKCELL/Turkcell-LLM-7b-v1
"""
import argparse
import time
import torch
from transformers import AutoModelForCausalLM, AutoTokenizer
from datasets import load_dataset
# Fixed Turkish prompt for all models β€” kept short so it fits every tokenizer
PROMPT = (
"Türkiye, Avrupa ile Asya arasında kâprü gârevi gâren eşsiz bir coğrafi konuma sahiptir. "
"İstanbul, dünyanın iki kıtaya yayılan tek büyükşehridir."
)
N_GENERATE = 100 # tokens to generate for speed test
N_WARMUP = 1 # warmup forward pass (discarded)
N_DOCS = 100 # Wikipedia docs for fertility measurement
SKIP_DOCS = 5_000
# ── Tokenizer fertility ────────────────────────────────────────────────────────
def measure_fertility(tokenizer, texts):
total_tokens = 0
total_words = 0
for text in texts:
text = text.strip()
if not text:
continue
words = text.split()
tokens = tokenizer.encode(text, add_special_tokens=False)
total_words += len(words)
total_tokens += len(tokens)
return total_tokens / total_words if total_words else 0.0
# ── Inference speed ────────────────────────────────────────────────────────────
def measure_speed(model, tokenizer, prompt, device, n_generate=N_GENERATE, n_warmup=N_WARMUP):
input_ids = tokenizer.encode(prompt, return_tensors="pt").to(device)
generate_kwargs = dict(
input_ids=input_ids,
max_new_tokens=n_generate,
do_sample=False, # greedy β€” deterministic, fair comparison
pad_token_id=tokenizer.eos_token_id,
)
# warmup
for _ in range(n_warmup):
with torch.no_grad():
model.generate(**generate_kwargs)
if device.type == "mps":
torch.mps.synchronize()
elif device.type == "cuda":
torch.cuda.synchronize()
start = time.perf_counter()
with torch.no_grad():
out = model.generate(**generate_kwargs)
if device.type == "mps":
torch.mps.synchronize()
elif device.type == "cuda":
torch.cuda.synchronize()
elapsed = time.perf_counter() - start
generated_tokens = out.shape[1] - input_ids.shape[1]
tokens_per_sec = generated_tokens / elapsed
ms_per_token = elapsed / generated_tokens * 1000
return tokens_per_sec, ms_per_token, generated_tokens
# ── Main ───────────────────────────────────────────────────────────────────────
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--model-path", required=True)
parser.add_argument("--n-docs", type=int, default=N_DOCS)
parser.add_argument("--skip-docs", type=int, default=SKIP_DOCS)
parser.add_argument("--n-generate",type=int, default=N_GENERATE)
args = parser.parse_args()
device = torch.device(
"cuda" if torch.cuda.is_available() else
"mps" if torch.backends.mps.is_available() else
"cpu"
)
print(f"Device : {device}")
print(f"Loading model : {args.model_path}")
tokenizer = AutoTokenizer.from_pretrained(args.model_path)
model = AutoModelForCausalLM.from_pretrained(args.model_path, torch_dtype=torch.bfloat16)
model = model.to(device).eval()
n_params = sum(p.numel() for p in model.parameters())
print(f"Parameters : {n_params / 1e6:.0f}M")
# ── Fertility ──────────────────────────────────────────────────────────────
print(f"\nLoading Turkish Wikipedia docs (skip={args.skip_docs}, n={args.n_docs}) ...")
ds = load_dataset("wikimedia/wikipedia", "20231101.tr", split="train", streaming=True)
texts = []
for i, ex in enumerate(ds):
if i < args.skip_docs:
continue
texts.append(ex["text"])
if len(texts) == args.n_docs:
break
fertility = measure_fertility(tokenizer, texts)
# ── Speed ──────────────────────────────────────────────────────────────────
print(f"Measuring inference speed ({args.n_generate} tokens) ...")
tok_per_sec, ms_per_tok, n_gen = measure_speed(
model, tokenizer, PROMPT, device, n_generate=args.n_generate
)
# ── Report ─────────────────────────────────────────────────────────────────
print(f"\n{'='*50}")
print(f"Model : {args.model_path}")
print(f"Parameters : {n_params / 1e6:.0f}M")
print(f"Vocab size : {len(tokenizer):,}")
print(f"─"*50)
print(f"Fertility : {fertility:.2f} tokens/word (lower = better)")
print(f"─"*50)
print(f"Inference speed : {tok_per_sec:.1f} tok/s")
print(f"Latency : {ms_per_tok:.1f} ms/token")
print(f"Tokens generated : {n_gen}")
print(f"{'='*50}")
if __name__ == "__main__":
main()
# The abandoned streaming dataset leaves pyarrow threads that deadlock at
# interpreter shutdown, which hangs run_all.sh. Results are already printed.
import os, sys
sys.stdout.flush()
sys.stderr.flush()
os._exit(0)