""" Measure two efficiency metrics for the paper's efficiency analysis section: 1. Tokenizer fertility — avg tokens per word on Turkish text (lower = better compression) 2. Inference speed — tokens/second for autoregressive generation Usage: python eval/eval_efficiency.py --model-path ./ckpt_scratch_small python eval/eval_efficiency.py --model-path Qwen/Qwen3-1.7B python eval/eval_efficiency.py --model-path TURKCELL/Turkcell-LLM-7b-v1 """ import argparse import time import torch from transformers import AutoModelForCausalLM, AutoTokenizer from datasets import load_dataset # Fixed Turkish prompt for all models — kept short so it fits every tokenizer PROMPT = ( "Türkiye, Avrupa ile Asya arasında köprü görevi gören eşsiz bir coğrafi konuma sahiptir. " "İstanbul, dünyanın iki kıtaya yayılan tek büyükşehridir." ) N_GENERATE = 100 # tokens to generate for speed test N_WARMUP = 1 # warmup forward pass (discarded) N_DOCS = 100 # Wikipedia docs for fertility measurement SKIP_DOCS = 5_000 # ── Tokenizer fertility ──────────────────────────────────────────────────────── def measure_fertility(tokenizer, texts): total_tokens = 0 total_words = 0 for text in texts: text = text.strip() if not text: continue words = text.split() tokens = tokenizer.encode(text, add_special_tokens=False) total_words += len(words) total_tokens += len(tokens) return total_tokens / total_words if total_words else 0.0 # ── Inference speed ──────────────────────────────────────────────────────────── def measure_speed(model, tokenizer, prompt, device, n_generate=N_GENERATE, n_warmup=N_WARMUP): input_ids = tokenizer.encode(prompt, return_tensors="pt").to(device) generate_kwargs = dict( input_ids=input_ids, max_new_tokens=n_generate, do_sample=False, # greedy — deterministic, fair comparison pad_token_id=tokenizer.eos_token_id, ) # warmup for _ in range(n_warmup): with torch.no_grad(): model.generate(**generate_kwargs) if device.type == "mps": torch.mps.synchronize() elif device.type == "cuda": torch.cuda.synchronize() start = time.perf_counter() with torch.no_grad(): out = model.generate(**generate_kwargs) if device.type == "mps": torch.mps.synchronize() elif device.type == "cuda": torch.cuda.synchronize() elapsed = time.perf_counter() - start generated_tokens = out.shape[1] - input_ids.shape[1] tokens_per_sec = generated_tokens / elapsed ms_per_token = elapsed / generated_tokens * 1000 return tokens_per_sec, ms_per_token, generated_tokens # ── Main ─────────────────────────────────────────────────────────────────────── def main(): parser = argparse.ArgumentParser() parser.add_argument("--model-path", required=True) parser.add_argument("--n-docs", type=int, default=N_DOCS) parser.add_argument("--skip-docs", type=int, default=SKIP_DOCS) parser.add_argument("--n-generate",type=int, default=N_GENERATE) args = parser.parse_args() device = torch.device( "cuda" if torch.cuda.is_available() else "mps" if torch.backends.mps.is_available() else "cpu" ) print(f"Device : {device}") print(f"Loading model : {args.model_path}") tokenizer = AutoTokenizer.from_pretrained(args.model_path) model = AutoModelForCausalLM.from_pretrained(args.model_path, torch_dtype=torch.bfloat16) model = model.to(device).eval() n_params = sum(p.numel() for p in model.parameters()) print(f"Parameters : {n_params / 1e6:.0f}M") # ── Fertility ────────────────────────────────────────────────────────────── print(f"\nLoading Turkish Wikipedia docs (skip={args.skip_docs}, n={args.n_docs}) ...") ds = load_dataset("wikimedia/wikipedia", "20231101.tr", split="train", streaming=True) texts = [] for i, ex in enumerate(ds): if i < args.skip_docs: continue texts.append(ex["text"]) if len(texts) == args.n_docs: break fertility = measure_fertility(tokenizer, texts) # ── Speed ────────────────────────────────────────────────────────────────── print(f"Measuring inference speed ({args.n_generate} tokens) ...") tok_per_sec, ms_per_tok, n_gen = measure_speed( model, tokenizer, PROMPT, device, n_generate=args.n_generate ) # ── Report ───────────────────────────────────────────────────────────────── print(f"\n{'='*50}") print(f"Model : {args.model_path}") print(f"Parameters : {n_params / 1e6:.0f}M") print(f"Vocab size : {len(tokenizer):,}") print(f"─"*50) print(f"Fertility : {fertility:.2f} tokens/word (lower = better)") print(f"─"*50) print(f"Inference speed : {tok_per_sec:.1f} tok/s") print(f"Latency : {ms_per_tok:.1f} ms/token") print(f"Tokens generated : {n_gen}") print(f"{'='*50}") if __name__ == "__main__": main() # The abandoned streaming dataset leaves pyarrow threads that deadlock at # interpreter shutdown, which hangs run_all.sh. Results are already printed. import os, sys sys.stdout.flush() sys.stderr.flush() os._exit(0)