File size: 6,187 Bytes
bfa0693
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
"""
Measure two efficiency metrics for the paper's efficiency analysis section:

  1. Tokenizer fertility  β€” avg tokens per word on Turkish text (lower = better compression)
  2. Inference speed      β€” tokens/second for autoregressive generation

Usage:
    python eval/eval_efficiency.py --model-path ./ckpt_scratch_small
    python eval/eval_efficiency.py --model-path Qwen/Qwen3-1.7B
    python eval/eval_efficiency.py --model-path TURKCELL/Turkcell-LLM-7b-v1
"""

import argparse
import time
import torch
from transformers import AutoModelForCausalLM, AutoTokenizer
from datasets import load_dataset

# Fixed Turkish prompt for all models β€” kept short so it fits every tokenizer
PROMPT = (
    "Türkiye, Avrupa ile Asya arasında kâprü gârevi gâren eşsiz bir coğrafi konuma sahiptir. "
    "İstanbul, dünyanın iki kıtaya yayılan tek büyükşehridir."
)

N_GENERATE    = 100   # tokens to generate for speed test
N_WARMUP      = 1     # warmup forward pass (discarded)
N_DOCS        = 100   # Wikipedia docs for fertility measurement
SKIP_DOCS     = 5_000


# ── Tokenizer fertility ────────────────────────────────────────────────────────

def measure_fertility(tokenizer, texts):
    total_tokens = 0
    total_words  = 0
    for text in texts:
        text = text.strip()
        if not text:
            continue
        words  = text.split()
        tokens = tokenizer.encode(text, add_special_tokens=False)
        total_words  += len(words)
        total_tokens += len(tokens)
    return total_tokens / total_words if total_words else 0.0


# ── Inference speed ────────────────────────────────────────────────────────────

def measure_speed(model, tokenizer, prompt, device, n_generate=N_GENERATE, n_warmup=N_WARMUP):
    input_ids = tokenizer.encode(prompt, return_tensors="pt").to(device)

    generate_kwargs = dict(
        input_ids=input_ids,
        max_new_tokens=n_generate,
        do_sample=False,        # greedy β€” deterministic, fair comparison
        pad_token_id=tokenizer.eos_token_id,
    )

    # warmup
    for _ in range(n_warmup):
        with torch.no_grad():
            model.generate(**generate_kwargs)

    if device.type == "mps":
        torch.mps.synchronize()
    elif device.type == "cuda":
        torch.cuda.synchronize()

    start = time.perf_counter()
    with torch.no_grad():
        out = model.generate(**generate_kwargs)
    if device.type == "mps":
        torch.mps.synchronize()
    elif device.type == "cuda":
        torch.cuda.synchronize()
    elapsed = time.perf_counter() - start

    generated_tokens = out.shape[1] - input_ids.shape[1]
    tokens_per_sec   = generated_tokens / elapsed
    ms_per_token     = elapsed / generated_tokens * 1000

    return tokens_per_sec, ms_per_token, generated_tokens


# ── Main ───────────────────────────────────────────────────────────────────────

def main():
    parser = argparse.ArgumentParser()
    parser.add_argument("--model-path", required=True)
    parser.add_argument("--n-docs",    type=int, default=N_DOCS)
    parser.add_argument("--skip-docs", type=int, default=SKIP_DOCS)
    parser.add_argument("--n-generate",type=int, default=N_GENERATE)
    args = parser.parse_args()

    device = torch.device(
        "cuda" if torch.cuda.is_available() else
        "mps"  if torch.backends.mps.is_available() else
        "cpu"
    )

    print(f"Device        : {device}")
    print(f"Loading model : {args.model_path}")
    tokenizer = AutoTokenizer.from_pretrained(args.model_path)
    model     = AutoModelForCausalLM.from_pretrained(args.model_path, torch_dtype=torch.bfloat16)
    model     = model.to(device).eval()

    n_params = sum(p.numel() for p in model.parameters())
    print(f"Parameters    : {n_params / 1e6:.0f}M")

    # ── Fertility ──────────────────────────────────────────────────────────────
    print(f"\nLoading Turkish Wikipedia docs (skip={args.skip_docs}, n={args.n_docs}) ...")
    ds    = load_dataset("wikimedia/wikipedia", "20231101.tr", split="train", streaming=True)
    texts = []
    for i, ex in enumerate(ds):
        if i < args.skip_docs:
            continue
        texts.append(ex["text"])
        if len(texts) == args.n_docs:
            break

    fertility = measure_fertility(tokenizer, texts)

    # ── Speed ──────────────────────────────────────────────────────────────────
    print(f"Measuring inference speed ({args.n_generate} tokens) ...")
    tok_per_sec, ms_per_tok, n_gen = measure_speed(
        model, tokenizer, PROMPT, device, n_generate=args.n_generate
    )

    # ── Report ─────────────────────────────────────────────────────────────────
    print(f"\n{'='*50}")
    print(f"Model            : {args.model_path}")
    print(f"Parameters       : {n_params / 1e6:.0f}M")
    print(f"Vocab size       : {len(tokenizer):,}")
    print(f"─"*50)
    print(f"Fertility        : {fertility:.2f} tokens/word  (lower = better)")
    print(f"─"*50)
    print(f"Inference speed  : {tok_per_sec:.1f} tok/s")
    print(f"Latency          : {ms_per_tok:.1f} ms/token")
    print(f"Tokens generated : {n_gen}")
    print(f"{'='*50}")


if __name__ == "__main__":
    main()
    # The abandoned streaming dataset leaves pyarrow threads that deadlock at
    # interpreter shutdown, which hangs run_all.sh. Results are already printed.
    import os, sys
    sys.stdout.flush()
    sys.stderr.flush()
    os._exit(0)