TurkLM-scratch-0.5B / eval /eval_xcopa.py
eademir's picture
Model card, config, tokenizer and evaluation bundle
bfa0693 verified
Raw History Blame Contribute Delete
4.04 kB
"""
Evaluate on XCOPA Turkish — commonsense reasoning, 2-choice.
500 test examples. Random baseline = 50%.
Scoring: length-normalised conditional log-probability.
No fine-tuning or instruction following needed.
Usage:
python eval/eval_xcopa.py --model-path ./models/ckpt_scratch_small
python eval/eval_xcopa.py --model-path ./models/ckpt_qwen_1_7b
"""
import argparse
import json
import math
import os
import urllib.request
from pathlib import Path
import torch
from transformers import AutoModelForCausalLM, AutoTokenizer
from tqdm import tqdm
QUESTION_MAP = {
"cause": "Bunun nedeni neydi?",
"effect": "Bunun ardından ne oldu?",
}
# Cached local path for Turkish XCOPA test data (500 examples)
_DATA_DIR = Path(__file__).parent / "data"
_XCOPA_TR = _DATA_DIR / "xcopa_tr_test.jsonl"
_XCOPA_URL = "https://raw.githubusercontent.com/cambridgeltl/xcopa/master/data/tr/test.tr.jsonl"
def load_xcopa_tr():
"""Load XCOPA-TR test set (500 examples).
Tries the HuggingFace `datasets` library first. If that fails (known
issue: huggingface_hub >=0.29 rejects un-namespaced legacy datasets
like 'xcopa'), falls back to downloading the raw JSONL from the
Cambridge LTL GitHub repository and caching it under eval/data/.
"""
try:
from datasets import load_dataset
ds = load_dataset("xcopa", "tr", split="test")
return list(ds)
except Exception:
pass
if not _XCOPA_TR.exists():
_DATA_DIR.mkdir(parents=True, exist_ok=True)
print(f"HuggingFace load failed — downloading XCOPA-TR from GitHub ...")
urllib.request.urlretrieve(_XCOPA_URL, _XCOPA_TR)
print(f"Cached to {_XCOPA_TR}")
examples = []
with open(_XCOPA_TR) as f:
for line in f:
line = line.strip()
if line:
examples.append(json.loads(line))
return examples
def score_continuation(model, tokenizer, context, continuation, device):
ctx_ids = tokenizer.encode(context, add_special_tokens=False)
cont_ids = tokenizer.encode(" " + continuation.strip(), add_special_tokens=False)
if not cont_ids:
return float("-inf")
input_ids = torch.tensor([ctx_ids + cont_ids], device=device)
labels = torch.tensor([[-100] * len(ctx_ids) + cont_ids], device=device)
with torch.no_grad():
with torch.autocast("cuda", dtype=torch.bfloat16, enabled=(device.type == "cuda")):
loss = model(input_ids=input_ids, labels=labels).loss
return -loss.item()
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--model-path", required=True)
args = parser.parse_args()
device = torch.device(
"cuda" if torch.cuda.is_available() else
"mps" if torch.backends.mps.is_available() else
"cpu"
)
print(f"Loading model : {args.model_path}")
tokenizer = AutoTokenizer.from_pretrained(args.model_path)
# eager attention: the default sdpa kernel on MPS (macOS 27 / torch 2.12.1) is not
# causal and corrupts the log-probabilities. Eager matches CPU fp32.
model = AutoModelForCausalLM.from_pretrained(
args.model_path, dtype=torch.bfloat16, attn_implementation="eager")
model = model.to(device).eval()
print("Loading XCOPA-TR ...")
examples = load_xcopa_tr()
correct = 0
for ex in tqdm(examples, desc="XCOPA-TR"):
question = QUESTION_MAP.get(ex["question"], "")
context = f"{ex['premise']} {question}"
s0 = score_continuation(model, tokenizer, context, ex["choice1"], device)
s1 = score_continuation(model, tokenizer, context, ex["choice2"], device)
pred = 0 if s0 > s1 else 1
if pred == ex["label"]:
correct += 1
acc = correct / len(examples) * 100
print(f"\n{'='*45}")
print(f"Model : {args.model_path}")
print(f"XCOPA-TR : {acc:.1f}% ({correct}/{len(examples)})")
print(f"Baseline : 50.0% (random)")
print(f"{'='*45}")
if __name__ == "__main__":
main()