""" Evaluate on XCOPA Turkish — commonsense reasoning, 2-choice. 500 test examples. Random baseline = 50%. Scoring: length-normalised conditional log-probability. No fine-tuning or instruction following needed. Usage: python eval/eval_xcopa.py --model-path ./models/ckpt_scratch_small python eval/eval_xcopa.py --model-path ./models/ckpt_qwen_1_7b """ import argparse import json import math import os import urllib.request from pathlib import Path import torch from transformers import AutoModelForCausalLM, AutoTokenizer from tqdm import tqdm QUESTION_MAP = { "cause": "Bunun nedeni neydi?", "effect": "Bunun ardından ne oldu?", } # Cached local path for Turkish XCOPA test data (500 examples) _DATA_DIR = Path(__file__).parent / "data" _XCOPA_TR = _DATA_DIR / "xcopa_tr_test.jsonl" _XCOPA_URL = "https://raw.githubusercontent.com/cambridgeltl/xcopa/master/data/tr/test.tr.jsonl" def load_xcopa_tr(): """Load XCOPA-TR test set (500 examples). Tries the HuggingFace `datasets` library first. If that fails (known issue: huggingface_hub >=0.29 rejects un-namespaced legacy datasets like 'xcopa'), falls back to downloading the raw JSONL from the Cambridge LTL GitHub repository and caching it under eval/data/. """ try: from datasets import load_dataset ds = load_dataset("xcopa", "tr", split="test") return list(ds) except Exception: pass if not _XCOPA_TR.exists(): _DATA_DIR.mkdir(parents=True, exist_ok=True) print(f"HuggingFace load failed — downloading XCOPA-TR from GitHub ...") urllib.request.urlretrieve(_XCOPA_URL, _XCOPA_TR) print(f"Cached to {_XCOPA_TR}") examples = [] with open(_XCOPA_TR) as f: for line in f: line = line.strip() if line: examples.append(json.loads(line)) return examples def score_continuation(model, tokenizer, context, continuation, device): ctx_ids = tokenizer.encode(context, add_special_tokens=False) cont_ids = tokenizer.encode(" " + continuation.strip(), add_special_tokens=False) if not cont_ids: return float("-inf") input_ids = torch.tensor([ctx_ids + cont_ids], device=device) labels = torch.tensor([[-100] * len(ctx_ids) + cont_ids], device=device) with torch.no_grad(): with torch.autocast("cuda", dtype=torch.bfloat16, enabled=(device.type == "cuda")): loss = model(input_ids=input_ids, labels=labels).loss return -loss.item() def main(): parser = argparse.ArgumentParser() parser.add_argument("--model-path", required=True) args = parser.parse_args() device = torch.device( "cuda" if torch.cuda.is_available() else "mps" if torch.backends.mps.is_available() else "cpu" ) print(f"Loading model : {args.model_path}") tokenizer = AutoTokenizer.from_pretrained(args.model_path) # eager attention: the default sdpa kernel on MPS (macOS 27 / torch 2.12.1) is not # causal and corrupts the log-probabilities. Eager matches CPU fp32. model = AutoModelForCausalLM.from_pretrained( args.model_path, dtype=torch.bfloat16, attn_implementation="eager") model = model.to(device).eval() print("Loading XCOPA-TR ...") examples = load_xcopa_tr() correct = 0 for ex in tqdm(examples, desc="XCOPA-TR"): question = QUESTION_MAP.get(ex["question"], "") context = f"{ex['premise']} {question}" s0 = score_continuation(model, tokenizer, context, ex["choice1"], device) s1 = score_continuation(model, tokenizer, context, ex["choice2"], device) pred = 0 if s0 > s1 else 1 if pred == ex["label"]: correct += 1 acc = correct / len(examples) * 100 print(f"\n{'='*45}") print(f"Model : {args.model_path}") print(f"XCOPA-TR : {acc:.1f}% ({correct}/{len(examples)})") print(f"Baseline : 50.0% (random)") print(f"{'='*45}") if __name__ == "__main__": main()