File size: 3,875 Bytes
04558eb | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 | #!/usr/bin/env python3
"""Compare v2 vs v3 on the full held-out test split (66 entries)."""
import json
import time
from collections import defaultdict
from mlx_lm import load, generate
SYS = "Reconstruct the intended syntax from the dictated text. Output only the result."
# Load test split
with open("datasets/finetune/chat/test.jsonl") as f:
tests = []
for line in f:
msg = json.loads(line)["messages"]
tests.append({
"dictated": msg[1]["content"],
"expected": msg[2]["content"],
})
print(f"Loaded {len(tests)} held-out test entries\n")
configs = [
{
"label": "LoRA v2",
"adapter": "datasets/finetune/adapters/qwen-0.5b-lora-v2",
},
{
"label": "LoRA v3",
"adapter": "datasets/finetune/adapters/qwen-0.5b-lora-v3",
},
]
all_results = {}
for cfg in configs:
print(f"\n{'='*60}")
print(f" {cfg['label']}")
print(f"{'='*60}\n")
model, tokenizer = load(
"mlx-community/Qwen2.5-0.5B-Instruct-4bit",
adapter_path=cfg["adapter"],
)
results = []
total_time = 0
for i, t in enumerate(tests):
messages = [
{"role": "system", "content": SYS},
{"role": "user", "content": t["dictated"]},
]
prompt = tokenizer.apply_chat_template(
messages, tokenize=False, add_generation_prompt=True
)
start = time.perf_counter()
got = generate(model, tokenizer, prompt=prompt, max_tokens=80, verbose=False)
elapsed = time.perf_counter() - start
total_time += elapsed
got = got.strip()
match = got == t["expected"]
results.append({"match": match, "got": got, "expected": t["expected"], "dictated": t["dictated"]})
icon = "\033[32m✓\033[0m" if match else "\033[31m✗\033[0m"
if not match:
print(f"{icon} {i+1:>2}. \"{t['dictated']}\"")
print(f" expected: {t['expected']}")
print(f" got: {got}")
correct = sum(1 for r in results if r["match"])
pct = round(correct / len(results) * 100, 1)
avg_ms = round(total_time / len(results) * 1000)
print(f"\nScore: {correct}/{len(results)} ({pct}%)")
print(f"Avg latency: {avg_ms}ms")
all_results[cfg["label"]] = results
del model, tokenizer
# Head-to-head comparison
print(f"\n{'='*60}")
print(" HEAD TO HEAD")
print(f"{'='*60}\n")
v2 = all_results["LoRA v2"]
v3 = all_results["LoRA v3"]
v2_only = [] # v2 right, v3 wrong
v3_only = [] # v3 right, v2 wrong
both_wrong = []
for i in range(len(tests)):
if v2[i]["match"] and not v3[i]["match"]:
v2_only.append((i, tests[i], v3[i]["got"]))
elif v3[i]["match"] and not v2[i]["match"]:
v3_only.append((i, tests[i], v2[i]["got"]))
elif not v2[i]["match"] and not v3[i]["match"]:
both_wrong.append((i, tests[i], v2[i]["got"], v3[i]["got"]))
v2_correct = sum(1 for r in v2 if r["match"])
v3_correct = sum(1 for r in v3 if r["match"])
print(f"LoRA v2: {v2_correct}/{len(tests)} ({round(v2_correct/len(tests)*100,1)}%)")
print(f"LoRA v3: {v3_correct}/{len(tests)} ({round(v3_correct/len(tests)*100,1)}%)")
if v2_only:
print(f"\nv2 correct, v3 wrong ({len(v2_only)}):")
for i, t, got in v2_only:
print(f" {i+1}. \"{t['dictated']}\" → expected \"{t['expected']}\" got \"{got}\"")
if v3_only:
print(f"\nv3 correct, v2 wrong ({len(v3_only)}):")
for i, t, got in v3_only:
print(f" {i+1}. \"{t['dictated']}\" → expected \"{t['expected']}\" got \"{got}\"")
if both_wrong:
print(f"\nBoth wrong ({len(both_wrong)}):")
for i, t, v2_got, v3_got in both_wrong:
print(f" {i+1}. \"{t['dictated']}\"")
print(f" expected: \"{t['expected']}\"")
print(f" v2: \"{v2_got}\"")
print(f" v3: \"{v3_got}\"")
|