File size: 7,662 Bytes
04558eb | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 | #!/usr/bin/env python3
"""Evaluate fine-tuned LoRA models on expanded bakeoff test set.
30 tests organized by tier:
Tier 1 (1-10): Core functionality — basic symbol subs, casing, paths, URLs
Tier 2 (11-20): Compound patterns — multi-symbol, git commands, env vars, pipes
Tier 3 (21-30): Known failure modes — dot dot, compound &&, numbers, fidelity
"""
import json
import time
from collections import defaultdict
from mlx_lm import load, generate
SYS = "Reconstruct the intended syntax from the dictated text. Output only the result."
tests = [
# --- Tier 1: Core functionality ---
{"id": 1, "cat": "symbols-basic", "dictated": "hello dash world", "expected": "hello-world"},
{"id": 2, "cat": "symbols-basic", "dictated": "hello underscore world", "expected": "hello_world"},
{"id": 3, "cat": "symbols-compound", "dictated": "dash dash verbose", "expected": "--verbose"},
{"id": 4, "cat": "symbols-compound", "dictated": "equals equals equals", "expected": "==="},
{"id": 5, "cat": "casing", "dictated": "camel case get user name", "expected": "getUserName"},
{"id": 6, "cat": "casing", "dictated": "snake case total tokens generated", "expected": "total_tokens_generated"},
{"id": 7, "cat": "casing", "dictated": "kebab case dark mode toggle", "expected": "dark-mode-toggle"},
{"id": 8, "cat": "quotes", "dictated": "quote hello world quote", "expected": "\"hello world\""},
{"id": 9, "cat": "paths", "dictated": "tilde slash dev slash talkie", "expected": "~/dev/talkie"},
{"id": 10, "cat": "urls", "dictated": "HTTPS colon slash slash GitHub dot com slash arach slash talkie", "expected": "https://github.com/arach/talkie"},
# --- Tier 2: Compound patterns ---
{"id": 11, "cat": "mixed", "dictated": "git commit dash M quote fix latency quote", "expected": "git commit -m \"fix latency\""},
{"id": 12, "cat": "mixed", "dictated": "export all caps API underscore KEY equals quote my dash key dash one two three quote", "expected": "export API_KEY=\"my-key-123\""},
{"id": 13, "cat": "mixed", "dictated": "shebang slash bin slash bash", "expected": "#!/bin/bash"},
{"id": 14, "cat": "mixed", "dictated": "docker run dash D dash P eighty eighty colon eighty eighty nginx", "expected": "docker run -d -p 8080:8080 nginx"},
{"id": 15, "cat": "mixed", "dictated": "func camel case view did load open paren close paren", "expected": "func viewDidLoad()"},
{"id": 16, "cat": "mixed", "dictated": "import open brace camel case use state close brace from single quote react single quote", "expected": "import { useState } from 'react'"},
{"id": 17, "cat": "mixed", "dictated": "LS dash L A pipe grep dot swift", "expected": "ls -la | grep .swift"},
{"id": 18, "cat": "mixed", "dictated": "GH PR create dash dash title quote fix inference latency quote dash dash body quote added TTFT tracking and latency instrumentation quote", "expected": "gh pr create --title \"fix inference latency\" --body \"Added TTFT tracking and latency instrumentation\""},
{"id": 19, "cat": "identifiers", "dictated": "dot E N V dot local", "expected": ".env.local"},
{"id": 20, "cat": "operators", "dictated": "open paren X close paren fat arrow open brace close brace", "expected": "(x) => {}"},
# --- Tier 3: Known failure modes ---
{"id": 21, "cat": "symbols-compound", "dictated": "dot dot slash dev", "expected": "../dev"},
{"id": 22, "cat": "paths", "dictated": "dot dot slash dot dot slash dot dot slash", "expected": "../../../"},
{"id": 23, "cat": "paths", "dictated": "dot dot slash configs", "expected": "../configs"},
{"id": 24, "cat": "operators", "dictated": "A and and B and and C", "expected": "a && b && c"},
{"id": 25, "cat": "numbers", "dictated": "zero point seven", "expected": "0.7"},
{"id": 26, "cat": "numbers", "dictated": "one two seven dot zero dot zero dot one", "expected": "127.0.0.1"},
{"id": 27, "cat": "mixed", "dictated": "git add dash A and and git commit dash M quote fix typo quote and and git push", "expected": "git add -A && git commit -m \"fix typo\" && git push"},
{"id": 28, "cat": "spacing", "dictated": "no space git hub", "expected": "github"},
{"id": 29, "cat": "brackets", "dictated": "open bracket colon colon dash one close bracket", "expected": "[::-1]"},
{"id": 30, "cat": "mixed", "dictated": "dash dash temp zero point seven dash dash tokens five twelve", "expected": "--temp 0.7 --tokens 512"},
]
configs = [
{
"label": "QWEN 0.5B + LoRA v3",
"model": "mlx-community/Qwen2.5-0.5B-Instruct-4bit",
"adapter": "/Users/arach/dev/talkie/datasets/finetune/adapters/qwen-0.5b-lora-v3",
},
]
all_results = {}
for cfg in configs:
print(f"\n{'='*60}")
print(f" {cfg['label']}")
print(f"{'='*60}\n")
model, tokenizer = load(cfg["model"], adapter_path=cfg["adapter"])
results = []
total_time = 0
for t in tests:
messages = [
{"role": "system", "content": SYS},
{"role": "user", "content": t["dictated"]},
]
prompt = tokenizer.apply_chat_template(
messages, tokenize=False, add_generation_prompt=True
)
start = time.perf_counter()
got = generate(
model, tokenizer, prompt=prompt,
max_tokens=80, verbose=False
)
elapsed = time.perf_counter() - start
total_time += elapsed
got = got.strip()
match = got == t["expected"]
results.append({"id": t["id"], "cat": t["cat"], "match": match, "got": got})
icon = "\033[32m✓\033[0m" if match else "\033[31m✗\033[0m"
print(f"{icon} {str(t['id']).rjust(2)}. \"{t['dictated']}\"")
print(f" expected: {t['expected']}")
if not match:
print(f" got: {got}")
correct = sum(1 for r in results if r["match"])
pct = round(correct / len(results) * 100)
avg_ms = round(total_time / len(results) * 1000)
print(f"\nScore: {correct}/{len(results)} ({pct}%)")
print(f"Avg latency: {avg_ms}ms per inference")
all_results[cfg["label"]] = results
# --- Per-category breakdown ---
print(f"\nPer-category accuracy:")
cat_results = defaultdict(lambda: {"correct": 0, "total": 0})
for r in results:
cat_results[r["cat"]]["total"] += 1
if r["match"]:
cat_results[r["cat"]]["correct"] += 1
for cat in sorted(cat_results.keys()):
cr = cat_results[cat]
cat_pct = round(cr["correct"] / cr["total"] * 100)
bar = "█" * cr["correct"] + "░" * (cr["total"] - cr["correct"])
print(f" {cat:<20} {cr['correct']}/{cr['total']} ({cat_pct:>3}%) {bar}")
# --- Per-tier breakdown ---
print(f"\nPer-tier accuracy:")
tiers = [
("Tier 1: Core", results[0:10]),
("Tier 2: Compound", results[10:20]),
("Tier 3: Failure modes", results[20:30]),
]
for tier_name, tier_results in tiers:
tier_correct = sum(1 for r in tier_results if r["match"])
tier_pct = round(tier_correct / len(tier_results) * 100)
print(f" {tier_name:<25} {tier_correct}/{len(tier_results)} ({tier_pct}%)")
del model, tokenizer
# Summary
print(f"\n{'='*60}")
print(" RESULTS SUMMARY")
print(f"{'='*60}\n")
for label, results in all_results.items():
correct = sum(1 for r in results if r["match"])
pct = round(correct / len(results) * 100)
print(f" {label}: {correct}/{len(results)} ({pct}%)")
print("\nBaselines:")
print(" LoRA v1 (240 train): 13/15 (87%)")
print(" LoRA v2 (474 train): 27/30 (90%)")
print(" Claude: 14/15 (93%)")
|