smol-neuron-studio / engine /scripts /calibrate_samples.py
lorcapoul's picture
Smol Neuron Studio
0624276 verified
Raw History Blame Contribute Delete
6.92 kB
"""Calibra Samples de steering: qué fuerza hace que un concepto aparezca sin romper la respuesta.
Para cada candidato (las mejores unidades etiquetadas de una búsqueda, o unidades
sueltas) prueba varias fuerzas relativas sobre un conjunto de preguntas neutras
y mide, por pregunta:
- concepto: la respuesta modificada trae palabras del concepto que la original no
tiene (palabras de la etiqueta, de los tokens marcados en sus mejores ejemplos
y de los tokens que promueve);
- fluidez: la respuesta no degenera (pocas repeticiones, vocabulario variado).
Uso:
cd engine && uv run python -m scripts.calibrate_samples --queries city,water,food --out ../data/calibration.json
cd engine && uv run python -m scripts.calibrate_samples --units L23.N25,L22.N333
"""
import argparse
import json
import re
import sys
import time
from collections import Counter
from pathlib import Path
from app.catalog import labeling, search, store
from app.model import load_model, model_id_from_env
from app.steering import Intervention, encode_prompt, greedy_tokens, to_absolute, unit_maxima
ROOT = Path(__file__).resolve().parents[2]
DEFAULT_CATALOG = ROOT / "data" / "catalog.sqlite"
PROMPTS = [
"What should I do today?",
"Tell me about your favorite hobby.",
"Write one sentence about your weekend.",
"What is the best way to learn something new?",
"Describe your ideal day.",
]
# Palabras demasiado genéricas para contar como "el concepto apareció".
GENERIC = {
"about", "after", "before", "their", "there", "these", "those", "which", "where", "while", "with", "would",
"could", "should", "other", "words", "names", "nouns", "verbs", "token", "tokens", "phrase", "first", "piece",
"pieces", "inside", "unit", "label", "every", "thing", "things", "being", "have", "from", "that", "this",
"into", "your", "what", "when", "more", "most", "some", "such", "they", "them", "then", "than", "also",
}
def stem(word: str) -> str:
return search.stem(word.lower())
def content_words(text: str) -> list[str]:
return [stem(w) for w in re.findall(r"[A-Za-z]{4,}", text) if w.lower() not in GENERIC]
def concept_keywords(conn, refs: list[tuple[int, int]]) -> set[str]:
words: set[str] = set()
for layer, unit in refs:
card = store.unit_card(conn, layer, unit)
if card is None:
continue
if card.label:
words.update(content_words(card.label["label"]))
for ex in card.top[:8]:
words.update(content_words(ex.tokens[ex.position]))
for token, _ in card.promoted[:10]:
words.update(content_words(token))
return words
def fluent(text: str) -> bool:
"""Sin degenerar: suficientes palabras, vocabulario variado y sin trigramas repetidos."""
words = re.findall(r"\w+", text.lower())
if len(words) < 8:
return False
if len(set(words)) / len(words) < 0.45:
return False
trigrams = Counter(zip(words, words[1:], words[2:]))
return max(trigrams.values(), default=0) <= 2
def concept_hit(keywords: set[str], original: str, modified: str) -> list[str]:
before = set(content_words(original))
return sorted({w for w in content_words(modified) if w in keywords and w not in before})
def main() -> None:
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
parser.add_argument("--queries", default="", help="búsquedas separadas por coma (top unidades etiquetadas de cada una)")
parser.add_argument("--units", default="", help="unidades sueltas, p. ej. L23.N25,L22.N333")
parser.add_argument("--per-query", type=int, default=3, help="unidades por búsqueda")
parser.add_argument("--min-confidence", type=float, default=0.75)
parser.add_argument("--values", default="1,1.5,2,2.5,3", help="fuerzas relativas a probar")
parser.add_argument("--max-new-tokens", type=int, default=48)
parser.add_argument("--catalog", type=Path, default=DEFAULT_CATALOG)
parser.add_argument("--out", type=Path, help="JSON con todos los resultados")
args = parser.parse_args()
conn = store.open_catalog(args.catalog)
candidates: list[tuple[str, list[tuple[int, int]]]] = []
for q in filter(None, (s.strip() for s in args.queries.split(","))):
found = [r for r in search.search(conn, q, limit=50)
if r["match"] == "label" and r["label"] != "unclear" and (r["confidence"] or 0) >= args.min_confidence]
if found:
candidates.append((q, [(r["layer"], r["unit"]) for r in found[: args.per_query]]))
else:
print(f"(sin unidades para '{q}')", file=sys.stderr)
for u in filter(None, (s.strip() for s in args.units.split(","))):
ref = labeling.parse_unit_id(u)
if ref:
candidates.append((u, [ref]))
if not candidates:
sys.exit("Nada que calibrar: usa --queries o --units")
values = [float(v) for v in args.values.split(",")]
loaded = load_model(model_id_from_env())
tok = loaded.tokenizer
t0 = time.perf_counter()
inputs = {p: encode_prompt(loaded, p, chat=True) for p in PROMPTS}
originals = {p: tok.decode(list(greedy_tokens(loaded, inputs[p], args.max_new_tokens)), skip_special_tokens=True)
for p in PROMPTS}
results = []
for name, refs in candidates:
keywords = concept_keywords(conn, refs)
maxima = unit_maxima(conn, set(refs))
best = None
for v in values:
ivs = to_absolute([Intervention(l, u, "add", v, f"sample:{name}", v) for l, u in refs], maxima)
rows = []
for p in PROMPTS:
text = tok.decode(list(greedy_tokens(loaded, inputs[p], args.max_new_tokens, ivs)), skip_special_tokens=True)
hits = concept_hit(keywords, originals[p], text)
rows.append({"prompt": p, "text": text, "changed": text != originals[p], "fluent": fluent(text), "hits": hits})
score = sum(r["fluent"] and bool(r["hits"]) for r in rows)
entry = {"name": name, "units": [f"L{l}.N{u}" for l, u in refs], "value": v, "score": score,
"fluent": sum(r["fluent"] for r in rows), "changed": sum(r["changed"] for r in rows), "rows": rows}
results.append(entry)
if best is None or score > best["score"]:
best = entry
print(f" {name:<14} {v:>4} concepto+fluido {score}/5 · fluido {entry['fluent']}/5 · cambia {entry['changed']}/5")
print(f"→ {name}: mejor fuerza {best['value']} ({best['score']}/5)\n")
print(f"{len(results)} combinaciones en {time.perf_counter() - t0:.0f} s")
if args.out:
args.out.write_text(json.dumps({"prompts": PROMPTS, "originals": originals, "results": results}, indent=2, ensure_ascii=False))
print(f"Resultados en {args.out}")
if __name__ == "__main__":
main()