Spaces:
Sleeping
Sleeping
File size: 6,919 Bytes
0624276 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 | """Calibra Samples de steering: qué fuerza hace que un concepto aparezca sin romper la respuesta.
Para cada candidato (las mejores unidades etiquetadas de una búsqueda, o unidades
sueltas) prueba varias fuerzas relativas sobre un conjunto de preguntas neutras
y mide, por pregunta:
- concepto: la respuesta modificada trae palabras del concepto que la original no
tiene (palabras de la etiqueta, de los tokens marcados en sus mejores ejemplos
y de los tokens que promueve);
- fluidez: la respuesta no degenera (pocas repeticiones, vocabulario variado).
Uso:
cd engine && uv run python -m scripts.calibrate_samples --queries city,water,food --out ../data/calibration.json
cd engine && uv run python -m scripts.calibrate_samples --units L23.N25,L22.N333
"""
import argparse
import json
import re
import sys
import time
from collections import Counter
from pathlib import Path
from app.catalog import labeling, search, store
from app.model import load_model, model_id_from_env
from app.steering import Intervention, encode_prompt, greedy_tokens, to_absolute, unit_maxima
ROOT = Path(__file__).resolve().parents[2]
DEFAULT_CATALOG = ROOT / "data" / "catalog.sqlite"
PROMPTS = [
"What should I do today?",
"Tell me about your favorite hobby.",
"Write one sentence about your weekend.",
"What is the best way to learn something new?",
"Describe your ideal day.",
]
# Palabras demasiado genéricas para contar como "el concepto apareció".
GENERIC = {
"about", "after", "before", "their", "there", "these", "those", "which", "where", "while", "with", "would",
"could", "should", "other", "words", "names", "nouns", "verbs", "token", "tokens", "phrase", "first", "piece",
"pieces", "inside", "unit", "label", "every", "thing", "things", "being", "have", "from", "that", "this",
"into", "your", "what", "when", "more", "most", "some", "such", "they", "them", "then", "than", "also",
}
def stem(word: str) -> str:
return search.stem(word.lower())
def content_words(text: str) -> list[str]:
return [stem(w) for w in re.findall(r"[A-Za-z]{4,}", text) if w.lower() not in GENERIC]
def concept_keywords(conn, refs: list[tuple[int, int]]) -> set[str]:
words: set[str] = set()
for layer, unit in refs:
card = store.unit_card(conn, layer, unit)
if card is None:
continue
if card.label:
words.update(content_words(card.label["label"]))
for ex in card.top[:8]:
words.update(content_words(ex.tokens[ex.position]))
for token, _ in card.promoted[:10]:
words.update(content_words(token))
return words
def fluent(text: str) -> bool:
"""Sin degenerar: suficientes palabras, vocabulario variado y sin trigramas repetidos."""
words = re.findall(r"\w+", text.lower())
if len(words) < 8:
return False
if len(set(words)) / len(words) < 0.45:
return False
trigrams = Counter(zip(words, words[1:], words[2:]))
return max(trigrams.values(), default=0) <= 2
def concept_hit(keywords: set[str], original: str, modified: str) -> list[str]:
before = set(content_words(original))
return sorted({w for w in content_words(modified) if w in keywords and w not in before})
def main() -> None:
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
parser.add_argument("--queries", default="", help="búsquedas separadas por coma (top unidades etiquetadas de cada una)")
parser.add_argument("--units", default="", help="unidades sueltas, p. ej. L23.N25,L22.N333")
parser.add_argument("--per-query", type=int, default=3, help="unidades por búsqueda")
parser.add_argument("--min-confidence", type=float, default=0.75)
parser.add_argument("--values", default="1,1.5,2,2.5,3", help="fuerzas relativas a probar")
parser.add_argument("--max-new-tokens", type=int, default=48)
parser.add_argument("--catalog", type=Path, default=DEFAULT_CATALOG)
parser.add_argument("--out", type=Path, help="JSON con todos los resultados")
args = parser.parse_args()
conn = store.open_catalog(args.catalog)
candidates: list[tuple[str, list[tuple[int, int]]]] = []
for q in filter(None, (s.strip() for s in args.queries.split(","))):
found = [r for r in search.search(conn, q, limit=50)
if r["match"] == "label" and r["label"] != "unclear" and (r["confidence"] or 0) >= args.min_confidence]
if found:
candidates.append((q, [(r["layer"], r["unit"]) for r in found[: args.per_query]]))
else:
print(f"(sin unidades para '{q}')", file=sys.stderr)
for u in filter(None, (s.strip() for s in args.units.split(","))):
ref = labeling.parse_unit_id(u)
if ref:
candidates.append((u, [ref]))
if not candidates:
sys.exit("Nada que calibrar: usa --queries o --units")
values = [float(v) for v in args.values.split(",")]
loaded = load_model(model_id_from_env())
tok = loaded.tokenizer
t0 = time.perf_counter()
inputs = {p: encode_prompt(loaded, p, chat=True) for p in PROMPTS}
originals = {p: tok.decode(list(greedy_tokens(loaded, inputs[p], args.max_new_tokens)), skip_special_tokens=True)
for p in PROMPTS}
results = []
for name, refs in candidates:
keywords = concept_keywords(conn, refs)
maxima = unit_maxima(conn, set(refs))
best = None
for v in values:
ivs = to_absolute([Intervention(l, u, "add", v, f"sample:{name}", v) for l, u in refs], maxima)
rows = []
for p in PROMPTS:
text = tok.decode(list(greedy_tokens(loaded, inputs[p], args.max_new_tokens, ivs)), skip_special_tokens=True)
hits = concept_hit(keywords, originals[p], text)
rows.append({"prompt": p, "text": text, "changed": text != originals[p], "fluent": fluent(text), "hits": hits})
score = sum(r["fluent"] and bool(r["hits"]) for r in rows)
entry = {"name": name, "units": [f"L{l}.N{u}" for l, u in refs], "value": v, "score": score,
"fluent": sum(r["fluent"] for r in rows), "changed": sum(r["changed"] for r in rows), "rows": rows}
results.append(entry)
if best is None or score > best["score"]:
best = entry
print(f" {name:<14} {v:>4} concepto+fluido {score}/5 · fluido {entry['fluent']}/5 · cambia {entry['changed']}/5")
print(f"→ {name}: mejor fuerza {best['value']} ({best['score']}/5)\n")
print(f"{len(results)} combinaciones en {time.perf_counter() - t0:.0f} s")
if args.out:
args.out.write_text(json.dumps({"prompts": PROMPTS, "originals": originals, "results": results}, indent=2, ensure_ascii=False))
print(f"Resultados en {args.out}")
if __name__ == "__main__":
main()
|