"""Calibra Samples de steering: qué fuerza hace que un concepto aparezca sin romper la respuesta. Para cada candidato (las mejores unidades etiquetadas de una búsqueda, o unidades sueltas) prueba varias fuerzas relativas sobre un conjunto de preguntas neutras y mide, por pregunta: - concepto: la respuesta modificada trae palabras del concepto que la original no tiene (palabras de la etiqueta, de los tokens marcados en sus mejores ejemplos y de los tokens que promueve); - fluidez: la respuesta no degenera (pocas repeticiones, vocabulario variado). Uso: cd engine && uv run python -m scripts.calibrate_samples --queries city,water,food --out ../data/calibration.json cd engine && uv run python -m scripts.calibrate_samples --units L23.N25,L22.N333 """ import argparse import json import re import sys import time from collections import Counter from pathlib import Path from app.catalog import labeling, search, store from app.model import load_model, model_id_from_env from app.steering import Intervention, encode_prompt, greedy_tokens, to_absolute, unit_maxima ROOT = Path(__file__).resolve().parents[2] DEFAULT_CATALOG = ROOT / "data" / "catalog.sqlite" PROMPTS = [ "What should I do today?", "Tell me about your favorite hobby.", "Write one sentence about your weekend.", "What is the best way to learn something new?", "Describe your ideal day.", ] # Palabras demasiado genéricas para contar como "el concepto apareció". GENERIC = { "about", "after", "before", "their", "there", "these", "those", "which", "where", "while", "with", "would", "could", "should", "other", "words", "names", "nouns", "verbs", "token", "tokens", "phrase", "first", "piece", "pieces", "inside", "unit", "label", "every", "thing", "things", "being", "have", "from", "that", "this", "into", "your", "what", "when", "more", "most", "some", "such", "they", "them", "then", "than", "also", } def stem(word: str) -> str: return search.stem(word.lower()) def content_words(text: str) -> list[str]: return [stem(w) for w in re.findall(r"[A-Za-z]{4,}", text) if w.lower() not in GENERIC] def concept_keywords(conn, refs: list[tuple[int, int]]) -> set[str]: words: set[str] = set() for layer, unit in refs: card = store.unit_card(conn, layer, unit) if card is None: continue if card.label: words.update(content_words(card.label["label"])) for ex in card.top[:8]: words.update(content_words(ex.tokens[ex.position])) for token, _ in card.promoted[:10]: words.update(content_words(token)) return words def fluent(text: str) -> bool: """Sin degenerar: suficientes palabras, vocabulario variado y sin trigramas repetidos.""" words = re.findall(r"\w+", text.lower()) if len(words) < 8: return False if len(set(words)) / len(words) < 0.45: return False trigrams = Counter(zip(words, words[1:], words[2:])) return max(trigrams.values(), default=0) <= 2 def concept_hit(keywords: set[str], original: str, modified: str) -> list[str]: before = set(content_words(original)) return sorted({w for w in content_words(modified) if w in keywords and w not in before}) def main() -> None: parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) parser.add_argument("--queries", default="", help="búsquedas separadas por coma (top unidades etiquetadas de cada una)") parser.add_argument("--units", default="", help="unidades sueltas, p. ej. L23.N25,L22.N333") parser.add_argument("--per-query", type=int, default=3, help="unidades por búsqueda") parser.add_argument("--min-confidence", type=float, default=0.75) parser.add_argument("--values", default="1,1.5,2,2.5,3", help="fuerzas relativas a probar") parser.add_argument("--max-new-tokens", type=int, default=48) parser.add_argument("--catalog", type=Path, default=DEFAULT_CATALOG) parser.add_argument("--out", type=Path, help="JSON con todos los resultados") args = parser.parse_args() conn = store.open_catalog(args.catalog) candidates: list[tuple[str, list[tuple[int, int]]]] = [] for q in filter(None, (s.strip() for s in args.queries.split(","))): found = [r for r in search.search(conn, q, limit=50) if r["match"] == "label" and r["label"] != "unclear" and (r["confidence"] or 0) >= args.min_confidence] if found: candidates.append((q, [(r["layer"], r["unit"]) for r in found[: args.per_query]])) else: print(f"(sin unidades para '{q}')", file=sys.stderr) for u in filter(None, (s.strip() for s in args.units.split(","))): ref = labeling.parse_unit_id(u) if ref: candidates.append((u, [ref])) if not candidates: sys.exit("Nada que calibrar: usa --queries o --units") values = [float(v) for v in args.values.split(",")] loaded = load_model(model_id_from_env()) tok = loaded.tokenizer t0 = time.perf_counter() inputs = {p: encode_prompt(loaded, p, chat=True) for p in PROMPTS} originals = {p: tok.decode(list(greedy_tokens(loaded, inputs[p], args.max_new_tokens)), skip_special_tokens=True) for p in PROMPTS} results = [] for name, refs in candidates: keywords = concept_keywords(conn, refs) maxima = unit_maxima(conn, set(refs)) best = None for v in values: ivs = to_absolute([Intervention(l, u, "add", v, f"sample:{name}", v) for l, u in refs], maxima) rows = [] for p in PROMPTS: text = tok.decode(list(greedy_tokens(loaded, inputs[p], args.max_new_tokens, ivs)), skip_special_tokens=True) hits = concept_hit(keywords, originals[p], text) rows.append({"prompt": p, "text": text, "changed": text != originals[p], "fluent": fluent(text), "hits": hits}) score = sum(r["fluent"] and bool(r["hits"]) for r in rows) entry = {"name": name, "units": [f"L{l}.N{u}" for l, u in refs], "value": v, "score": score, "fluent": sum(r["fluent"] for r in rows), "changed": sum(r["changed"] for r in rows), "rows": rows} results.append(entry) if best is None or score > best["score"]: best = entry print(f" {name:<14} {v:>4} concepto+fluido {score}/5 · fluido {entry['fluent']}/5 · cambia {entry['changed']}/5") print(f"→ {name}: mejor fuerza {best['value']} ({best['score']}/5)\n") print(f"{len(results)} combinaciones en {time.perf_counter() - t0:.0f} s") if args.out: args.out.write_text(json.dumps({"prompts": PROMPTS, "originals": originals, "results": results}, indent=2, ensure_ascii=False)) print(f"Resultados en {args.out}") if __name__ == "__main__": main()