Spaces:
Sleeping
Sleeping
Download engine/scripts/calibrate_samples.py from lorcapoul/smol-neuron-studio: direct link, hf CLI and curl.
- Browser
- Download file 6.92 kB
-
https://huggingface.co/spaces/lorcapoul/smol-neuron-studio/resolve/main/engine/scripts/calibrate_samples.py
- Command line
-
hf download hf://spaces/lorcapoul/smol-neuron-studio/engine/scripts/calibrate_samples.py
-
curl -L -o calibrate_samples.py https://huggingface.co/spaces/lorcapoul/smol-neuron-studio/resolve/main/engine/scripts/calibrate_samples.py
6.92 kB
| """Calibra Samples de steering: qué fuerza hace que un concepto aparezca sin romper la respuesta. | |
| Para cada candidato (las mejores unidades etiquetadas de una búsqueda, o unidades | |
| sueltas) prueba varias fuerzas relativas sobre un conjunto de preguntas neutras | |
| y mide, por pregunta: | |
| - concepto: la respuesta modificada trae palabras del concepto que la original no | |
| tiene (palabras de la etiqueta, de los tokens marcados en sus mejores ejemplos | |
| y de los tokens que promueve); | |
| - fluidez: la respuesta no degenera (pocas repeticiones, vocabulario variado). | |
| Uso: | |
| cd engine && uv run python -m scripts.calibrate_samples --queries city,water,food --out ../data/calibration.json | |
| cd engine && uv run python -m scripts.calibrate_samples --units L23.N25,L22.N333 | |
| """ | |
| import argparse | |
| import json | |
| import re | |
| import sys | |
| import time | |
| from collections import Counter | |
| from pathlib import Path | |
| from app.catalog import labeling, search, store | |
| from app.model import load_model, model_id_from_env | |
| from app.steering import Intervention, encode_prompt, greedy_tokens, to_absolute, unit_maxima | |
| ROOT = Path(__file__).resolve().parents[2] | |
| DEFAULT_CATALOG = ROOT / "data" / "catalog.sqlite" | |
| PROMPTS = [ | |
| "What should I do today?", | |
| "Tell me about your favorite hobby.", | |
| "Write one sentence about your weekend.", | |
| "What is the best way to learn something new?", | |
| "Describe your ideal day.", | |
| ] | |
| # Palabras demasiado genéricas para contar como "el concepto apareció". | |
| GENERIC = { | |
| "about", "after", "before", "their", "there", "these", "those", "which", "where", "while", "with", "would", | |
| "could", "should", "other", "words", "names", "nouns", "verbs", "token", "tokens", "phrase", "first", "piece", | |
| "pieces", "inside", "unit", "label", "every", "thing", "things", "being", "have", "from", "that", "this", | |
| "into", "your", "what", "when", "more", "most", "some", "such", "they", "them", "then", "than", "also", | |
| } | |
| def stem(word: str) -> str: | |
| return search.stem(word.lower()) | |
| def content_words(text: str) -> list[str]: | |
| return [stem(w) for w in re.findall(r"[A-Za-z]{4,}", text) if w.lower() not in GENERIC] | |
| def concept_keywords(conn, refs: list[tuple[int, int]]) -> set[str]: | |
| words: set[str] = set() | |
| for layer, unit in refs: | |
| card = store.unit_card(conn, layer, unit) | |
| if card is None: | |
| continue | |
| if card.label: | |
| words.update(content_words(card.label["label"])) | |
| for ex in card.top[:8]: | |
| words.update(content_words(ex.tokens[ex.position])) | |
| for token, _ in card.promoted[:10]: | |
| words.update(content_words(token)) | |
| return words | |
| def fluent(text: str) -> bool: | |
| """Sin degenerar: suficientes palabras, vocabulario variado y sin trigramas repetidos.""" | |
| words = re.findall(r"\w+", text.lower()) | |
| if len(words) < 8: | |
| return False | |
| if len(set(words)) / len(words) < 0.45: | |
| return False | |
| trigrams = Counter(zip(words, words[1:], words[2:])) | |
| return max(trigrams.values(), default=0) <= 2 | |
| def concept_hit(keywords: set[str], original: str, modified: str) -> list[str]: | |
| before = set(content_words(original)) | |
| return sorted({w for w in content_words(modified) if w in keywords and w not in before}) | |
| def main() -> None: | |
| parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) | |
| parser.add_argument("--queries", default="", help="búsquedas separadas por coma (top unidades etiquetadas de cada una)") | |
| parser.add_argument("--units", default="", help="unidades sueltas, p. ej. L23.N25,L22.N333") | |
| parser.add_argument("--per-query", type=int, default=3, help="unidades por búsqueda") | |
| parser.add_argument("--min-confidence", type=float, default=0.75) | |
| parser.add_argument("--values", default="1,1.5,2,2.5,3", help="fuerzas relativas a probar") | |
| parser.add_argument("--max-new-tokens", type=int, default=48) | |
| parser.add_argument("--catalog", type=Path, default=DEFAULT_CATALOG) | |
| parser.add_argument("--out", type=Path, help="JSON con todos los resultados") | |
| args = parser.parse_args() | |
| conn = store.open_catalog(args.catalog) | |
| candidates: list[tuple[str, list[tuple[int, int]]]] = [] | |
| for q in filter(None, (s.strip() for s in args.queries.split(","))): | |
| found = [r for r in search.search(conn, q, limit=50) | |
| if r["match"] == "label" and r["label"] != "unclear" and (r["confidence"] or 0) >= args.min_confidence] | |
| if found: | |
| candidates.append((q, [(r["layer"], r["unit"]) for r in found[: args.per_query]])) | |
| else: | |
| print(f"(sin unidades para '{q}')", file=sys.stderr) | |
| for u in filter(None, (s.strip() for s in args.units.split(","))): | |
| ref = labeling.parse_unit_id(u) | |
| if ref: | |
| candidates.append((u, [ref])) | |
| if not candidates: | |
| sys.exit("Nada que calibrar: usa --queries o --units") | |
| values = [float(v) for v in args.values.split(",")] | |
| loaded = load_model(model_id_from_env()) | |
| tok = loaded.tokenizer | |
| t0 = time.perf_counter() | |
| inputs = {p: encode_prompt(loaded, p, chat=True) for p in PROMPTS} | |
| originals = {p: tok.decode(list(greedy_tokens(loaded, inputs[p], args.max_new_tokens)), skip_special_tokens=True) | |
| for p in PROMPTS} | |
| results = [] | |
| for name, refs in candidates: | |
| keywords = concept_keywords(conn, refs) | |
| maxima = unit_maxima(conn, set(refs)) | |
| best = None | |
| for v in values: | |
| ivs = to_absolute([Intervention(l, u, "add", v, f"sample:{name}", v) for l, u in refs], maxima) | |
| rows = [] | |
| for p in PROMPTS: | |
| text = tok.decode(list(greedy_tokens(loaded, inputs[p], args.max_new_tokens, ivs)), skip_special_tokens=True) | |
| hits = concept_hit(keywords, originals[p], text) | |
| rows.append({"prompt": p, "text": text, "changed": text != originals[p], "fluent": fluent(text), "hits": hits}) | |
| score = sum(r["fluent"] and bool(r["hits"]) for r in rows) | |
| entry = {"name": name, "units": [f"L{l}.N{u}" for l, u in refs], "value": v, "score": score, | |
| "fluent": sum(r["fluent"] for r in rows), "changed": sum(r["changed"] for r in rows), "rows": rows} | |
| results.append(entry) | |
| if best is None or score > best["score"]: | |
| best = entry | |
| print(f" {name:<14} {v:>4} concepto+fluido {score}/5 · fluido {entry['fluent']}/5 · cambia {entry['changed']}/5") | |
| print(f"→ {name}: mejor fuerza {best['value']} ({best['score']}/5)\n") | |
| print(f"{len(results)} combinaciones en {time.perf_counter() - t0:.0f} s") | |
| if args.out: | |
| args.out.write_text(json.dumps({"prompts": PROMPTS, "originals": originals, "results": results}, indent=2, ensure_ascii=False)) | |
| print(f"Resultados en {args.out}") | |
| if __name__ == "__main__": | |
| main() | |