File size: 6,919 Bytes
0624276
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
"""Calibra Samples de steering: qué fuerza hace que un concepto aparezca sin romper la respuesta.

Para cada candidato (las mejores unidades etiquetadas de una búsqueda, o unidades
sueltas) prueba varias fuerzas relativas sobre un conjunto de preguntas neutras
y mide, por pregunta:

- concepto: la respuesta modificada trae palabras del concepto que la original no
  tiene (palabras de la etiqueta, de los tokens marcados en sus mejores ejemplos
  y de los tokens que promueve);
- fluidez: la respuesta no degenera (pocas repeticiones, vocabulario variado).

Uso:
  cd engine && uv run python -m scripts.calibrate_samples --queries city,water,food --out ../data/calibration.json
  cd engine && uv run python -m scripts.calibrate_samples --units L23.N25,L22.N333
"""

import argparse
import json
import re
import sys
import time
from collections import Counter
from pathlib import Path

from app.catalog import labeling, search, store
from app.model import load_model, model_id_from_env
from app.steering import Intervention, encode_prompt, greedy_tokens, to_absolute, unit_maxima

ROOT = Path(__file__).resolve().parents[2]
DEFAULT_CATALOG = ROOT / "data" / "catalog.sqlite"

PROMPTS = [
    "What should I do today?",
    "Tell me about your favorite hobby.",
    "Write one sentence about your weekend.",
    "What is the best way to learn something new?",
    "Describe your ideal day.",
]

# Palabras demasiado genéricas para contar como "el concepto apareció".
GENERIC = {
    "about", "after", "before", "their", "there", "these", "those", "which", "where", "while", "with", "would",
    "could", "should", "other", "words", "names", "nouns", "verbs", "token", "tokens", "phrase", "first", "piece",
    "pieces", "inside", "unit", "label", "every", "thing", "things", "being", "have", "from", "that", "this",
    "into", "your", "what", "when", "more", "most", "some", "such", "they", "them", "then", "than", "also",
}


def stem(word: str) -> str:
    return search.stem(word.lower())


def content_words(text: str) -> list[str]:
    return [stem(w) for w in re.findall(r"[A-Za-z]{4,}", text) if w.lower() not in GENERIC]


def concept_keywords(conn, refs: list[tuple[int, int]]) -> set[str]:
    words: set[str] = set()
    for layer, unit in refs:
        card = store.unit_card(conn, layer, unit)
        if card is None:
            continue
        if card.label:
            words.update(content_words(card.label["label"]))
        for ex in card.top[:8]:
            words.update(content_words(ex.tokens[ex.position]))
        for token, _ in card.promoted[:10]:
            words.update(content_words(token))
    return words


def fluent(text: str) -> bool:
    """Sin degenerar: suficientes palabras, vocabulario variado y sin trigramas repetidos."""
    words = re.findall(r"\w+", text.lower())
    if len(words) < 8:
        return False
    if len(set(words)) / len(words) < 0.45:
        return False
    trigrams = Counter(zip(words, words[1:], words[2:]))
    return max(trigrams.values(), default=0) <= 2


def concept_hit(keywords: set[str], original: str, modified: str) -> list[str]:
    before = set(content_words(original))
    return sorted({w for w in content_words(modified) if w in keywords and w not in before})


def main() -> None:
    parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
    parser.add_argument("--queries", default="", help="búsquedas separadas por coma (top unidades etiquetadas de cada una)")
    parser.add_argument("--units", default="", help="unidades sueltas, p. ej. L23.N25,L22.N333")
    parser.add_argument("--per-query", type=int, default=3, help="unidades por búsqueda")
    parser.add_argument("--min-confidence", type=float, default=0.75)
    parser.add_argument("--values", default="1,1.5,2,2.5,3", help="fuerzas relativas a probar")
    parser.add_argument("--max-new-tokens", type=int, default=48)
    parser.add_argument("--catalog", type=Path, default=DEFAULT_CATALOG)
    parser.add_argument("--out", type=Path, help="JSON con todos los resultados")
    args = parser.parse_args()

    conn = store.open_catalog(args.catalog)
    candidates: list[tuple[str, list[tuple[int, int]]]] = []
    for q in filter(None, (s.strip() for s in args.queries.split(","))):
        found = [r for r in search.search(conn, q, limit=50)
                 if r["match"] == "label" and r["label"] != "unclear" and (r["confidence"] or 0) >= args.min_confidence]
        if found:
            candidates.append((q, [(r["layer"], r["unit"]) for r in found[: args.per_query]]))
        else:
            print(f"(sin unidades para '{q}')", file=sys.stderr)
    for u in filter(None, (s.strip() for s in args.units.split(","))):
        ref = labeling.parse_unit_id(u)
        if ref:
            candidates.append((u, [ref]))
    if not candidates:
        sys.exit("Nada que calibrar: usa --queries o --units")

    values = [float(v) for v in args.values.split(",")]
    loaded = load_model(model_id_from_env())
    tok = loaded.tokenizer
    t0 = time.perf_counter()

    inputs = {p: encode_prompt(loaded, p, chat=True) for p in PROMPTS}
    originals = {p: tok.decode(list(greedy_tokens(loaded, inputs[p], args.max_new_tokens)), skip_special_tokens=True)
                 for p in PROMPTS}

    results = []
    for name, refs in candidates:
        keywords = concept_keywords(conn, refs)
        maxima = unit_maxima(conn, set(refs))
        best = None
        for v in values:
            ivs = to_absolute([Intervention(l, u, "add", v, f"sample:{name}", v) for l, u in refs], maxima)
            rows = []
            for p in PROMPTS:
                text = tok.decode(list(greedy_tokens(loaded, inputs[p], args.max_new_tokens, ivs)), skip_special_tokens=True)
                hits = concept_hit(keywords, originals[p], text)
                rows.append({"prompt": p, "text": text, "changed": text != originals[p], "fluent": fluent(text), "hits": hits})
            score = sum(r["fluent"] and bool(r["hits"]) for r in rows)
            entry = {"name": name, "units": [f"L{l}.N{u}" for l, u in refs], "value": v, "score": score,
                     "fluent": sum(r["fluent"] for r in rows), "changed": sum(r["changed"] for r in rows), "rows": rows}
            results.append(entry)
            if best is None or score > best["score"]:
                best = entry
            print(f"  {name:<14} {v:>4}  concepto+fluido {score}/5 · fluido {entry['fluent']}/5 · cambia {entry['changed']}/5")
        print(f"→ {name}: mejor fuerza {best['value']} ({best['score']}/5)\n")

    print(f"{len(results)} combinaciones en {time.perf_counter() - t0:.0f} s")
    if args.out:
        args.out.write_text(json.dumps({"prompts": PROMPTS, "originals": originals, "results": results}, indent=2, ensure_ascii=False))
        print(f"Resultados en {args.out}")


if __name__ == "__main__":
    main()