lorcapoul's picture
Smol Neuron Studio
0624276 verified
Raw History Blame Contribute Delete
5.79 kB
"""Construye el catálogo de unidades: fragmentos → traceo por lotes → top-k, estadísticas y logit lens → SQLite.
Uso:
cd engine && uv run python -m scripts.collect --fragments 500 # ~1,5 min en CPU con 135M
cd engine && uv run python -m scripts.collect --fragments 20000 --out ../data/catalog.sqlite
cd engine && uv run python -m scripts.collect --texts textos.jsonl # sin red: {"id", "text"} por línea
"""
import argparse
import os
import sys
import time
from datetime import datetime, timezone
from pathlib import Path
from app.catalog import store
from app.catalog.collector import TopExamples, UnitStats, trace_fragments
from app.catalog.fragments import FINEWEB_REPO, FINEWEB_SHARD, fineweb_texts, fragment_texts, jsonl_texts
from app.catalog.logit_lens import logit_lens
from app.model import load_model, model_id_from_env
DEFAULT_OUT = Path(__file__).resolve().parents[2] / "data" / "catalog.sqlite"
def main() -> None:
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
parser.add_argument("--fragments", type=int, default=4000, help="número de fragmentos (por defecto 4000)")
parser.add_argument("--fragment-tokens", type=int, default=64, help="tokens por fragmento (por defecto 64)")
parser.add_argument("--max-per-doc", type=int, default=4, help="máximo de fragmentos por documento")
parser.add_argument("--batch-size", type=int, default=16)
parser.add_argument("--top-k", type=int, default=20, help="ejemplos por unidad y signo")
parser.add_argument("--logit-k", type=int, default=10, help="tokens promovidos/suprimidos por unidad")
parser.add_argument("--active-ratio", type=float, default=0.25, help="activo = |a| >= ratio × máx |a| de la unidad")
parser.add_argument("--texts", type=Path, help="JSONL local en vez de FineWeb-Edu")
parser.add_argument("--out", type=Path, default=DEFAULT_OUT)
args = parser.parse_args()
t0 = time.perf_counter()
loaded = load_model(model_id_from_env())
info = loaded.info
units = info.units_per_layer[0]
if len(set(info.units_per_layer)) != 1:
sys.exit("El catálogo necesita el mismo número de unidades en todas las capas")
print(f"Modelo {info.model_id}: {info.num_layers} capas × {units} unidades ({info.device})")
source = str(args.texts) if args.texts else f"{FINEWEB_REPO}/{FINEWEB_SHARD}"
texts = jsonl_texts(args.texts) if args.texts else fineweb_texts()
fragments = fragment_texts(texts, loaded.tokenizer, args.fragment_tokens, args.fragments, args.max_per_doc)
if not fragments:
sys.exit("No hay fragmentos: los textos son más cortos que --fragment-tokens")
n_tokens = len(fragments) * args.fragment_tokens
print(f"{len(fragments)} fragmentos de {args.fragment_tokens} tokens ({n_tokens:,} tokens) de {source}")
top = TopExamples(info.num_layers, units, k=args.top_k)
stats = UnitStats(info.num_layers, units, active_ratio=args.active_ratio)
t_trace = time.perf_counter()
def progress(done: int, total: int) -> None:
elapsed = time.perf_counter() - t_trace
rate = done * args.fragment_tokens / elapsed
eta = (total - done) * args.fragment_tokens / rate
print(f"\r traceo {done}/{total} fragmentos · {rate:,.0f} tokens/s · faltan {eta:,.0f} s ", end="", flush=True)
trace_fragments(loaded, fragments, args.batch_size, [top, stats], progress)
print(f"\n traceo listo en {time.perf_counter() - t_trace:.0f} s")
t_save = time.perf_counter()
# Escribir en un temporal y reemplazar al final: una corrida a medias no deja un catálogo roto.
args.out.parent.mkdir(parents=True, exist_ok=True)
tmp = args.out.with_suffix(".tmp")
conn = store.create(tmp)
tokenizer = loaded.tokenizer
decode_cache: dict[int, str] = {}
def decode(token_id: int) -> str:
if token_id not in decode_cache:
decode_cache[token_id] = tokenizer.decode([token_id])
return decode_cache[token_id]
store.write_meta(
conn,
{
"model_id": info.model_id,
"layers": info.num_layers,
"units": units,
"source": source,
"fragments": len(fragments),
"fragment_tokens": args.fragment_tokens,
"tokens": n_tokens,
"skipped_positions": "0 (sumidero de atención)",
"top_k": args.top_k,
"logit_k": args.logit_k,
"active_ratio": args.active_ratio,
"created_at": datetime.now(timezone.utc).isoformat(timespec="seconds"),
},
)
store.write_fragments(
conn, [(i, f.doc_id, [decode(t) for t in f.token_ids], f.token_ids) for i, f in enumerate(fragments)]
)
for layer in range(info.num_layers):
print(f"\r guardando capa {layer + 1}/{info.num_layers} (con logit lens) ", end="", flush=True)
store.write_layer(
conn,
layer,
stats.result(layer),
{1: top.result(layer, 1), -1: top.result(layer, -1)},
logit_lens(loaded, layer, k=args.logit_k),
decode,
)
conn.commit()
conn.close()
if args.out.exists():
old = store.open_catalog(args.out)
n_labels = len(store.labeled_units(old))
old.close()
if n_labels:
print(f"\n aviso: el catálogo anterior tenía {n_labels} etiquetas; reimpórtalas con "
"`uv run python -m scripts.import_labels --all`")
os.replace(tmp, args.out)
print(f"\n guardado listo en {time.perf_counter() - t_save:.0f} s")
print(f"Catálogo en {args.out} ({args.out.stat().st_size / 2**20:.1f} MB) en {time.perf_counter() - t0:.0f} s")
if __name__ == "__main__":
main()