"""Construye el catálogo de unidades: fragmentos → traceo por lotes → top-k, estadísticas y logit lens → SQLite. Uso: cd engine && uv run python -m scripts.collect --fragments 500 # ~1,5 min en CPU con 135M cd engine && uv run python -m scripts.collect --fragments 20000 --out ../data/catalog.sqlite cd engine && uv run python -m scripts.collect --texts textos.jsonl # sin red: {"id", "text"} por línea """ import argparse import os import sys import time from datetime import datetime, timezone from pathlib import Path from app.catalog import store from app.catalog.collector import TopExamples, UnitStats, trace_fragments from app.catalog.fragments import FINEWEB_REPO, FINEWEB_SHARD, fineweb_texts, fragment_texts, jsonl_texts from app.catalog.logit_lens import logit_lens from app.model import load_model, model_id_from_env DEFAULT_OUT = Path(__file__).resolve().parents[2] / "data" / "catalog.sqlite" def main() -> None: parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) parser.add_argument("--fragments", type=int, default=4000, help="número de fragmentos (por defecto 4000)") parser.add_argument("--fragment-tokens", type=int, default=64, help="tokens por fragmento (por defecto 64)") parser.add_argument("--max-per-doc", type=int, default=4, help="máximo de fragmentos por documento") parser.add_argument("--batch-size", type=int, default=16) parser.add_argument("--top-k", type=int, default=20, help="ejemplos por unidad y signo") parser.add_argument("--logit-k", type=int, default=10, help="tokens promovidos/suprimidos por unidad") parser.add_argument("--active-ratio", type=float, default=0.25, help="activo = |a| >= ratio × máx |a| de la unidad") parser.add_argument("--texts", type=Path, help="JSONL local en vez de FineWeb-Edu") parser.add_argument("--out", type=Path, default=DEFAULT_OUT) args = parser.parse_args() t0 = time.perf_counter() loaded = load_model(model_id_from_env()) info = loaded.info units = info.units_per_layer[0] if len(set(info.units_per_layer)) != 1: sys.exit("El catálogo necesita el mismo número de unidades en todas las capas") print(f"Modelo {info.model_id}: {info.num_layers} capas × {units} unidades ({info.device})") source = str(args.texts) if args.texts else f"{FINEWEB_REPO}/{FINEWEB_SHARD}" texts = jsonl_texts(args.texts) if args.texts else fineweb_texts() fragments = fragment_texts(texts, loaded.tokenizer, args.fragment_tokens, args.fragments, args.max_per_doc) if not fragments: sys.exit("No hay fragmentos: los textos son más cortos que --fragment-tokens") n_tokens = len(fragments) * args.fragment_tokens print(f"{len(fragments)} fragmentos de {args.fragment_tokens} tokens ({n_tokens:,} tokens) de {source}") top = TopExamples(info.num_layers, units, k=args.top_k) stats = UnitStats(info.num_layers, units, active_ratio=args.active_ratio) t_trace = time.perf_counter() def progress(done: int, total: int) -> None: elapsed = time.perf_counter() - t_trace rate = done * args.fragment_tokens / elapsed eta = (total - done) * args.fragment_tokens / rate print(f"\r traceo {done}/{total} fragmentos · {rate:,.0f} tokens/s · faltan {eta:,.0f} s ", end="", flush=True) trace_fragments(loaded, fragments, args.batch_size, [top, stats], progress) print(f"\n traceo listo en {time.perf_counter() - t_trace:.0f} s") t_save = time.perf_counter() # Escribir en un temporal y reemplazar al final: una corrida a medias no deja un catálogo roto. args.out.parent.mkdir(parents=True, exist_ok=True) tmp = args.out.with_suffix(".tmp") conn = store.create(tmp) tokenizer = loaded.tokenizer decode_cache: dict[int, str] = {} def decode(token_id: int) -> str: if token_id not in decode_cache: decode_cache[token_id] = tokenizer.decode([token_id]) return decode_cache[token_id] store.write_meta( conn, { "model_id": info.model_id, "layers": info.num_layers, "units": units, "source": source, "fragments": len(fragments), "fragment_tokens": args.fragment_tokens, "tokens": n_tokens, "skipped_positions": "0 (sumidero de atención)", "top_k": args.top_k, "logit_k": args.logit_k, "active_ratio": args.active_ratio, "created_at": datetime.now(timezone.utc).isoformat(timespec="seconds"), }, ) store.write_fragments( conn, [(i, f.doc_id, [decode(t) for t in f.token_ids], f.token_ids) for i, f in enumerate(fragments)] ) for layer in range(info.num_layers): print(f"\r guardando capa {layer + 1}/{info.num_layers} (con logit lens) ", end="", flush=True) store.write_layer( conn, layer, stats.result(layer), {1: top.result(layer, 1), -1: top.result(layer, -1)}, logit_lens(loaded, layer, k=args.logit_k), decode, ) conn.commit() conn.close() if args.out.exists(): old = store.open_catalog(args.out) n_labels = len(store.labeled_units(old)) old.close() if n_labels: print(f"\n aviso: el catálogo anterior tenía {n_labels} etiquetas; reimpórtalas con " "`uv run python -m scripts.import_labels --all`") os.replace(tmp, args.out) print(f"\n guardado listo en {time.perf_counter() - t_save:.0f} s") print(f"Catálogo en {args.out} ({args.out.stat().st_size / 2**20:.1f} MB) en {time.perf_counter() - t0:.0f} s") if __name__ == "__main__": main()