Spaces:
Sleeping
Sleeping
Download engine/scripts/collect.py from lorcapoul/smol-neuron-studio: direct link, hf CLI and curl.
- Browser
- Download file 5.79 kB
-
https://huggingface.co/spaces/lorcapoul/smol-neuron-studio/resolve/main/engine/scripts/collect.py
- Command line
-
hf download hf://spaces/lorcapoul/smol-neuron-studio/engine/scripts/collect.py
-
curl -L -o collect.py https://huggingface.co/spaces/lorcapoul/smol-neuron-studio/resolve/main/engine/scripts/collect.py
5.79 kB
| """Construye el catálogo de unidades: fragmentos → traceo por lotes → top-k, estadísticas y logit lens → SQLite. | |
| Uso: | |
| cd engine && uv run python -m scripts.collect --fragments 500 # ~1,5 min en CPU con 135M | |
| cd engine && uv run python -m scripts.collect --fragments 20000 --out ../data/catalog.sqlite | |
| cd engine && uv run python -m scripts.collect --texts textos.jsonl # sin red: {"id", "text"} por línea | |
| """ | |
| import argparse | |
| import os | |
| import sys | |
| import time | |
| from datetime import datetime, timezone | |
| from pathlib import Path | |
| from app.catalog import store | |
| from app.catalog.collector import TopExamples, UnitStats, trace_fragments | |
| from app.catalog.fragments import FINEWEB_REPO, FINEWEB_SHARD, fineweb_texts, fragment_texts, jsonl_texts | |
| from app.catalog.logit_lens import logit_lens | |
| from app.model import load_model, model_id_from_env | |
| DEFAULT_OUT = Path(__file__).resolve().parents[2] / "data" / "catalog.sqlite" | |
| def main() -> None: | |
| parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) | |
| parser.add_argument("--fragments", type=int, default=4000, help="número de fragmentos (por defecto 4000)") | |
| parser.add_argument("--fragment-tokens", type=int, default=64, help="tokens por fragmento (por defecto 64)") | |
| parser.add_argument("--max-per-doc", type=int, default=4, help="máximo de fragmentos por documento") | |
| parser.add_argument("--batch-size", type=int, default=16) | |
| parser.add_argument("--top-k", type=int, default=20, help="ejemplos por unidad y signo") | |
| parser.add_argument("--logit-k", type=int, default=10, help="tokens promovidos/suprimidos por unidad") | |
| parser.add_argument("--active-ratio", type=float, default=0.25, help="activo = |a| >= ratio × máx |a| de la unidad") | |
| parser.add_argument("--texts", type=Path, help="JSONL local en vez de FineWeb-Edu") | |
| parser.add_argument("--out", type=Path, default=DEFAULT_OUT) | |
| args = parser.parse_args() | |
| t0 = time.perf_counter() | |
| loaded = load_model(model_id_from_env()) | |
| info = loaded.info | |
| units = info.units_per_layer[0] | |
| if len(set(info.units_per_layer)) != 1: | |
| sys.exit("El catálogo necesita el mismo número de unidades en todas las capas") | |
| print(f"Modelo {info.model_id}: {info.num_layers} capas × {units} unidades ({info.device})") | |
| source = str(args.texts) if args.texts else f"{FINEWEB_REPO}/{FINEWEB_SHARD}" | |
| texts = jsonl_texts(args.texts) if args.texts else fineweb_texts() | |
| fragments = fragment_texts(texts, loaded.tokenizer, args.fragment_tokens, args.fragments, args.max_per_doc) | |
| if not fragments: | |
| sys.exit("No hay fragmentos: los textos son más cortos que --fragment-tokens") | |
| n_tokens = len(fragments) * args.fragment_tokens | |
| print(f"{len(fragments)} fragmentos de {args.fragment_tokens} tokens ({n_tokens:,} tokens) de {source}") | |
| top = TopExamples(info.num_layers, units, k=args.top_k) | |
| stats = UnitStats(info.num_layers, units, active_ratio=args.active_ratio) | |
| t_trace = time.perf_counter() | |
| def progress(done: int, total: int) -> None: | |
| elapsed = time.perf_counter() - t_trace | |
| rate = done * args.fragment_tokens / elapsed | |
| eta = (total - done) * args.fragment_tokens / rate | |
| print(f"\r traceo {done}/{total} fragmentos · {rate:,.0f} tokens/s · faltan {eta:,.0f} s ", end="", flush=True) | |
| trace_fragments(loaded, fragments, args.batch_size, [top, stats], progress) | |
| print(f"\n traceo listo en {time.perf_counter() - t_trace:.0f} s") | |
| t_save = time.perf_counter() | |
| # Escribir en un temporal y reemplazar al final: una corrida a medias no deja un catálogo roto. | |
| args.out.parent.mkdir(parents=True, exist_ok=True) | |
| tmp = args.out.with_suffix(".tmp") | |
| conn = store.create(tmp) | |
| tokenizer = loaded.tokenizer | |
| decode_cache: dict[int, str] = {} | |
| def decode(token_id: int) -> str: | |
| if token_id not in decode_cache: | |
| decode_cache[token_id] = tokenizer.decode([token_id]) | |
| return decode_cache[token_id] | |
| store.write_meta( | |
| conn, | |
| { | |
| "model_id": info.model_id, | |
| "layers": info.num_layers, | |
| "units": units, | |
| "source": source, | |
| "fragments": len(fragments), | |
| "fragment_tokens": args.fragment_tokens, | |
| "tokens": n_tokens, | |
| "skipped_positions": "0 (sumidero de atención)", | |
| "top_k": args.top_k, | |
| "logit_k": args.logit_k, | |
| "active_ratio": args.active_ratio, | |
| "created_at": datetime.now(timezone.utc).isoformat(timespec="seconds"), | |
| }, | |
| ) | |
| store.write_fragments( | |
| conn, [(i, f.doc_id, [decode(t) for t in f.token_ids], f.token_ids) for i, f in enumerate(fragments)] | |
| ) | |
| for layer in range(info.num_layers): | |
| print(f"\r guardando capa {layer + 1}/{info.num_layers} (con logit lens) ", end="", flush=True) | |
| store.write_layer( | |
| conn, | |
| layer, | |
| stats.result(layer), | |
| {1: top.result(layer, 1), -1: top.result(layer, -1)}, | |
| logit_lens(loaded, layer, k=args.logit_k), | |
| decode, | |
| ) | |
| conn.commit() | |
| conn.close() | |
| if args.out.exists(): | |
| old = store.open_catalog(args.out) | |
| n_labels = len(store.labeled_units(old)) | |
| old.close() | |
| if n_labels: | |
| print(f"\n aviso: el catálogo anterior tenía {n_labels} etiquetas; reimpórtalas con " | |
| "`uv run python -m scripts.import_labels --all`") | |
| os.replace(tmp, args.out) | |
| print(f"\n guardado listo en {time.perf_counter() - t_save:.0f} s") | |
| print(f"Catálogo en {args.out} ({args.out.stat().st_size / 2**20:.1f} MB) en {time.perf_counter() - t0:.0f} s") | |
| if __name__ == "__main__": | |
| main() | |