Spaces:
Sleeping
Sleeping
Download engine/scripts/export_labeling.py from lorcapoul/smol-neuron-studio: direct link, hf CLI and curl.
- Browser
- Download file 3.91 kB
-
https://huggingface.co/spaces/lorcapoul/smol-neuron-studio/resolve/main/engine/scripts/export_labeling.py
- Command line
-
hf download hf://spaces/lorcapoul/smol-neuron-studio/engine/scripts/export_labeling.py
-
curl -L -o export_labeling.py https://huggingface.co/spaces/lorcapoul/smol-neuron-studio/resolve/main/engine/scripts/export_labeling.py
3.91 kB
| """Exporta lotes de unidades por etiquetar a data/labeling/batch_XXX.md. | |
| Toma las unidades en orden de prioridad (capas del medio y densidad útil | |
| primero) y salta las ya etiquetadas o ya exportadas. Criterio: docs/labeling.md. | |
| Uso: | |
| cd engine && uv run python -m scripts.export_labeling [--batches 2] [--batch-size 25] | |
| cd engine && uv run python -m scripts.export_labeling --prefer concepts --layers 18-24 | |
| """ | |
| import argparse | |
| import sys | |
| from pathlib import Path | |
| from app.catalog import labeling, store | |
| ROOT = Path(__file__).resolve().parents[2] | |
| DEFAULT_CATALOG = ROOT / "data" / "catalog.sqlite" | |
| DEFAULT_FOLDER = ROOT / "data" / "labeling" | |
| LABELING_DOC = ROOT / "docs" / "labeling.md" | |
| def main() -> None: | |
| parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) | |
| parser.add_argument("--batches", type=int, default=1) | |
| parser.add_argument("--batch-size", type=int, default=25) | |
| parser.add_argument("--examples", type=int, default=8, help="ejemplos por unidad") | |
| parser.add_argument("--min-density", type=float, default=0.001, help="frac_active mínima") | |
| parser.add_argument("--max-density", type=float, default=0.05, help="frac_active máxima") | |
| parser.add_argument( | |
| "--min-max", | |
| type=float, | |
| default=6.0, | |
| help="activación máxima mínima (por defecto 6): por debajo los ejemplos suelen ser ruido", | |
| ) | |
| parser.add_argument( | |
| "--prefer", | |
| choices=["sharp", "concepts"], | |
| default="sharp", | |
| help="sharp: se disparan fuerte y rara vez (suelen ser de token/sintaxis); " | |
| "concepts: tokens marcados distintos, de documentos distintos (suelen ser temáticas)", | |
| ) | |
| parser.add_argument( | |
| "--min-concept", type=float, default=0.0, help="con --prefer concepts: puntaje conceptual mínimo (p. ej. 2.5)" | |
| ) | |
| parser.add_argument("--layers", type=labeling.parse_layers, help="solo estas capas, p. ej. 18-24") | |
| parser.add_argument("--catalog", type=Path, default=DEFAULT_CATALOG) | |
| parser.add_argument("--folder", type=Path, default=DEFAULT_FOLDER) | |
| args = parser.parse_args() | |
| if not args.catalog.exists(): | |
| sys.exit(f"No existe {args.catalog}: corre antes `uv run python -m scripts.collect`") | |
| conn = store.open_catalog(args.catalog) | |
| args.folder.mkdir(parents=True, exist_ok=True) | |
| version = labeling.prompt_version(LABELING_DOC) | |
| exclude = store.labeled_units(conn) | labeling.exported_units(args.folder) | |
| queue = labeling.candidates( | |
| conn, exclude, args.min_density, args.max_density, args.examples, | |
| prefer=args.prefer, layers=args.layers, min_max=args.min_max, min_concept=args.min_concept, | |
| ) | |
| layers_text = f"capas {args.layers.start}–{args.layers.stop - 1}" if args.layers else "capas del medio primero" | |
| selection = ( | |
| f"--prefer {args.prefer} · {layers_text} · frac_active {args.min_density:g}–{args.max_density:g}" | |
| f" · max ≥ {args.min_max:g}" | |
| + (f" · concept ≥ {args.min_concept:g}" if args.prefer == "concepts" and args.min_concept else "") | |
| ) | |
| if not queue: | |
| sys.exit("No quedan unidades que cumplan los filtros") | |
| number = labeling.next_batch_number(args.folder) | |
| for b in range(args.batches): | |
| units = queue[b * args.batch_size : (b + 1) * args.batch_size] | |
| if not units: | |
| break | |
| name = f"batch_{number + b:03d}" | |
| path = args.folder / f"{name}.md" | |
| path.write_text( | |
| labeling.render_batch(conn, units, name, version, args.examples, selection=selection), encoding="utf-8" | |
| ) | |
| layers = sorted({layer for layer, _ in units}) | |
| print(f"{path}: {len(units)} unidades (capas {layers[0]}–{layers[-1]})") | |
| print(f"Quedan {max(0, len(queue) - args.batches * args.batch_size)} candidatas con estos filtros") | |
| if __name__ == "__main__": | |
| main() | |