ATOM (atom-ai): code snapshot (100fae0 + batched-train), README with measured numbers, curated checkpoints (part 4)
b9301f1 verified Download tools/eval_last_atom_ablation.py from thefinalboss/atom-ai: direct link, hf CLI and curl.
- Browser
- Download file 5.08 kB
-
https://huggingface.co/thefinalboss/atom-ai/resolve/main/tools/eval_last_atom_ablation.py
- Command line
-
hf download hf://thefinalboss/atom-ai/tools/eval_last_atom_ablation.py
-
curl -L -o eval_last_atom_ablation.py https://huggingface.co/thefinalboss/atom-ai/resolve/main/tools/eval_last_atom_ablation.py
5.08 kB
| """Val CE with and without the last_atom readout term (field-path-alone diagnostic). | |
| Rebuilds the exact stream-ring validation used by run_atom_native at the end of a | |
| chunk (same shards/chunk-bytes, skip=END-base packets, last 256 ring transitions), | |
| then evaluates each checkpoint twice under eval/no_grad: | |
| normal : logits as trained / as chat sees them | |
| la_zeroed : surface.last_atom_force_off=True (la_byte := 0) — what the field | |
| path (α-MLP + frozen JL [+ payload]) predicts alone. | |
| Usage: | |
| PYTHONPATH=. python tools/eval_last_atom_ablation.py --end-steps 4868000 4876000 \ | |
| --checkpoints A.pt B.pt --out docs/artifacts/last_atom_dropout/val_ablation.json | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import json | |
| import sys | |
| import time | |
| from pathlib import Path | |
| import numpy as np | |
| import torch | |
| sys.path.insert(0, str(Path(__file__).resolve().parents[1])) | |
| sys.path.insert(0, str(Path(__file__).resolve().parent)) | |
| from chat_atom_native import load_model # noqa: E402 | |
| from src.io.atomizer import Atomizer # noqa: E402 | |
| from src.io.stream_corpus import StreamingPacketSource, resolve_shard_paths # noqa: E402 | |
| def build_val_pairs_multi(end_steps, base: int, glob: str, chunk_bytes: int, n_val: int = 256): | |
| """One stream pass; snapshot the ring's last ``n_val`` transitions at each end step.""" | |
| src = StreamingPacketSource( | |
| Atomizer(max_span_bytes=1), | |
| resolve_shard_paths(None, glob), | |
| chunk_bytes=chunk_bytes, | |
| loop=True, | |
| reset_atomizer=True, | |
| ) | |
| out = {} | |
| consumed = 0 | |
| for end in sorted(end_steps): | |
| for _ in range(max(0, end - base) - consumed): | |
| next(src) | |
| consumed = max(consumed, end - base) | |
| pairs = src.validation_transitions() | |
| out[end] = pairs[-n_val:] if len(pairs) > n_val else pairs | |
| return out | |
| def evaluate(model, pairs, force_off: bool) -> dict: | |
| model.surface.last_atom_force_off = bool(force_off) | |
| model.reset_state(reset_atomizer=True) | |
| model.eval() | |
| losses, byte_losses = [], [] | |
| for cur, tgt in pairs: | |
| loss, info = model.transition_loss(cur, tgt) | |
| losses.append(float(loss.item())) | |
| byte_losses.append(float(info["byte_loss"])) | |
| model.surface.last_atom_force_off = False | |
| return { | |
| "loss": float(np.mean(losses)), | |
| "byte_loss": float(np.mean(byte_losses)), | |
| "n": len(losses), | |
| } | |
| def main() -> None: | |
| ap = argparse.ArgumentParser() | |
| ap.add_argument("--end-steps", type=int, nargs="+", required=True) | |
| ap.add_argument("--base", type=int, default=496000) | |
| ap.add_argument("--data-glob", default="data/corpus_fr_medium*.txt") | |
| ap.add_argument("--chunk-bytes", type=int, default=65536) | |
| ap.add_argument("--checkpoints", nargs="+", required=True) | |
| ap.add_argument("--out", default=None) | |
| ap.add_argument( | |
| "--ring-cache", default=None, | |
| help="optional .pt cache of the built rings (keyed by end steps/base/glob/chunk); " | |
| "loaded if present and matching, else built and saved atomically", | |
| ) | |
| args = ap.parse_args() | |
| t0 = time.time() | |
| key = {"end_steps": sorted(args.end_steps), "base": args.base, | |
| "data_glob": args.data_glob, "chunk_bytes": args.chunk_bytes} | |
| rings = None | |
| if args.ring_cache and Path(args.ring_cache).exists(): | |
| blob = torch.load(args.ring_cache, map_location="cpu", weights_only=False) | |
| if blob.get("key") == key: | |
| rings = blob["rings"] | |
| print(f"val rings loaded from cache {args.ring_cache}", flush=True) | |
| if rings is None: | |
| rings = build_val_pairs_multi(args.end_steps, args.base, args.data_glob, args.chunk_bytes) | |
| print(f"val rings built for {sorted(rings)} in {time.time()-t0:.1f}s", flush=True) | |
| if args.ring_cache: | |
| tmp = Path(str(args.ring_cache) + f".tmp{time.time_ns()}") | |
| tmp.parent.mkdir(parents=True, exist_ok=True) | |
| torch.save({"key": key, "rings": rings}, tmp) | |
| tmp.replace(args.ring_cache) | |
| results = {"val_source": "stream_ring_last256 (same as run_atom_native stream val)", "rings": {}} | |
| models = {ck: load_model(ck) for ck in args.checkpoints} | |
| for end, pairs in sorted(rings.items()): | |
| results["rings"][str(end)] = {"n_pairs": len(pairs), "checkpoints": {}} | |
| for ck, model in models.items(): | |
| normal = evaluate(model, pairs, force_off=False) | |
| zeroed = evaluate(model, pairs, force_off=True) | |
| results["rings"][str(end)]["checkpoints"][ck] = {"normal": normal, "la_zeroed": zeroed} | |
| print( | |
| f"[ring@{end}] {ck}\n normal loss={normal['loss']:.4f} byte_ce={normal['byte_loss']:.4f}\n" | |
| f" la_zero loss={zeroed['loss']:.4f} byte_ce={zeroed['byte_loss']:.4f}", | |
| flush=True, | |
| ) | |
| if args.out: | |
| Path(args.out).parent.mkdir(parents=True, exist_ok=True) | |
| Path(args.out).write_text(json.dumps(results, indent=2), encoding="utf-8") | |
| if __name__ == "__main__": | |
| main() | |