atom-ai / tools /eval_last_atom_ablation.py
thefinalboss's picture
ATOM (atom-ai): code snapshot (100fae0 + batched-train), README with measured numbers, curated checkpoints (part 4)
b9301f1 verified
Raw History Blame Contribute Delete
5.08 kB
"""Val CE with and without the last_atom readout term (field-path-alone diagnostic).
Rebuilds the exact stream-ring validation used by run_atom_native at the end of a
chunk (same shards/chunk-bytes, skip=END-base packets, last 256 ring transitions),
then evaluates each checkpoint twice under eval/no_grad:
normal : logits as trained / as chat sees them
la_zeroed : surface.last_atom_force_off=True (la_byte := 0) — what the field
path (α-MLP + frozen JL [+ payload]) predicts alone.
Usage:
PYTHONPATH=. python tools/eval_last_atom_ablation.py --end-steps 4868000 4876000 \
--checkpoints A.pt B.pt --out docs/artifacts/last_atom_dropout/val_ablation.json
"""
from __future__ import annotations
import argparse
import json
import sys
import time
from pathlib import Path
import numpy as np
import torch
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
sys.path.insert(0, str(Path(__file__).resolve().parent))
from chat_atom_native import load_model # noqa: E402
from src.io.atomizer import Atomizer # noqa: E402
from src.io.stream_corpus import StreamingPacketSource, resolve_shard_paths # noqa: E402
def build_val_pairs_multi(end_steps, base: int, glob: str, chunk_bytes: int, n_val: int = 256):
"""One stream pass; snapshot the ring's last ``n_val`` transitions at each end step."""
src = StreamingPacketSource(
Atomizer(max_span_bytes=1),
resolve_shard_paths(None, glob),
chunk_bytes=chunk_bytes,
loop=True,
reset_atomizer=True,
)
out = {}
consumed = 0
for end in sorted(end_steps):
for _ in range(max(0, end - base) - consumed):
next(src)
consumed = max(consumed, end - base)
pairs = src.validation_transitions()
out[end] = pairs[-n_val:] if len(pairs) > n_val else pairs
return out
@torch.no_grad()
def evaluate(model, pairs, force_off: bool) -> dict:
model.surface.last_atom_force_off = bool(force_off)
model.reset_state(reset_atomizer=True)
model.eval()
losses, byte_losses = [], []
for cur, tgt in pairs:
loss, info = model.transition_loss(cur, tgt)
losses.append(float(loss.item()))
byte_losses.append(float(info["byte_loss"]))
model.surface.last_atom_force_off = False
return {
"loss": float(np.mean(losses)),
"byte_loss": float(np.mean(byte_losses)),
"n": len(losses),
}
def main() -> None:
ap = argparse.ArgumentParser()
ap.add_argument("--end-steps", type=int, nargs="+", required=True)
ap.add_argument("--base", type=int, default=496000)
ap.add_argument("--data-glob", default="data/corpus_fr_medium*.txt")
ap.add_argument("--chunk-bytes", type=int, default=65536)
ap.add_argument("--checkpoints", nargs="+", required=True)
ap.add_argument("--out", default=None)
ap.add_argument(
"--ring-cache", default=None,
help="optional .pt cache of the built rings (keyed by end steps/base/glob/chunk); "
"loaded if present and matching, else built and saved atomically",
)
args = ap.parse_args()
t0 = time.time()
key = {"end_steps": sorted(args.end_steps), "base": args.base,
"data_glob": args.data_glob, "chunk_bytes": args.chunk_bytes}
rings = None
if args.ring_cache and Path(args.ring_cache).exists():
blob = torch.load(args.ring_cache, map_location="cpu", weights_only=False)
if blob.get("key") == key:
rings = blob["rings"]
print(f"val rings loaded from cache {args.ring_cache}", flush=True)
if rings is None:
rings = build_val_pairs_multi(args.end_steps, args.base, args.data_glob, args.chunk_bytes)
print(f"val rings built for {sorted(rings)} in {time.time()-t0:.1f}s", flush=True)
if args.ring_cache:
tmp = Path(str(args.ring_cache) + f".tmp{time.time_ns()}")
tmp.parent.mkdir(parents=True, exist_ok=True)
torch.save({"key": key, "rings": rings}, tmp)
tmp.replace(args.ring_cache)
results = {"val_source": "stream_ring_last256 (same as run_atom_native stream val)", "rings": {}}
models = {ck: load_model(ck) for ck in args.checkpoints}
for end, pairs in sorted(rings.items()):
results["rings"][str(end)] = {"n_pairs": len(pairs), "checkpoints": {}}
for ck, model in models.items():
normal = evaluate(model, pairs, force_off=False)
zeroed = evaluate(model, pairs, force_off=True)
results["rings"][str(end)]["checkpoints"][ck] = {"normal": normal, "la_zeroed": zeroed}
print(
f"[ring@{end}] {ck}\n normal loss={normal['loss']:.4f} byte_ce={normal['byte_loss']:.4f}\n"
f" la_zero loss={zeroed['loss']:.4f} byte_ce={zeroed['byte_loss']:.4f}",
flush=True,
)
if args.out:
Path(args.out).parent.mkdir(parents=True, exist_ok=True)
Path(args.out).write_text(json.dumps(results, indent=2), encoding="utf-8")
if __name__ == "__main__":
main()