atom-ai / tools /train_batched.py
thefinalboss's picture
ATOM (atom-ai): code snapshot (100fae0 + batched-train), README with measured numbers, curated checkpoints (part 4)
b9301f1 verified
Raw History Blame Contribute Delete
18.3 kB
"""Batched byte-tick trainer: B independent byte streams in parallel, CPU or CUDA.
docs/BATCHED_TRAINING.md. Same recipe as ``run_atom_native.py`` in byte-tick
BPTT mode (``scripts/train_bptt.sh``), but every tick advances B streams, and the
BPTT window loss is the mean over the window's ticks AND the B streams.
Checkpoints are ordinary single-stream checkpoints (``AtomNativeModel.save``
layout, CPU tensors) plus a ``training.batched`` block (stream offsets, feature
context, per-stream field state) for exact resume. Chat on CPU with
``tools/chat_atom_native.py --checkpoint <ckpt>``.
Example (the runbook is scripts/train_batched.sh):
PYTHONPATH=. python tools/train_batched.py --data-glob 'data/fr/*.txt' \
--batch-size 256 --device cuda --total-ticks 2000000 --output-dir checkpoints/batched_b256
"""
from __future__ import annotations
import argparse
import copy
import json
import math
import os
import random
import sys
import time
from pathlib import Path
import numpy as np
import torch
ROOT = Path(__file__).resolve().parents[1]
if str(ROOT) not in sys.path:
sys.path.insert(0, str(ROOT))
from src.atom_batched import BatchedByteTick, BatchedUnsupported # noqa: E402
from src.atom_native import DEFAULT_ENERGY_DECAY_BOUNDS, AtomNativeModel # noqa: E402
from src.io.atomizer import Atomizer # noqa: E402
from src.io.batched_stream import BatchedByteStream, ByteCorpus # noqa: E402
from src.io.stream_corpus import resolve_shard_paths # noqa: E402
PROMPTS = ["Bonjour, comment ça va ?", "Qui es-tu ?", "Il était une fois"]
def parse_args(argv=None) -> argparse.Namespace:
p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
p.add_argument("--data-glob", required=True, help="training shards (sorted glob), one circular byte stream")
p.add_argument("--val-glob", default=None, help="held-out shards for the val CE (optional)")
p.add_argument("--val-windows", type=int, default=4)
p.add_argument("--val-length", type=int, default=512)
p.add_argument("--batch-size", "-B", type=int, default=32)
p.add_argument("--device", default="cpu", choices=["cpu", "cuda"])
p.add_argument("--threads", type=int, default=0, help="torch CPU threads (0 = torch default)")
p.add_argument("--output-dir", required=True)
p.add_argument("--suffix", default="batched")
p.add_argument("--init", default=None, help="checkpoint to start from (single-stream or batched); fresh if omitted")
p.add_argument("--resume", default="auto", help="'auto' = newest ckpt in --output-dir with --suffix; 'none'; or a path")
p.add_argument("--total-ticks", type=int, required=True, help="stop at this global tick (1 tick = B bytes)")
p.add_argument("--run-ticks", type=int, default=0, help="stop after this many ticks in this process (0 = no limit)")
p.add_argument("--ckpt-every", type=int, default=8000)
p.add_argument("--log-every", type=int, default=500)
p.add_argument("--tsv", default=None, help="progress TSV (default <output-dir>/progress.tsv)")
p.add_argument("--samples", action=argparse.BooleanOptionalAction, default=True,
help="chat samples (CPU reload of each checkpoint) in the TSV/log")
p.add_argument("--offset-seed", type=int, default=-1, help="-1 = evenly spaced stream offsets; else random offsets")
p.add_argument("--chunk", type=int, default=4096, help="bytes read per stream per refill")
# model / recipe (defaults = scripts/train_bptt.sh recipe)
p.add_argument("--d-model", type=int, default=32)
p.add_argument("--n-modes", type=int, default=32)
p.add_argument("--n-atoms-max", type=int, default=64)
p.add_argument("--field-max-rms", type=float, default=3.0)
p.add_argument("--atomizer-pack", default="linguistic", choices=["linguistic", "eager"])
p.add_argument("--seed", type=int, default=20260913)
p.add_argument("--learning-rate", type=float, default=2e-3)
p.add_argument("--surface-learning-rate", type=float, default=None)
p.add_argument("--weight-decay", type=float, default=0.01)
p.add_argument("--grad-clip", type=float, default=1.0)
p.add_argument("--field-bptt", type=int, default=128)
p.add_argument("--field-leak", type=float, default=0.05)
p.add_argument("--field-leak-learnable", action=argparse.BooleanOptionalAction, default=True)
p.add_argument("--last-atom-dropout", type=float, default=0.5)
p.add_argument("--field-loss-weight", type=float, default=0.05)
p.add_argument("--episode-length", type=int, default=8000)
p.add_argument("--atom-flush-every", type=int, default=64)
p.add_argument("--tf32", action="store_true", help="allow TF32 matmuls on CUDA (faster, less exact)")
return p.parse_args(argv)
def to_cpu(obj):
if torch.is_tensor(obj):
return obj.detach().cpu()
if isinstance(obj, dict):
return {k: to_cpu(v) for k, v in obj.items()}
if isinstance(obj, list):
return [to_cpu(v) for v in obj]
if isinstance(obj, tuple):
return tuple(to_cpu(v) for v in obj)
return obj
def save_checkpoint(model: AtomNativeModel, path: Path, extra: dict) -> None:
"""model.save layout with CPU tensors; atomic; never overwrites."""
if path.exists():
raise FileExistsError(f"refusing to overwrite checkpoint {path}")
ck = model._checkpoint()
ck["training"] = extra
ck = to_cpu(ck)
path.parent.mkdir(parents=True, exist_ok=True)
tmp = path.with_suffix(path.suffix + f".tmp{os.getpid()}")
torch.save(ck, tmp)
os.replace(tmp, path)
def newest_checkpoint(out: Path, suffix: str) -> Path | None:
best, best_t = None, -1
for f in out.glob(f"atom_native_step_*_{suffix}.pt"):
try:
t = int(f.name[len("atom_native_step_"):].split("_", 1)[0])
except ValueError:
continue
if t > best_t:
best, best_t = f, t
return best
def build_model(args) -> AtomNativeModel:
torch.manual_seed(args.seed)
random.seed(args.seed)
np.random.seed(args.seed)
return AtomNativeModel(
d_model=args.d_model, n_modes=args.n_modes, n_atoms_max=args.n_atoms_max, max_payload_bytes=1,
atomizer=Atomizer(max_span_bytes=1, pack_mode=args.atomizer_pack), field_max_rms=args.field_max_rms,
field_leak=float(args.field_leak), energy_decay_bounds=DEFAULT_ENERGY_DECAY_BOUNDS, enable_merge=False,
slow_every=1, field_loss_weight=float(args.field_loss_weight), field_obligatory_readout=True,
field_obligatory_hard=True, last_atom_readout=True,
)
def apply_cli(model: AtomNativeModel, args) -> None:
"""Same 'CLI wins after load' rules as run_atom_native.py for this recipe."""
model.enable_merge = False
model.field_obligatory_hard = True
model.field_obligatory_readout = True
model.surface.set_obligatory_hard(True)
model.surface.payload_enabled = False
model.field_loss_weight = float(args.field_loss_weight)
model.field_contrast_weight = 0.0
model.surface.last_atom_dropout = float(args.last_atom_dropout)
model.field_leak = float(args.field_leak)
model._efference_every = 0
model.slow_every = 1
model.set_field_bptt(int(args.field_bptt))
if args.field_leak_learnable:
if model.field_leak_logit is None:
model.set_field_leak_learnable(True, init=float(args.field_leak))
else:
model.set_field_leak_learnable(False)
def make_optimizer(model: AtomNativeModel, args) -> torch.optim.Optimizer:
slr = float(args.surface_learning_rate) if args.surface_learning_rate is not None else float(args.learning_rate)
surface_ids = {id(p) for p in model.surface.parameters() if p.requires_grad}
surface_params = [p for p in model.trainable_parameters if id(p) in surface_ids]
other_params = [p for p in model.trainable_parameters if id(p) not in surface_ids]
return torch.optim.AdamW(
[{"params": other_params, "lr": float(args.learning_rate)}, {"params": surface_params, "lr": slr}],
weight_decay=float(args.weight_decay),
)
@torch.no_grad()
def batched_val(model: AtomNativeModel, corpus: ByteCorpus, windows: int, length: int, cap: int,
pack_mode: str) -> dict:
"""Byte CE over `windows` windows of `length` bytes (reset per window), normal
and field-alone (last-atom readout off). Same protocol as eval_windows.py
(4 x 512, reset per window) when run on the same val text."""
was = model.training
model.eval()
out = {}
offs = [(w * corpus.total) // windows for w in range(windows)]
for mode in ("normal", "field_alone"):
model.surface.last_atom_force_off = mode == "field_alone"
run = BatchedByteTick(model, windows, atom_flush_every=max(cap, length))
st = BatchedByteStream(corpus, windows, offsets=offs, chunk=length, pack_mode=pack_mode, warmup=0,
device=model.core.state.alpha.device)
tot = torch.zeros(windows, device=model.core.state.alpha.device)
for _ in range(length - 1):
c, f, t = st.next()
tot += run.step(c, f, t)["byte_loss"]
out[mode] = float((tot / (length - 1)).mean())
model.surface.last_atom_force_off = False
model.train(was)
return out
def chat_samples(ckpt: Path) -> dict:
sys.path.insert(0, str(ROOT / "tools"))
from chat_atom_native import generate_reply, load_model # noqa: WPS433
m = load_model(ckpt, device="cpu")
res = {}
for pr in PROMPTS:
torch.manual_seed(1)
res[pr] = generate_reply(m, pr, max_packets=96, max_length=100, temperature=0.8, top_k=12)
return res
def main(argv=None) -> None:
args = parse_args(argv)
if args.threads > 0:
torch.set_num_threads(args.threads)
if args.device == "cuda":
if not torch.cuda.is_available():
raise SystemExit("error: --device cuda but torch.cuda.is_available() is False")
torch.backends.cuda.matmul.allow_tf32 = bool(args.tf32)
torch.backends.cudnn.allow_tf32 = bool(args.tf32)
device = torch.device(args.device)
out = Path(args.output_dir)
out.mkdir(parents=True, exist_ok=True)
tsv = Path(args.tsv) if args.tsv else out / "progress.tsv"
shards = resolve_shard_paths(None, args.data_glob)
corpus = ByteCorpus(shards)
val_corpus = ByteCorpus(resolve_shard_paths(None, args.val_glob)) if args.val_glob else None
print(f"corpus: {len(shards)} shard(s), {corpus.total:,} bytes; B={args.batch_size} device={device}")
resume = None
if args.resume == "auto":
resume = newest_checkpoint(out, args.suffix)
elif args.resume not in ("none", "", None):
resume = Path(args.resume)
src = resume or (Path(args.init) if args.init else None)
model = build_model(args)
training = {}
tick = 0
if src is not None:
training = model.load(src) or {}
print(f"loaded {src} (recorded step={training.get('step')})")
if resume is not None:
tick = int(training.get("step", 0) or 0)
apply_cli(model, args)
model.to(device)
model.train()
try:
runner = BatchedByteTick(model, args.batch_size, atom_flush_every=args.atom_flush_every)
except BatchedUnsupported as exc:
raise SystemExit(f"error: {exc}")
optimizer = make_optimizer(model, args)
bstate = training.get("batched") if resume is not None else None
if resume is not None and "optimizer" in training:
try:
optimizer.load_state_dict(training["optimizer"])
print("restored optimizer state")
except Exception as exc: # group layout differs
print(f"optimizer restore skipped: {exc}")
if args.offset_seed >= 0:
rng = np.random.default_rng(args.offset_seed)
offsets = sorted(int(x) for x in rng.integers(0, corpus.total, size=args.batch_size))
else:
offsets = None
stream = BatchedByteStream(corpus, args.batch_size, offsets=offsets, chunk=args.chunk,
pack_mode=args.atomizer_pack, device=device)
if bstate and int(bstate.get("B", -1)) == args.batch_size:
stream.load_state_dict(bstate["stream"], strict_corpus=False)
runner.load_state_dict(bstate["runtime"])
if "torch_rng" in bstate:
torch.set_rng_state(bstate["torch_rng"])
if bstate["stream"]["corpus"]["sizes"] != corpus.signature()["sizes"]:
print("warning: corpus differs from the checkpoint's; positions taken modulo the new size")
print(f"resumed batched state at tick {tick} (B={args.batch_size})")
elif bstate:
print(f"batch size changed ({bstate.get('B')} -> {args.batch_size}): fresh stream offsets/field state")
K = int(model.field_bptt)
EP, FL = int(args.episode_length), int(args.atom_flush_every)
print(f"recipe: K={K} leak={args.field_leak} learnable={args.field_leak_learnable} lr={args.learning_rate} "
f"la_drop={args.last_atom_dropout} ep={EP} flush={FL} field_loss={args.field_loss_weight}")
if not tsv.exists():
tsv.write_text("time\ttick\tbytes_total\tbytes_per_s\ttrain_loss\tbyte_ce\tpersist\tgrad_norm\t"
"leak_mean\tval_ce\tval_field_alone\tckpt\tsamples\n", encoding="utf-8")
bytes_before = int(training.get("bytes_seen_total", 0) or 0) if resume is not None else 0
acc = None
n_win = 0
sums = torch.zeros(3, device=device)
n_sum = 0
last_grad = float("nan")
t0 = time.perf_counter()
t_log = t0
tick_log = tick
ticks_run = 0
end = int(args.total_ticks)
def window_step():
nonlocal acc, n_win, last_grad
loss = acc / float(n_win)
loss.backward()
gn = torch.nn.utils.clip_grad_norm_(list(model.trainable_parameters), float(args.grad_clip))
if not torch.isfinite(gn):
raise FloatingPointError(f"non-finite gradient at tick {tick}")
optimizer.step()
optimizer.zero_grad(set_to_none=True)
runner.detach_field_graph()
model.stabilize_dynamics_parameters()
last_grad = float(gn)
acc, n_win = None, 0
def checkpoint(tag_tick: int) -> Path:
path = out / f"atom_native_step_{tag_tick}_{args.suffix}.pt"
runner.sync_to_model(0)
extra = {
"step": tag_tick, "optimizer": optimizer.state_dict(), "seed": args.seed,
"episode_length": EP, "episode_reset": True, "atom_flush_every": FL, "stream": True,
"bytes_seen": (bytes_before + (tag_tick - start_tick) * args.batch_size),
"bytes_seen_total": (bytes_before + (tag_tick - start_tick) * args.batch_size),
"batched": {"B": args.batch_size, "stream": stream.state_dict(), "runtime": runner.state_dict(),
"args": vars(args), "torch_rng": torch.get_rng_state()},
}
save_checkpoint(model, path, extra)
return path
start_tick = tick
while tick < end and (args.run_ticks <= 0 or ticks_run < args.run_ticks):
if tick % EP == 0:
runner.reset_state()
c, f, t = stream.next()
o = runner.step(c, f, t)
lmean = o["loss"].mean()
acc = lmean if acc is None else acc + lmean
n_win += 1
with torch.no_grad():
sums += torch.stack([lmean.detach(), o["byte_loss"].mean(), o["persist_loss"].mean()])
n_sum += 1
tick += 1
ticks_run += 1
last = tick >= end or (args.run_ticks > 0 and ticks_run >= args.run_ticks)
if K > 0:
if n_win >= K or tick % EP == 0 or tick % args.ckpt_every == 0 or last:
window_step()
else:
window_step()
if tick % FL == 0:
runner.flush_atoms()
if tick % args.log_every == 0 or tick % args.ckpt_every == 0 or last:
now = time.perf_counter()
v = (sums / max(n_sum, 1)).tolist()
bps = (tick - tick_log) * args.batch_size / max(now - t_log, 1e-9)
lam = model.learnable_leak_rates()
lam_s = f"{float(lam.detach().mean()):.4f}" if lam is not None else "nan"
print(f"tick={tick} bytes={bytes_before + (tick - start_tick) * args.batch_size:,} loss={v[0]:.4f} "
f"byte_ce={v[1]:.4f} persist={v[2]:.4f} grad={last_grad:.3f} leak={lam_s} "
f"bytes/s={bps:,.0f}", flush=True)
if not all(math.isfinite(x) for x in v):
raise FloatingPointError(f"non-finite loss at tick {tick}")
log_row = (v, bps, lam_s)
sums.zero_()
n_sum = 0
t_log, tick_log = now, tick
if tick % args.ckpt_every == 0 or last:
path = out / f"atom_native_step_{tick}_{args.suffix}.pt"
if path.exists():
print(f"checkpoint exists, not overwritten: {path}")
continue
path = checkpoint(tick)
val = batched_val(model, val_corpus, args.val_windows, args.val_length, FL, args.atomizer_pack) \
if val_corpus is not None else {}
samples = {}
if args.samples:
rng = torch.get_rng_state()
try:
samples = chat_samples(path)
except Exception as exc: # never kill training for a sample
samples = {"error": repr(exc)}
finally:
torch.set_rng_state(rng)
v, bps, lam_s = log_row
row = [time.strftime("%Y-%m-%dT%H:%M:%S"), str(tick),
str(bytes_before + (tick - start_tick) * args.batch_size), f"{bps:.0f}",
f"{v[0]:.4f}", f"{v[1]:.4f}", f"{v[2]:.4f}", f"{last_grad:.3f}", lam_s,
f"{val.get('normal', float('nan')):.4f}", f"{val.get('field_alone', float('nan')):.4f}",
str(path), json.dumps(samples, ensure_ascii=False).replace("\t", " ")]
with tsv.open("a", encoding="utf-8") as fh:
fh.write("\t".join(row) + "\n")
print(f"saved {path} val={val} samples={json.dumps(samples, ensure_ascii=False)[:300]}", flush=True)
print(f"done at tick {tick} ({time.perf_counter() - t0:.1f}s this run)")
if __name__ == "__main__":
main()