"""Batched byte-tick trainer: B independent byte streams in parallel, CPU or CUDA. docs/BATCHED_TRAINING.md. Same recipe as ``run_atom_native.py`` in byte-tick BPTT mode (``scripts/train_bptt.sh``), but every tick advances B streams, and the BPTT window loss is the mean over the window's ticks AND the B streams. Checkpoints are ordinary single-stream checkpoints (``AtomNativeModel.save`` layout, CPU tensors) plus a ``training.batched`` block (stream offsets, feature context, per-stream field state) for exact resume. Chat on CPU with ``tools/chat_atom_native.py --checkpoint ``. Example (the runbook is scripts/train_batched.sh): PYTHONPATH=. python tools/train_batched.py --data-glob 'data/fr/*.txt' \ --batch-size 256 --device cuda --total-ticks 2000000 --output-dir checkpoints/batched_b256 """ from __future__ import annotations import argparse import copy import json import math import os import random import sys import time from pathlib import Path import numpy as np import torch ROOT = Path(__file__).resolve().parents[1] if str(ROOT) not in sys.path: sys.path.insert(0, str(ROOT)) from src.atom_batched import BatchedByteTick, BatchedUnsupported # noqa: E402 from src.atom_native import DEFAULT_ENERGY_DECAY_BOUNDS, AtomNativeModel # noqa: E402 from src.io.atomizer import Atomizer # noqa: E402 from src.io.batched_stream import BatchedByteStream, ByteCorpus # noqa: E402 from src.io.stream_corpus import resolve_shard_paths # noqa: E402 PROMPTS = ["Bonjour, comment ça va ?", "Qui es-tu ?", "Il était une fois"] def parse_args(argv=None) -> argparse.Namespace: p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) p.add_argument("--data-glob", required=True, help="training shards (sorted glob), one circular byte stream") p.add_argument("--val-glob", default=None, help="held-out shards for the val CE (optional)") p.add_argument("--val-windows", type=int, default=4) p.add_argument("--val-length", type=int, default=512) p.add_argument("--batch-size", "-B", type=int, default=32) p.add_argument("--device", default="cpu", choices=["cpu", "cuda"]) p.add_argument("--threads", type=int, default=0, help="torch CPU threads (0 = torch default)") p.add_argument("--output-dir", required=True) p.add_argument("--suffix", default="batched") p.add_argument("--init", default=None, help="checkpoint to start from (single-stream or batched); fresh if omitted") p.add_argument("--resume", default="auto", help="'auto' = newest ckpt in --output-dir with --suffix; 'none'; or a path") p.add_argument("--total-ticks", type=int, required=True, help="stop at this global tick (1 tick = B bytes)") p.add_argument("--run-ticks", type=int, default=0, help="stop after this many ticks in this process (0 = no limit)") p.add_argument("--ckpt-every", type=int, default=8000) p.add_argument("--log-every", type=int, default=500) p.add_argument("--tsv", default=None, help="progress TSV (default /progress.tsv)") p.add_argument("--samples", action=argparse.BooleanOptionalAction, default=True, help="chat samples (CPU reload of each checkpoint) in the TSV/log") p.add_argument("--offset-seed", type=int, default=-1, help="-1 = evenly spaced stream offsets; else random offsets") p.add_argument("--chunk", type=int, default=4096, help="bytes read per stream per refill") # model / recipe (defaults = scripts/train_bptt.sh recipe) p.add_argument("--d-model", type=int, default=32) p.add_argument("--n-modes", type=int, default=32) p.add_argument("--n-atoms-max", type=int, default=64) p.add_argument("--field-max-rms", type=float, default=3.0) p.add_argument("--atomizer-pack", default="linguistic", choices=["linguistic", "eager"]) p.add_argument("--seed", type=int, default=20260913) p.add_argument("--learning-rate", type=float, default=2e-3) p.add_argument("--surface-learning-rate", type=float, default=None) p.add_argument("--weight-decay", type=float, default=0.01) p.add_argument("--grad-clip", type=float, default=1.0) p.add_argument("--field-bptt", type=int, default=128) p.add_argument("--field-leak", type=float, default=0.05) p.add_argument("--field-leak-learnable", action=argparse.BooleanOptionalAction, default=True) p.add_argument("--last-atom-dropout", type=float, default=0.5) p.add_argument("--field-loss-weight", type=float, default=0.05) p.add_argument("--episode-length", type=int, default=8000) p.add_argument("--atom-flush-every", type=int, default=64) p.add_argument("--tf32", action="store_true", help="allow TF32 matmuls on CUDA (faster, less exact)") return p.parse_args(argv) def to_cpu(obj): if torch.is_tensor(obj): return obj.detach().cpu() if isinstance(obj, dict): return {k: to_cpu(v) for k, v in obj.items()} if isinstance(obj, list): return [to_cpu(v) for v in obj] if isinstance(obj, tuple): return tuple(to_cpu(v) for v in obj) return obj def save_checkpoint(model: AtomNativeModel, path: Path, extra: dict) -> None: """model.save layout with CPU tensors; atomic; never overwrites.""" if path.exists(): raise FileExistsError(f"refusing to overwrite checkpoint {path}") ck = model._checkpoint() ck["training"] = extra ck = to_cpu(ck) path.parent.mkdir(parents=True, exist_ok=True) tmp = path.with_suffix(path.suffix + f".tmp{os.getpid()}") torch.save(ck, tmp) os.replace(tmp, path) def newest_checkpoint(out: Path, suffix: str) -> Path | None: best, best_t = None, -1 for f in out.glob(f"atom_native_step_*_{suffix}.pt"): try: t = int(f.name[len("atom_native_step_"):].split("_", 1)[0]) except ValueError: continue if t > best_t: best, best_t = f, t return best def build_model(args) -> AtomNativeModel: torch.manual_seed(args.seed) random.seed(args.seed) np.random.seed(args.seed) return AtomNativeModel( d_model=args.d_model, n_modes=args.n_modes, n_atoms_max=args.n_atoms_max, max_payload_bytes=1, atomizer=Atomizer(max_span_bytes=1, pack_mode=args.atomizer_pack), field_max_rms=args.field_max_rms, field_leak=float(args.field_leak), energy_decay_bounds=DEFAULT_ENERGY_DECAY_BOUNDS, enable_merge=False, slow_every=1, field_loss_weight=float(args.field_loss_weight), field_obligatory_readout=True, field_obligatory_hard=True, last_atom_readout=True, ) def apply_cli(model: AtomNativeModel, args) -> None: """Same 'CLI wins after load' rules as run_atom_native.py for this recipe.""" model.enable_merge = False model.field_obligatory_hard = True model.field_obligatory_readout = True model.surface.set_obligatory_hard(True) model.surface.payload_enabled = False model.field_loss_weight = float(args.field_loss_weight) model.field_contrast_weight = 0.0 model.surface.last_atom_dropout = float(args.last_atom_dropout) model.field_leak = float(args.field_leak) model._efference_every = 0 model.slow_every = 1 model.set_field_bptt(int(args.field_bptt)) if args.field_leak_learnable: if model.field_leak_logit is None: model.set_field_leak_learnable(True, init=float(args.field_leak)) else: model.set_field_leak_learnable(False) def make_optimizer(model: AtomNativeModel, args) -> torch.optim.Optimizer: slr = float(args.surface_learning_rate) if args.surface_learning_rate is not None else float(args.learning_rate) surface_ids = {id(p) for p in model.surface.parameters() if p.requires_grad} surface_params = [p for p in model.trainable_parameters if id(p) in surface_ids] other_params = [p for p in model.trainable_parameters if id(p) not in surface_ids] return torch.optim.AdamW( [{"params": other_params, "lr": float(args.learning_rate)}, {"params": surface_params, "lr": slr}], weight_decay=float(args.weight_decay), ) @torch.no_grad() def batched_val(model: AtomNativeModel, corpus: ByteCorpus, windows: int, length: int, cap: int, pack_mode: str) -> dict: """Byte CE over `windows` windows of `length` bytes (reset per window), normal and field-alone (last-atom readout off). Same protocol as eval_windows.py (4 x 512, reset per window) when run on the same val text.""" was = model.training model.eval() out = {} offs = [(w * corpus.total) // windows for w in range(windows)] for mode in ("normal", "field_alone"): model.surface.last_atom_force_off = mode == "field_alone" run = BatchedByteTick(model, windows, atom_flush_every=max(cap, length)) st = BatchedByteStream(corpus, windows, offsets=offs, chunk=length, pack_mode=pack_mode, warmup=0, device=model.core.state.alpha.device) tot = torch.zeros(windows, device=model.core.state.alpha.device) for _ in range(length - 1): c, f, t = st.next() tot += run.step(c, f, t)["byte_loss"] out[mode] = float((tot / (length - 1)).mean()) model.surface.last_atom_force_off = False model.train(was) return out def chat_samples(ckpt: Path) -> dict: sys.path.insert(0, str(ROOT / "tools")) from chat_atom_native import generate_reply, load_model # noqa: WPS433 m = load_model(ckpt, device="cpu") res = {} for pr in PROMPTS: torch.manual_seed(1) res[pr] = generate_reply(m, pr, max_packets=96, max_length=100, temperature=0.8, top_k=12) return res def main(argv=None) -> None: args = parse_args(argv) if args.threads > 0: torch.set_num_threads(args.threads) if args.device == "cuda": if not torch.cuda.is_available(): raise SystemExit("error: --device cuda but torch.cuda.is_available() is False") torch.backends.cuda.matmul.allow_tf32 = bool(args.tf32) torch.backends.cudnn.allow_tf32 = bool(args.tf32) device = torch.device(args.device) out = Path(args.output_dir) out.mkdir(parents=True, exist_ok=True) tsv = Path(args.tsv) if args.tsv else out / "progress.tsv" shards = resolve_shard_paths(None, args.data_glob) corpus = ByteCorpus(shards) val_corpus = ByteCorpus(resolve_shard_paths(None, args.val_glob)) if args.val_glob else None print(f"corpus: {len(shards)} shard(s), {corpus.total:,} bytes; B={args.batch_size} device={device}") resume = None if args.resume == "auto": resume = newest_checkpoint(out, args.suffix) elif args.resume not in ("none", "", None): resume = Path(args.resume) src = resume or (Path(args.init) if args.init else None) model = build_model(args) training = {} tick = 0 if src is not None: training = model.load(src) or {} print(f"loaded {src} (recorded step={training.get('step')})") if resume is not None: tick = int(training.get("step", 0) or 0) apply_cli(model, args) model.to(device) model.train() try: runner = BatchedByteTick(model, args.batch_size, atom_flush_every=args.atom_flush_every) except BatchedUnsupported as exc: raise SystemExit(f"error: {exc}") optimizer = make_optimizer(model, args) bstate = training.get("batched") if resume is not None else None if resume is not None and "optimizer" in training: try: optimizer.load_state_dict(training["optimizer"]) print("restored optimizer state") except Exception as exc: # group layout differs print(f"optimizer restore skipped: {exc}") if args.offset_seed >= 0: rng = np.random.default_rng(args.offset_seed) offsets = sorted(int(x) for x in rng.integers(0, corpus.total, size=args.batch_size)) else: offsets = None stream = BatchedByteStream(corpus, args.batch_size, offsets=offsets, chunk=args.chunk, pack_mode=args.atomizer_pack, device=device) if bstate and int(bstate.get("B", -1)) == args.batch_size: stream.load_state_dict(bstate["stream"], strict_corpus=False) runner.load_state_dict(bstate["runtime"]) if "torch_rng" in bstate: torch.set_rng_state(bstate["torch_rng"]) if bstate["stream"]["corpus"]["sizes"] != corpus.signature()["sizes"]: print("warning: corpus differs from the checkpoint's; positions taken modulo the new size") print(f"resumed batched state at tick {tick} (B={args.batch_size})") elif bstate: print(f"batch size changed ({bstate.get('B')} -> {args.batch_size}): fresh stream offsets/field state") K = int(model.field_bptt) EP, FL = int(args.episode_length), int(args.atom_flush_every) print(f"recipe: K={K} leak={args.field_leak} learnable={args.field_leak_learnable} lr={args.learning_rate} " f"la_drop={args.last_atom_dropout} ep={EP} flush={FL} field_loss={args.field_loss_weight}") if not tsv.exists(): tsv.write_text("time\ttick\tbytes_total\tbytes_per_s\ttrain_loss\tbyte_ce\tpersist\tgrad_norm\t" "leak_mean\tval_ce\tval_field_alone\tckpt\tsamples\n", encoding="utf-8") bytes_before = int(training.get("bytes_seen_total", 0) or 0) if resume is not None else 0 acc = None n_win = 0 sums = torch.zeros(3, device=device) n_sum = 0 last_grad = float("nan") t0 = time.perf_counter() t_log = t0 tick_log = tick ticks_run = 0 end = int(args.total_ticks) def window_step(): nonlocal acc, n_win, last_grad loss = acc / float(n_win) loss.backward() gn = torch.nn.utils.clip_grad_norm_(list(model.trainable_parameters), float(args.grad_clip)) if not torch.isfinite(gn): raise FloatingPointError(f"non-finite gradient at tick {tick}") optimizer.step() optimizer.zero_grad(set_to_none=True) runner.detach_field_graph() model.stabilize_dynamics_parameters() last_grad = float(gn) acc, n_win = None, 0 def checkpoint(tag_tick: int) -> Path: path = out / f"atom_native_step_{tag_tick}_{args.suffix}.pt" runner.sync_to_model(0) extra = { "step": tag_tick, "optimizer": optimizer.state_dict(), "seed": args.seed, "episode_length": EP, "episode_reset": True, "atom_flush_every": FL, "stream": True, "bytes_seen": (bytes_before + (tag_tick - start_tick) * args.batch_size), "bytes_seen_total": (bytes_before + (tag_tick - start_tick) * args.batch_size), "batched": {"B": args.batch_size, "stream": stream.state_dict(), "runtime": runner.state_dict(), "args": vars(args), "torch_rng": torch.get_rng_state()}, } save_checkpoint(model, path, extra) return path start_tick = tick while tick < end and (args.run_ticks <= 0 or ticks_run < args.run_ticks): if tick % EP == 0: runner.reset_state() c, f, t = stream.next() o = runner.step(c, f, t) lmean = o["loss"].mean() acc = lmean if acc is None else acc + lmean n_win += 1 with torch.no_grad(): sums += torch.stack([lmean.detach(), o["byte_loss"].mean(), o["persist_loss"].mean()]) n_sum += 1 tick += 1 ticks_run += 1 last = tick >= end or (args.run_ticks > 0 and ticks_run >= args.run_ticks) if K > 0: if n_win >= K or tick % EP == 0 or tick % args.ckpt_every == 0 or last: window_step() else: window_step() if tick % FL == 0: runner.flush_atoms() if tick % args.log_every == 0 or tick % args.ckpt_every == 0 or last: now = time.perf_counter() v = (sums / max(n_sum, 1)).tolist() bps = (tick - tick_log) * args.batch_size / max(now - t_log, 1e-9) lam = model.learnable_leak_rates() lam_s = f"{float(lam.detach().mean()):.4f}" if lam is not None else "nan" print(f"tick={tick} bytes={bytes_before + (tick - start_tick) * args.batch_size:,} loss={v[0]:.4f} " f"byte_ce={v[1]:.4f} persist={v[2]:.4f} grad={last_grad:.3f} leak={lam_s} " f"bytes/s={bps:,.0f}", flush=True) if not all(math.isfinite(x) for x in v): raise FloatingPointError(f"non-finite loss at tick {tick}") log_row = (v, bps, lam_s) sums.zero_() n_sum = 0 t_log, tick_log = now, tick if tick % args.ckpt_every == 0 or last: path = out / f"atom_native_step_{tick}_{args.suffix}.pt" if path.exists(): print(f"checkpoint exists, not overwritten: {path}") continue path = checkpoint(tick) val = batched_val(model, val_corpus, args.val_windows, args.val_length, FL, args.atomizer_pack) \ if val_corpus is not None else {} samples = {} if args.samples: rng = torch.get_rng_state() try: samples = chat_samples(path) except Exception as exc: # never kill training for a sample samples = {"error": repr(exc)} finally: torch.set_rng_state(rng) v, bps, lam_s = log_row row = [time.strftime("%Y-%m-%dT%H:%M:%S"), str(tick), str(bytes_before + (tick - start_tick) * args.batch_size), f"{bps:.0f}", f"{v[0]:.4f}", f"{v[1]:.4f}", f"{v[2]:.4f}", f"{last_grad:.3f}", lam_s, f"{val.get('normal', float('nan')):.4f}", f"{val.get('field_alone', float('nan')):.4f}", str(path), json.dumps(samples, ensure_ascii=False).replace("\t", " ")] with tsv.open("a", encoding="utf-8") as fh: fh.write("\t".join(row) + "\n") print(f"saved {path} val={val} samples={json.dumps(samples, ensure_ascii=False)[:300]}", flush=True) print(f"done at tick {tick} ({time.perf_counter() - t0:.1f}s this run)") if __name__ == "__main__": main()