ATOM (atom-ai): code snapshot (100fae0 + batched-train), README with measured numbers, curated checkpoints (part 4)
b9301f1 verified Download tools/train_batched.py from thefinalboss/atom-ai: direct link, hf CLI and curl.
- Browser
- Download file 18.3 kB
-
https://huggingface.co/thefinalboss/atom-ai/resolve/main/tools/train_batched.py
- Command line
-
hf download hf://thefinalboss/atom-ai/tools/train_batched.py
-
curl -L -o train_batched.py https://huggingface.co/thefinalboss/atom-ai/resolve/main/tools/train_batched.py
18.3 kB
| """Batched byte-tick trainer: B independent byte streams in parallel, CPU or CUDA. | |
| docs/BATCHED_TRAINING.md. Same recipe as ``run_atom_native.py`` in byte-tick | |
| BPTT mode (``scripts/train_bptt.sh``), but every tick advances B streams, and the | |
| BPTT window loss is the mean over the window's ticks AND the B streams. | |
| Checkpoints are ordinary single-stream checkpoints (``AtomNativeModel.save`` | |
| layout, CPU tensors) plus a ``training.batched`` block (stream offsets, feature | |
| context, per-stream field state) for exact resume. Chat on CPU with | |
| ``tools/chat_atom_native.py --checkpoint <ckpt>``. | |
| Example (the runbook is scripts/train_batched.sh): | |
| PYTHONPATH=. python tools/train_batched.py --data-glob 'data/fr/*.txt' \ | |
| --batch-size 256 --device cuda --total-ticks 2000000 --output-dir checkpoints/batched_b256 | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import copy | |
| import json | |
| import math | |
| import os | |
| import random | |
| import sys | |
| import time | |
| from pathlib import Path | |
| import numpy as np | |
| import torch | |
| ROOT = Path(__file__).resolve().parents[1] | |
| if str(ROOT) not in sys.path: | |
| sys.path.insert(0, str(ROOT)) | |
| from src.atom_batched import BatchedByteTick, BatchedUnsupported # noqa: E402 | |
| from src.atom_native import DEFAULT_ENERGY_DECAY_BOUNDS, AtomNativeModel # noqa: E402 | |
| from src.io.atomizer import Atomizer # noqa: E402 | |
| from src.io.batched_stream import BatchedByteStream, ByteCorpus # noqa: E402 | |
| from src.io.stream_corpus import resolve_shard_paths # noqa: E402 | |
| PROMPTS = ["Bonjour, comment ça va ?", "Qui es-tu ?", "Il était une fois"] | |
| def parse_args(argv=None) -> argparse.Namespace: | |
| p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) | |
| p.add_argument("--data-glob", required=True, help="training shards (sorted glob), one circular byte stream") | |
| p.add_argument("--val-glob", default=None, help="held-out shards for the val CE (optional)") | |
| p.add_argument("--val-windows", type=int, default=4) | |
| p.add_argument("--val-length", type=int, default=512) | |
| p.add_argument("--batch-size", "-B", type=int, default=32) | |
| p.add_argument("--device", default="cpu", choices=["cpu", "cuda"]) | |
| p.add_argument("--threads", type=int, default=0, help="torch CPU threads (0 = torch default)") | |
| p.add_argument("--output-dir", required=True) | |
| p.add_argument("--suffix", default="batched") | |
| p.add_argument("--init", default=None, help="checkpoint to start from (single-stream or batched); fresh if omitted") | |
| p.add_argument("--resume", default="auto", help="'auto' = newest ckpt in --output-dir with --suffix; 'none'; or a path") | |
| p.add_argument("--total-ticks", type=int, required=True, help="stop at this global tick (1 tick = B bytes)") | |
| p.add_argument("--run-ticks", type=int, default=0, help="stop after this many ticks in this process (0 = no limit)") | |
| p.add_argument("--ckpt-every", type=int, default=8000) | |
| p.add_argument("--log-every", type=int, default=500) | |
| p.add_argument("--tsv", default=None, help="progress TSV (default <output-dir>/progress.tsv)") | |
| p.add_argument("--samples", action=argparse.BooleanOptionalAction, default=True, | |
| help="chat samples (CPU reload of each checkpoint) in the TSV/log") | |
| p.add_argument("--offset-seed", type=int, default=-1, help="-1 = evenly spaced stream offsets; else random offsets") | |
| p.add_argument("--chunk", type=int, default=4096, help="bytes read per stream per refill") | |
| # model / recipe (defaults = scripts/train_bptt.sh recipe) | |
| p.add_argument("--d-model", type=int, default=32) | |
| p.add_argument("--n-modes", type=int, default=32) | |
| p.add_argument("--n-atoms-max", type=int, default=64) | |
| p.add_argument("--field-max-rms", type=float, default=3.0) | |
| p.add_argument("--atomizer-pack", default="linguistic", choices=["linguistic", "eager"]) | |
| p.add_argument("--seed", type=int, default=20260913) | |
| p.add_argument("--learning-rate", type=float, default=2e-3) | |
| p.add_argument("--surface-learning-rate", type=float, default=None) | |
| p.add_argument("--weight-decay", type=float, default=0.01) | |
| p.add_argument("--grad-clip", type=float, default=1.0) | |
| p.add_argument("--field-bptt", type=int, default=128) | |
| p.add_argument("--field-leak", type=float, default=0.05) | |
| p.add_argument("--field-leak-learnable", action=argparse.BooleanOptionalAction, default=True) | |
| p.add_argument("--last-atom-dropout", type=float, default=0.5) | |
| p.add_argument("--field-loss-weight", type=float, default=0.05) | |
| p.add_argument("--episode-length", type=int, default=8000) | |
| p.add_argument("--atom-flush-every", type=int, default=64) | |
| p.add_argument("--tf32", action="store_true", help="allow TF32 matmuls on CUDA (faster, less exact)") | |
| return p.parse_args(argv) | |
| def to_cpu(obj): | |
| if torch.is_tensor(obj): | |
| return obj.detach().cpu() | |
| if isinstance(obj, dict): | |
| return {k: to_cpu(v) for k, v in obj.items()} | |
| if isinstance(obj, list): | |
| return [to_cpu(v) for v in obj] | |
| if isinstance(obj, tuple): | |
| return tuple(to_cpu(v) for v in obj) | |
| return obj | |
| def save_checkpoint(model: AtomNativeModel, path: Path, extra: dict) -> None: | |
| """model.save layout with CPU tensors; atomic; never overwrites.""" | |
| if path.exists(): | |
| raise FileExistsError(f"refusing to overwrite checkpoint {path}") | |
| ck = model._checkpoint() | |
| ck["training"] = extra | |
| ck = to_cpu(ck) | |
| path.parent.mkdir(parents=True, exist_ok=True) | |
| tmp = path.with_suffix(path.suffix + f".tmp{os.getpid()}") | |
| torch.save(ck, tmp) | |
| os.replace(tmp, path) | |
| def newest_checkpoint(out: Path, suffix: str) -> Path | None: | |
| best, best_t = None, -1 | |
| for f in out.glob(f"atom_native_step_*_{suffix}.pt"): | |
| try: | |
| t = int(f.name[len("atom_native_step_"):].split("_", 1)[0]) | |
| except ValueError: | |
| continue | |
| if t > best_t: | |
| best, best_t = f, t | |
| return best | |
| def build_model(args) -> AtomNativeModel: | |
| torch.manual_seed(args.seed) | |
| random.seed(args.seed) | |
| np.random.seed(args.seed) | |
| return AtomNativeModel( | |
| d_model=args.d_model, n_modes=args.n_modes, n_atoms_max=args.n_atoms_max, max_payload_bytes=1, | |
| atomizer=Atomizer(max_span_bytes=1, pack_mode=args.atomizer_pack), field_max_rms=args.field_max_rms, | |
| field_leak=float(args.field_leak), energy_decay_bounds=DEFAULT_ENERGY_DECAY_BOUNDS, enable_merge=False, | |
| slow_every=1, field_loss_weight=float(args.field_loss_weight), field_obligatory_readout=True, | |
| field_obligatory_hard=True, last_atom_readout=True, | |
| ) | |
| def apply_cli(model: AtomNativeModel, args) -> None: | |
| """Same 'CLI wins after load' rules as run_atom_native.py for this recipe.""" | |
| model.enable_merge = False | |
| model.field_obligatory_hard = True | |
| model.field_obligatory_readout = True | |
| model.surface.set_obligatory_hard(True) | |
| model.surface.payload_enabled = False | |
| model.field_loss_weight = float(args.field_loss_weight) | |
| model.field_contrast_weight = 0.0 | |
| model.surface.last_atom_dropout = float(args.last_atom_dropout) | |
| model.field_leak = float(args.field_leak) | |
| model._efference_every = 0 | |
| model.slow_every = 1 | |
| model.set_field_bptt(int(args.field_bptt)) | |
| if args.field_leak_learnable: | |
| if model.field_leak_logit is None: | |
| model.set_field_leak_learnable(True, init=float(args.field_leak)) | |
| else: | |
| model.set_field_leak_learnable(False) | |
| def make_optimizer(model: AtomNativeModel, args) -> torch.optim.Optimizer: | |
| slr = float(args.surface_learning_rate) if args.surface_learning_rate is not None else float(args.learning_rate) | |
| surface_ids = {id(p) for p in model.surface.parameters() if p.requires_grad} | |
| surface_params = [p for p in model.trainable_parameters if id(p) in surface_ids] | |
| other_params = [p for p in model.trainable_parameters if id(p) not in surface_ids] | |
| return torch.optim.AdamW( | |
| [{"params": other_params, "lr": float(args.learning_rate)}, {"params": surface_params, "lr": slr}], | |
| weight_decay=float(args.weight_decay), | |
| ) | |
| def batched_val(model: AtomNativeModel, corpus: ByteCorpus, windows: int, length: int, cap: int, | |
| pack_mode: str) -> dict: | |
| """Byte CE over `windows` windows of `length` bytes (reset per window), normal | |
| and field-alone (last-atom readout off). Same protocol as eval_windows.py | |
| (4 x 512, reset per window) when run on the same val text.""" | |
| was = model.training | |
| model.eval() | |
| out = {} | |
| offs = [(w * corpus.total) // windows for w in range(windows)] | |
| for mode in ("normal", "field_alone"): | |
| model.surface.last_atom_force_off = mode == "field_alone" | |
| run = BatchedByteTick(model, windows, atom_flush_every=max(cap, length)) | |
| st = BatchedByteStream(corpus, windows, offsets=offs, chunk=length, pack_mode=pack_mode, warmup=0, | |
| device=model.core.state.alpha.device) | |
| tot = torch.zeros(windows, device=model.core.state.alpha.device) | |
| for _ in range(length - 1): | |
| c, f, t = st.next() | |
| tot += run.step(c, f, t)["byte_loss"] | |
| out[mode] = float((tot / (length - 1)).mean()) | |
| model.surface.last_atom_force_off = False | |
| model.train(was) | |
| return out | |
| def chat_samples(ckpt: Path) -> dict: | |
| sys.path.insert(0, str(ROOT / "tools")) | |
| from chat_atom_native import generate_reply, load_model # noqa: WPS433 | |
| m = load_model(ckpt, device="cpu") | |
| res = {} | |
| for pr in PROMPTS: | |
| torch.manual_seed(1) | |
| res[pr] = generate_reply(m, pr, max_packets=96, max_length=100, temperature=0.8, top_k=12) | |
| return res | |
| def main(argv=None) -> None: | |
| args = parse_args(argv) | |
| if args.threads > 0: | |
| torch.set_num_threads(args.threads) | |
| if args.device == "cuda": | |
| if not torch.cuda.is_available(): | |
| raise SystemExit("error: --device cuda but torch.cuda.is_available() is False") | |
| torch.backends.cuda.matmul.allow_tf32 = bool(args.tf32) | |
| torch.backends.cudnn.allow_tf32 = bool(args.tf32) | |
| device = torch.device(args.device) | |
| out = Path(args.output_dir) | |
| out.mkdir(parents=True, exist_ok=True) | |
| tsv = Path(args.tsv) if args.tsv else out / "progress.tsv" | |
| shards = resolve_shard_paths(None, args.data_glob) | |
| corpus = ByteCorpus(shards) | |
| val_corpus = ByteCorpus(resolve_shard_paths(None, args.val_glob)) if args.val_glob else None | |
| print(f"corpus: {len(shards)} shard(s), {corpus.total:,} bytes; B={args.batch_size} device={device}") | |
| resume = None | |
| if args.resume == "auto": | |
| resume = newest_checkpoint(out, args.suffix) | |
| elif args.resume not in ("none", "", None): | |
| resume = Path(args.resume) | |
| src = resume or (Path(args.init) if args.init else None) | |
| model = build_model(args) | |
| training = {} | |
| tick = 0 | |
| if src is not None: | |
| training = model.load(src) or {} | |
| print(f"loaded {src} (recorded step={training.get('step')})") | |
| if resume is not None: | |
| tick = int(training.get("step", 0) or 0) | |
| apply_cli(model, args) | |
| model.to(device) | |
| model.train() | |
| try: | |
| runner = BatchedByteTick(model, args.batch_size, atom_flush_every=args.atom_flush_every) | |
| except BatchedUnsupported as exc: | |
| raise SystemExit(f"error: {exc}") | |
| optimizer = make_optimizer(model, args) | |
| bstate = training.get("batched") if resume is not None else None | |
| if resume is not None and "optimizer" in training: | |
| try: | |
| optimizer.load_state_dict(training["optimizer"]) | |
| print("restored optimizer state") | |
| except Exception as exc: # group layout differs | |
| print(f"optimizer restore skipped: {exc}") | |
| if args.offset_seed >= 0: | |
| rng = np.random.default_rng(args.offset_seed) | |
| offsets = sorted(int(x) for x in rng.integers(0, corpus.total, size=args.batch_size)) | |
| else: | |
| offsets = None | |
| stream = BatchedByteStream(corpus, args.batch_size, offsets=offsets, chunk=args.chunk, | |
| pack_mode=args.atomizer_pack, device=device) | |
| if bstate and int(bstate.get("B", -1)) == args.batch_size: | |
| stream.load_state_dict(bstate["stream"], strict_corpus=False) | |
| runner.load_state_dict(bstate["runtime"]) | |
| if "torch_rng" in bstate: | |
| torch.set_rng_state(bstate["torch_rng"]) | |
| if bstate["stream"]["corpus"]["sizes"] != corpus.signature()["sizes"]: | |
| print("warning: corpus differs from the checkpoint's; positions taken modulo the new size") | |
| print(f"resumed batched state at tick {tick} (B={args.batch_size})") | |
| elif bstate: | |
| print(f"batch size changed ({bstate.get('B')} -> {args.batch_size}): fresh stream offsets/field state") | |
| K = int(model.field_bptt) | |
| EP, FL = int(args.episode_length), int(args.atom_flush_every) | |
| print(f"recipe: K={K} leak={args.field_leak} learnable={args.field_leak_learnable} lr={args.learning_rate} " | |
| f"la_drop={args.last_atom_dropout} ep={EP} flush={FL} field_loss={args.field_loss_weight}") | |
| if not tsv.exists(): | |
| tsv.write_text("time\ttick\tbytes_total\tbytes_per_s\ttrain_loss\tbyte_ce\tpersist\tgrad_norm\t" | |
| "leak_mean\tval_ce\tval_field_alone\tckpt\tsamples\n", encoding="utf-8") | |
| bytes_before = int(training.get("bytes_seen_total", 0) or 0) if resume is not None else 0 | |
| acc = None | |
| n_win = 0 | |
| sums = torch.zeros(3, device=device) | |
| n_sum = 0 | |
| last_grad = float("nan") | |
| t0 = time.perf_counter() | |
| t_log = t0 | |
| tick_log = tick | |
| ticks_run = 0 | |
| end = int(args.total_ticks) | |
| def window_step(): | |
| nonlocal acc, n_win, last_grad | |
| loss = acc / float(n_win) | |
| loss.backward() | |
| gn = torch.nn.utils.clip_grad_norm_(list(model.trainable_parameters), float(args.grad_clip)) | |
| if not torch.isfinite(gn): | |
| raise FloatingPointError(f"non-finite gradient at tick {tick}") | |
| optimizer.step() | |
| optimizer.zero_grad(set_to_none=True) | |
| runner.detach_field_graph() | |
| model.stabilize_dynamics_parameters() | |
| last_grad = float(gn) | |
| acc, n_win = None, 0 | |
| def checkpoint(tag_tick: int) -> Path: | |
| path = out / f"atom_native_step_{tag_tick}_{args.suffix}.pt" | |
| runner.sync_to_model(0) | |
| extra = { | |
| "step": tag_tick, "optimizer": optimizer.state_dict(), "seed": args.seed, | |
| "episode_length": EP, "episode_reset": True, "atom_flush_every": FL, "stream": True, | |
| "bytes_seen": (bytes_before + (tag_tick - start_tick) * args.batch_size), | |
| "bytes_seen_total": (bytes_before + (tag_tick - start_tick) * args.batch_size), | |
| "batched": {"B": args.batch_size, "stream": stream.state_dict(), "runtime": runner.state_dict(), | |
| "args": vars(args), "torch_rng": torch.get_rng_state()}, | |
| } | |
| save_checkpoint(model, path, extra) | |
| return path | |
| start_tick = tick | |
| while tick < end and (args.run_ticks <= 0 or ticks_run < args.run_ticks): | |
| if tick % EP == 0: | |
| runner.reset_state() | |
| c, f, t = stream.next() | |
| o = runner.step(c, f, t) | |
| lmean = o["loss"].mean() | |
| acc = lmean if acc is None else acc + lmean | |
| n_win += 1 | |
| with torch.no_grad(): | |
| sums += torch.stack([lmean.detach(), o["byte_loss"].mean(), o["persist_loss"].mean()]) | |
| n_sum += 1 | |
| tick += 1 | |
| ticks_run += 1 | |
| last = tick >= end or (args.run_ticks > 0 and ticks_run >= args.run_ticks) | |
| if K > 0: | |
| if n_win >= K or tick % EP == 0 or tick % args.ckpt_every == 0 or last: | |
| window_step() | |
| else: | |
| window_step() | |
| if tick % FL == 0: | |
| runner.flush_atoms() | |
| if tick % args.log_every == 0 or tick % args.ckpt_every == 0 or last: | |
| now = time.perf_counter() | |
| v = (sums / max(n_sum, 1)).tolist() | |
| bps = (tick - tick_log) * args.batch_size / max(now - t_log, 1e-9) | |
| lam = model.learnable_leak_rates() | |
| lam_s = f"{float(lam.detach().mean()):.4f}" if lam is not None else "nan" | |
| print(f"tick={tick} bytes={bytes_before + (tick - start_tick) * args.batch_size:,} loss={v[0]:.4f} " | |
| f"byte_ce={v[1]:.4f} persist={v[2]:.4f} grad={last_grad:.3f} leak={lam_s} " | |
| f"bytes/s={bps:,.0f}", flush=True) | |
| if not all(math.isfinite(x) for x in v): | |
| raise FloatingPointError(f"non-finite loss at tick {tick}") | |
| log_row = (v, bps, lam_s) | |
| sums.zero_() | |
| n_sum = 0 | |
| t_log, tick_log = now, tick | |
| if tick % args.ckpt_every == 0 or last: | |
| path = out / f"atom_native_step_{tick}_{args.suffix}.pt" | |
| if path.exists(): | |
| print(f"checkpoint exists, not overwritten: {path}") | |
| continue | |
| path = checkpoint(tick) | |
| val = batched_val(model, val_corpus, args.val_windows, args.val_length, FL, args.atomizer_pack) \ | |
| if val_corpus is not None else {} | |
| samples = {} | |
| if args.samples: | |
| rng = torch.get_rng_state() | |
| try: | |
| samples = chat_samples(path) | |
| except Exception as exc: # never kill training for a sample | |
| samples = {"error": repr(exc)} | |
| finally: | |
| torch.set_rng_state(rng) | |
| v, bps, lam_s = log_row | |
| row = [time.strftime("%Y-%m-%dT%H:%M:%S"), str(tick), | |
| str(bytes_before + (tick - start_tick) * args.batch_size), f"{bps:.0f}", | |
| f"{v[0]:.4f}", f"{v[1]:.4f}", f"{v[2]:.4f}", f"{last_grad:.3f}", lam_s, | |
| f"{val.get('normal', float('nan')):.4f}", f"{val.get('field_alone', float('nan')):.4f}", | |
| str(path), json.dumps(samples, ensure_ascii=False).replace("\t", " ")] | |
| with tsv.open("a", encoding="utf-8") as fh: | |
| fh.write("\t".join(row) + "\n") | |
| print(f"saved {path} val={val} samples={json.dumps(samples, ensure_ascii=False)[:300]}", flush=True) | |
| print(f"done at tick {tick} ({time.perf_counter() - t0:.1f}s this run)") | |
| if __name__ == "__main__": | |
| main() | |