Release experimental Transition Foundation Model v0.1 with reproducible source and validation
1baf215 verified Download tfm/training/train.py from maxffarrell/transition-foundation-model: direct link, hf CLI and curl.
- Browser
- Download file 18.4 kB
-
https://huggingface.co/maxffarrell/transition-foundation-model/resolve/main/tfm/training/train.py
- Command line
-
hf download hf://maxffarrell/transition-foundation-model/tfm/training/train.py
-
curl -L -o train.py https://huggingface.co/maxffarrell/transition-foundation-model/resolve/main/tfm/training/train.py
18.4 kB
| """Reproducible FP32 training with system-disjoint model selection/calibration.""" | |
| from __future__ import annotations | |
| import argparse | |
| import hashlib | |
| import json | |
| import math | |
| import random | |
| import time | |
| from pathlib import Path | |
| from typing import Any | |
| import numpy as np | |
| import torch | |
| from torch.utils.data import DataLoader | |
| from tfm.models import build_model, gaussian_nll | |
| CHECKPOINT_VERSION = "tfm-checkpoint-v1" | |
| SCHEMA_VERSION = "tfm-schema-v1" | |
| def state_dict_sha256(state: dict[str, torch.Tensor]) -> str: | |
| """Deterministic digest independent of torch.save's container encoding.""" | |
| digest = hashlib.sha256() | |
| for name in sorted(state): | |
| value = state[name].detach().cpu().contiguous() | |
| descriptor = json.dumps([name, str(value.dtype), list(value.shape)], separators=(",", ":")) | |
| digest.update(descriptor.encode() + b"\0") | |
| digest.update(value.numpy().tobytes()) | |
| return digest.hexdigest() | |
| def seed_everything(seed: int) -> None: | |
| random.seed(seed) | |
| np.random.seed(seed) | |
| torch.manual_seed(seed) | |
| if torch.backends.mps.is_available(): | |
| torch.mps.manual_seed(seed) | |
| # Determinism is requested where supported. FP32 is used on every backend. | |
| torch.use_deterministic_algorithms(True, warn_only=True) | |
| def resolve_device(device: str = "auto") -> torch.device: | |
| if device == "auto": | |
| return torch.device("mps" if torch.backends.mps.is_available() else "cpu") | |
| if device == "mps" and not torch.backends.mps.is_available(): | |
| raise RuntimeError("MPS is unavailable; choose --device cpu explicitly.") | |
| return torch.device(device) | |
| def load_manifest_episodes(data: str | Path) -> tuple[dict[str, Any], dict[str, list[Any]]]: | |
| from tfm.data import load_episode | |
| data = Path(data) | |
| manifest_path = data if data.is_file() else data / "manifest.json" | |
| manifest = json.loads(manifest_path.read_text()) | |
| splits: dict[str, list[Any]] = {"train": [], "validation": [], "test": []} | |
| systems: dict[str, str] = {} | |
| for entry in manifest["episodes"]: | |
| split = entry["split"] | |
| if split not in splits: | |
| raise ValueError(f"Unrecognized manifest split: {split}") | |
| sid = str(entry["system_id"]) | |
| if sid in systems and systems[sid] != split: | |
| raise ValueError(f"System {sid} occurs in more than one manifest split") | |
| systems[sid] = split | |
| episode = load_episode(manifest_path.parent / entry["path"]) | |
| if str(episode.system_id) != sid: | |
| raise ValueError(f"Episode system ID differs from manifest: {entry['path']}") | |
| splits[split].append(episode) | |
| if not splits["train"] or not splits["validation"]: | |
| raise ValueError("Training requires nonempty train and validation system partitions") | |
| return manifest, splits | |
| def split_validation_systems(episodes: list[Any], seed: int = 42) -> tuple[list[Any], list[Any]]: | |
| """Stable system split, stratified by family where at least two systems exist. | |
| A singleton validation partition is reserved for selection; it cannot support | |
| an independent calibration claim. No test episodes enter this function. | |
| """ | |
| grouped: dict[str, set[str]] = {} | |
| for episode in episodes: | |
| grouped.setdefault(str(episode.family_id), set()).add(str(episode.system_id)) | |
| rng = np.random.default_rng(seed) | |
| calibration: set[str] = set() | |
| selection: set[str] = set() | |
| for family in sorted(grouped): | |
| ids = sorted(grouped[family]) | |
| rng.shuffle(ids) | |
| count = max(1, len(ids) // 2) if len(ids) > 1 else 0 | |
| calibration.update(ids[:count]) | |
| selection.update(ids[count:]) | |
| # Several singleton families still allow a disjoint global split. | |
| if not calibration and len(selection) > 1: | |
| ids = sorted(selection) | |
| rng.shuffle(ids) | |
| calibration.update(ids[: max(1, len(ids) // 2)]) | |
| selection.difference_update(calibration) | |
| return ( | |
| [e for e in episodes if str(e.system_id) in selection], | |
| [e for e in episodes if str(e.system_id) in calibration], | |
| ) | |
| def to_device(batch: dict[str, torch.Tensor], device: torch.device | str) -> dict[str, torch.Tensor]: | |
| return {key: value.to(device) for key, value in batch.items()} | |
| def load_checkpoint(path: str | Path, device: str | torch.device = "cpu") -> tuple[torch.nn.Module, dict]: | |
| from tfm.data.schema import FIELD_SPECS | |
| checkpoint = torch.load(Path(path), map_location="cpu", weights_only=True) | |
| if checkpoint.get("checkpoint_version", CHECKPOINT_VERSION) != CHECKPOINT_VERSION: | |
| raise ValueError("Unsupported checkpoint version") | |
| if checkpoint.get("schema_version", SCHEMA_VERSION) != SCHEMA_VERSION: | |
| raise ValueError("Unsupported checkpoint schema version") | |
| if "field_names" in checkpoint and checkpoint["field_names"] != [field.name for field in FIELD_SPECS]: | |
| raise ValueError("Checkpoint field vocabulary differs from the active schema") | |
| if ( | |
| "model_state_sha256" in checkpoint | |
| and state_dict_sha256(checkpoint["model_state"]) != checkpoint["model_state_sha256"] | |
| ): | |
| raise ValueError("Checkpoint model state digest mismatch") | |
| model = build_model(checkpoint["model_kind"], checkpoint["model_config"]) | |
| model.load_state_dict(checkpoint["model_state"]) | |
| model.to(device).float().eval() | |
| return model, checkpoint | |
| def _ids(episodes: list[Any]) -> list[str]: | |
| return sorted({str(episode.system_id) for episode in episodes}) | |
| def evaluate_metrics(model: torch.nn.Module, loader: DataLoader, device: torch.device) -> dict[str, float]: | |
| model.eval() | |
| total, absolute_error, squared_error, count = 0.0, 0.0, 0.0, 0 | |
| with torch.no_grad(): | |
| for batch in loader: | |
| batch = to_device(batch, device) | |
| mean, log_std = model(batch) | |
| valid = batch["target_mask"].bool() & torch.isfinite(batch["target"]) | |
| valid_count = int(valid.sum().item()) | |
| total += gaussian_nll(mean, log_std, batch["target"], batch["target_mask"]).item() * valid_count | |
| errors = (mean - torch.where(valid, batch["target"], 0.0))[valid] | |
| absolute_error += errors.abs().sum().item() | |
| squared_error += errors.square().sum().item() | |
| count += valid_count | |
| if not count: | |
| raise ValueError("Validation windows contain no observed targets") | |
| return {"nll": total / count, "mae": absolute_error / count, "rmse": math.sqrt(squared_error / count)} | |
| def evaluate_loss(model: torch.nn.Module, loader: DataLoader, device: torch.device) -> float: | |
| return evaluate_metrics(model, loader, device)["nll"] | |
| def backend_smoke(model: torch.nn.Module, batch: dict[str, torch.Tensor], device: torch.device) -> None: | |
| model.train() | |
| batch = to_device(batch, device) | |
| mean, log_std = model(batch) | |
| loss = gaussian_nll(mean, log_std, batch["target"], batch["target_mask"]) | |
| if not torch.isfinite(loss): | |
| raise RuntimeError("Device smoke produced a nonfinite loss") | |
| loss.backward() | |
| if not all(p.grad is None or torch.isfinite(p.grad).all() for p in model.parameters()): | |
| raise RuntimeError("Device smoke produced nonfinite gradients") | |
| model.zero_grad(set_to_none=True) | |
| if device.type == "mps": | |
| torch.mps.synchronize() | |
| def train_one(args: argparse.Namespace, seed: int, output: Path) -> dict[str, Any]: | |
| from tfm.data import FIELD_SPECS, SCHEMAS, Normalization, WindowDataset | |
| started = time.time() | |
| seed_everything(seed) | |
| manifest, splits = load_manifest_episodes(args.data) | |
| selection, calibration = split_validation_systems(splits["validation"], args.split_seed) | |
| normalization = Normalization.fit(splits["train"]) | |
| output.mkdir(parents=True, exist_ok=True) | |
| normalization.save(output / "normalization.json") | |
| artifact_metadata = { | |
| "checkpoint_version": CHECKPOINT_VERSION, | |
| "schema_version": SCHEMA_VERSION, | |
| "field_names": [field.name for field in FIELD_SPECS], | |
| "normalization_version": 1, | |
| "normalization_sha256": hashlib.sha256((output / "normalization.json").read_bytes()).hexdigest(), | |
| "training_families": sorted({str(episode.family_id) for episode in splits["train"]}), | |
| "excluded_families": sorted(set(SCHEMAS) - {str(episode.family_id) for episode in splits["train"]}), | |
| } | |
| config = { | |
| "history_steps": args.history_steps, | |
| "future_steps": 8, | |
| "d_model": args.d_model, | |
| "layers": args.layers, | |
| "heads": args.heads, | |
| "ff_dim": args.ff_dim, | |
| "dropout": args.dropout, | |
| "use_history": not args.no_history, | |
| } | |
| model = build_model(args.model, config).float() | |
| train_data = WindowDataset( | |
| splits["train"], | |
| normalization, | |
| history_steps=args.history_steps, | |
| future_steps=8, | |
| samples=args.samples, | |
| seed=seed, | |
| ) | |
| validation_data = WindowDataset( | |
| selection, | |
| normalization, | |
| history_steps=args.history_steps, | |
| future_steps=8, | |
| samples=args.validation_samples, | |
| seed=args.split_seed + 1000, | |
| ) | |
| generator = torch.Generator().manual_seed(seed) | |
| train_loader = DataLoader( | |
| train_data, batch_size=args.batch_size, shuffle=True, num_workers=0, generator=generator | |
| ) | |
| validation_loader = DataLoader(validation_data, batch_size=args.batch_size, shuffle=False, num_workers=0) | |
| device = resolve_device(args.device) | |
| fallback_reason = None | |
| model.to(device) | |
| try: | |
| backend_smoke(model, next(iter(train_loader)), device) | |
| except RuntimeError as exc: | |
| if device.type != "mps" or not args.allow_cpu_fallback: | |
| raise RuntimeError( | |
| "Device smoke failed. Explicit --allow-cpu-fallback permits CPU retry, or use --device cpu." | |
| ) from exc | |
| fallback_reason = str(exc) | |
| device = torch.device("cpu") | |
| model.to(device) | |
| backend_smoke(model, next(iter(train_loader)), device) | |
| # Smoke must not advance the training shuffle stream. | |
| generator.manual_seed(seed) | |
| model.zero_grad(set_to_none=True) | |
| optimizer = torch.optim.AdamW(model.parameters(), lr=args.learning_rate, weight_decay=args.weight_decay) | |
| total_steps = max(1, args.epochs * len(train_loader)) | |
| warmup_steps = min(max(1, round(total_steps * args.warmup_fraction)), total_steps) | |
| def lr_factor(step: int) -> float: | |
| if step < warmup_steps: | |
| return (step + 1) / warmup_steps | |
| progress = (step - warmup_steps) / max(1, total_steps - warmup_steps) | |
| return 0.5 * (1 + math.cos(math.pi * min(progress, 1.0))) | |
| scheduler = torch.optim.lr_scheduler.LambdaLR(optimizer, lr_factor) | |
| best_loss, stale, records = float("inf"), 0, [] | |
| for epoch in range(args.epochs): | |
| model.train() | |
| train_total, train_absolute_error, train_squared_error, train_count = 0.0, 0.0, 0.0, 0 | |
| for batch in train_loader: | |
| batch = to_device(batch, device) | |
| optimizer.zero_grad(set_to_none=True) | |
| mean, log_std = model(batch) | |
| loss = gaussian_nll(mean, log_std, batch["target"], batch["target_mask"]) | |
| if not torch.isfinite(loss): | |
| raise RuntimeError(f"Nonfinite training loss at epoch {epoch + 1}") | |
| loss.backward() | |
| gradient_norm = torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0, error_if_nonfinite=True) | |
| optimizer.step() | |
| scheduler.step() | |
| valid = batch["target_mask"].bool() & torch.isfinite(batch["target"]) | |
| count = int(valid.sum().item()) | |
| train_total += loss.item() * count | |
| errors = (mean.detach() - torch.where(valid, batch["target"], 0.0))[valid] | |
| train_absolute_error += errors.abs().sum().item() | |
| train_squared_error += errors.square().sum().item() | |
| train_count += count | |
| validation_metrics = evaluate_metrics(model, validation_loader, device) | |
| validation_loss = validation_metrics["nll"] | |
| record = { | |
| "epoch": epoch + 1, | |
| "train_nll": train_total / max(1, train_count), | |
| "train_mae": train_absolute_error / max(1, train_count), | |
| "train_rmse": math.sqrt(train_squared_error / max(1, train_count)), | |
| "validation_nll": validation_loss, | |
| "validation_mae": validation_metrics["mae"], | |
| "validation_rmse": validation_metrics["rmse"], | |
| "learning_rate": optimizer.param_groups[0]["lr"], | |
| "last_gradient_norm": float(gradient_norm), | |
| } | |
| records.append(record) | |
| print(json.dumps(record), flush=True) | |
| if validation_loss < best_loss - args.min_delta: | |
| best_loss, stale = validation_loss, 0 | |
| model_state = {key: value.detach().cpu() for key, value in model.state_dict().items()} | |
| torch.save( | |
| { | |
| **artifact_metadata, | |
| "model_kind": args.model, | |
| "model_config": model.config, | |
| "model_state": model_state, | |
| "model_state_sha256": state_dict_sha256(model_state), | |
| "seed": seed, | |
| "epoch": epoch + 1, | |
| "validation_nll": validation_loss, | |
| "training_system_ids": _ids(splits["train"]), | |
| "validation_selection_system_ids": _ids(selection), | |
| "calibration_system_ids": _ids(calibration), | |
| "test_system_ids": _ids(splits["test"]), | |
| "split_seed": args.split_seed, | |
| }, | |
| output / "checkpoint.pt", | |
| ) | |
| else: | |
| stale += 1 | |
| if stale >= args.patience: | |
| break | |
| manifest_path = Path(args.data) | |
| if manifest_path.is_dir(): | |
| manifest_path = manifest_path / "manifest.json" | |
| run = { | |
| **artifact_metadata, | |
| "model_kind": args.model, | |
| "model_config": model.config, | |
| "seed": seed, | |
| "split_seed": args.split_seed, | |
| "device": str(device), | |
| "requested_device": args.device, | |
| "cpu_fallback_reason": fallback_reason, | |
| "dtype": "float32", | |
| "backend_smoke_passed": True, | |
| "parameter_count": sum(p.numel() for p in model.parameters()), | |
| "training_system_ids": _ids(splits["train"]), | |
| "validation_system_ids": _ids(splits["validation"]), | |
| "validation_selection_system_ids": _ids(selection), | |
| "calibration_system_ids": _ids(calibration), | |
| "test_system_ids": _ids(splits["test"]), | |
| "normalization_fit_partition": "train", | |
| "manifest_path": str(manifest_path.resolve()), | |
| "manifest_sha256": hashlib.sha256(manifest_path.read_bytes()).hexdigest(), | |
| "generator_version": manifest.get("generator_version"), | |
| "best_validation_nll": best_loss, | |
| "epochs_completed": len(records), | |
| "early_stopping_patience": args.patience, | |
| "elapsed_seconds": time.time() - started, | |
| "arguments": vars(args), | |
| "epochs": records, | |
| "checkpoint_sha256": hashlib.sha256((output / "checkpoint.pt").read_bytes()).hexdigest(), | |
| } | |
| (output / "run.json").write_text(json.dumps(run, indent=2) + "\n") | |
| return run | |
| def parser() -> argparse.ArgumentParser: | |
| result = argparse.ArgumentParser(description=__doc__) | |
| result.add_argument("--data", required=True) | |
| result.add_argument("--output", required=True) | |
| result.add_argument("--model", choices=["transformer", "mlp", "gru"], default="transformer") | |
| result.add_argument("--epochs", type=int, default=50) | |
| result.add_argument("--batch-size", type=int, default=256) | |
| result.add_argument("--samples", type=int, default=150000) | |
| result.add_argument("--validation-samples", type=int, default=1024) | |
| result.add_argument("--history-steps", type=int, default=24) | |
| result.add_argument("--no-history", action="store_true") | |
| result.add_argument("--seed", type=int, default=42) | |
| result.add_argument("--split-seed", type=int, default=42) | |
| result.add_argument("--device", choices=["auto", "cpu", "mps"], default="auto") | |
| result.add_argument("--allow-cpu-fallback", action="store_true") | |
| result.add_argument("--ensemble-size", type=int, default=1) | |
| result.add_argument("--seeds", help="Comma-separated independent seeds; overrides --ensemble-size") | |
| result.add_argument("--d-model", type=int, default=128) | |
| result.add_argument("--layers", type=int, default=4) | |
| result.add_argument("--heads", type=int, default=4) | |
| result.add_argument("--ff-dim", type=int, default=512) | |
| result.add_argument("--dropout", type=float, default=0.0) | |
| result.add_argument("--learning-rate", type=float, default=3e-4) | |
| result.add_argument("--weight-decay", type=float, default=1e-2) | |
| result.add_argument("--warmup-fraction", type=float, default=0.05) | |
| result.add_argument("--patience", type=int, default=8) | |
| result.add_argument("--min-delta", type=float, default=1e-4) | |
| return result | |
| def main(argv: list[str] | None = None) -> None: | |
| args = parser().parse_args(argv) | |
| if ( | |
| min( | |
| args.epochs, | |
| args.batch_size, | |
| args.samples, | |
| args.validation_samples, | |
| args.history_steps, | |
| args.patience, | |
| args.ensemble_size, | |
| ) | |
| < 1 | |
| ): | |
| raise ValueError("Epochs, sample counts, dimensions and patience must be positive") | |
| seeds = ( | |
| [int(s) for s in args.seeds.split(",")] | |
| if args.seeds | |
| else [args.seed + i for i in range(args.ensemble_size)] | |
| ) | |
| if not seeds or len(set(seeds)) != len(seeds): | |
| raise ValueError("An ensemble requires distinct seeds") | |
| output = Path(args.output) | |
| if len(seeds) == 1: | |
| train_one(args, seeds[0], output) | |
| else: | |
| runs = [] | |
| for seed in seeds: | |
| train_one(args, seed, output / f"seed-{seed}") | |
| runs.append( | |
| { | |
| "seed": seed, | |
| "checkpoint": f"seed-{seed}/checkpoint.pt", | |
| "normalization": f"seed-{seed}/normalization.json", | |
| "run": f"seed-{seed}/run.json", | |
| } | |
| ) | |
| (output / "ensemble.json").write_text(json.dumps({"version": 1, "members": runs}, indent=2) + "\n") | |
| if __name__ == "__main__": | |
| main() | |