"""Reproducible FP32 training with system-disjoint model selection/calibration.""" from __future__ import annotations import argparse import hashlib import json import math import random import time from pathlib import Path from typing import Any import numpy as np import torch from torch.utils.data import DataLoader from tfm.models import build_model, gaussian_nll CHECKPOINT_VERSION = "tfm-checkpoint-v1" SCHEMA_VERSION = "tfm-schema-v1" def state_dict_sha256(state: dict[str, torch.Tensor]) -> str: """Deterministic digest independent of torch.save's container encoding.""" digest = hashlib.sha256() for name in sorted(state): value = state[name].detach().cpu().contiguous() descriptor = json.dumps([name, str(value.dtype), list(value.shape)], separators=(",", ":")) digest.update(descriptor.encode() + b"\0") digest.update(value.numpy().tobytes()) return digest.hexdigest() def seed_everything(seed: int) -> None: random.seed(seed) np.random.seed(seed) torch.manual_seed(seed) if torch.backends.mps.is_available(): torch.mps.manual_seed(seed) # Determinism is requested where supported. FP32 is used on every backend. torch.use_deterministic_algorithms(True, warn_only=True) def resolve_device(device: str = "auto") -> torch.device: if device == "auto": return torch.device("mps" if torch.backends.mps.is_available() else "cpu") if device == "mps" and not torch.backends.mps.is_available(): raise RuntimeError("MPS is unavailable; choose --device cpu explicitly.") return torch.device(device) def load_manifest_episodes(data: str | Path) -> tuple[dict[str, Any], dict[str, list[Any]]]: from tfm.data import load_episode data = Path(data) manifest_path = data if data.is_file() else data / "manifest.json" manifest = json.loads(manifest_path.read_text()) splits: dict[str, list[Any]] = {"train": [], "validation": [], "test": []} systems: dict[str, str] = {} for entry in manifest["episodes"]: split = entry["split"] if split not in splits: raise ValueError(f"Unrecognized manifest split: {split}") sid = str(entry["system_id"]) if sid in systems and systems[sid] != split: raise ValueError(f"System {sid} occurs in more than one manifest split") systems[sid] = split episode = load_episode(manifest_path.parent / entry["path"]) if str(episode.system_id) != sid: raise ValueError(f"Episode system ID differs from manifest: {entry['path']}") splits[split].append(episode) if not splits["train"] or not splits["validation"]: raise ValueError("Training requires nonempty train and validation system partitions") return manifest, splits def split_validation_systems(episodes: list[Any], seed: int = 42) -> tuple[list[Any], list[Any]]: """Stable system split, stratified by family where at least two systems exist. A singleton validation partition is reserved for selection; it cannot support an independent calibration claim. No test episodes enter this function. """ grouped: dict[str, set[str]] = {} for episode in episodes: grouped.setdefault(str(episode.family_id), set()).add(str(episode.system_id)) rng = np.random.default_rng(seed) calibration: set[str] = set() selection: set[str] = set() for family in sorted(grouped): ids = sorted(grouped[family]) rng.shuffle(ids) count = max(1, len(ids) // 2) if len(ids) > 1 else 0 calibration.update(ids[:count]) selection.update(ids[count:]) # Several singleton families still allow a disjoint global split. if not calibration and len(selection) > 1: ids = sorted(selection) rng.shuffle(ids) calibration.update(ids[: max(1, len(ids) // 2)]) selection.difference_update(calibration) return ( [e for e in episodes if str(e.system_id) in selection], [e for e in episodes if str(e.system_id) in calibration], ) def to_device(batch: dict[str, torch.Tensor], device: torch.device | str) -> dict[str, torch.Tensor]: return {key: value.to(device) for key, value in batch.items()} def load_checkpoint(path: str | Path, device: str | torch.device = "cpu") -> tuple[torch.nn.Module, dict]: from tfm.data.schema import FIELD_SPECS checkpoint = torch.load(Path(path), map_location="cpu", weights_only=True) if checkpoint.get("checkpoint_version", CHECKPOINT_VERSION) != CHECKPOINT_VERSION: raise ValueError("Unsupported checkpoint version") if checkpoint.get("schema_version", SCHEMA_VERSION) != SCHEMA_VERSION: raise ValueError("Unsupported checkpoint schema version") if "field_names" in checkpoint and checkpoint["field_names"] != [field.name for field in FIELD_SPECS]: raise ValueError("Checkpoint field vocabulary differs from the active schema") if ( "model_state_sha256" in checkpoint and state_dict_sha256(checkpoint["model_state"]) != checkpoint["model_state_sha256"] ): raise ValueError("Checkpoint model state digest mismatch") model = build_model(checkpoint["model_kind"], checkpoint["model_config"]) model.load_state_dict(checkpoint["model_state"]) model.to(device).float().eval() return model, checkpoint def _ids(episodes: list[Any]) -> list[str]: return sorted({str(episode.system_id) for episode in episodes}) def evaluate_metrics(model: torch.nn.Module, loader: DataLoader, device: torch.device) -> dict[str, float]: model.eval() total, absolute_error, squared_error, count = 0.0, 0.0, 0.0, 0 with torch.no_grad(): for batch in loader: batch = to_device(batch, device) mean, log_std = model(batch) valid = batch["target_mask"].bool() & torch.isfinite(batch["target"]) valid_count = int(valid.sum().item()) total += gaussian_nll(mean, log_std, batch["target"], batch["target_mask"]).item() * valid_count errors = (mean - torch.where(valid, batch["target"], 0.0))[valid] absolute_error += errors.abs().sum().item() squared_error += errors.square().sum().item() count += valid_count if not count: raise ValueError("Validation windows contain no observed targets") return {"nll": total / count, "mae": absolute_error / count, "rmse": math.sqrt(squared_error / count)} def evaluate_loss(model: torch.nn.Module, loader: DataLoader, device: torch.device) -> float: return evaluate_metrics(model, loader, device)["nll"] def backend_smoke(model: torch.nn.Module, batch: dict[str, torch.Tensor], device: torch.device) -> None: model.train() batch = to_device(batch, device) mean, log_std = model(batch) loss = gaussian_nll(mean, log_std, batch["target"], batch["target_mask"]) if not torch.isfinite(loss): raise RuntimeError("Device smoke produced a nonfinite loss") loss.backward() if not all(p.grad is None or torch.isfinite(p.grad).all() for p in model.parameters()): raise RuntimeError("Device smoke produced nonfinite gradients") model.zero_grad(set_to_none=True) if device.type == "mps": torch.mps.synchronize() def train_one(args: argparse.Namespace, seed: int, output: Path) -> dict[str, Any]: from tfm.data import FIELD_SPECS, SCHEMAS, Normalization, WindowDataset started = time.time() seed_everything(seed) manifest, splits = load_manifest_episodes(args.data) selection, calibration = split_validation_systems(splits["validation"], args.split_seed) normalization = Normalization.fit(splits["train"]) output.mkdir(parents=True, exist_ok=True) normalization.save(output / "normalization.json") artifact_metadata = { "checkpoint_version": CHECKPOINT_VERSION, "schema_version": SCHEMA_VERSION, "field_names": [field.name for field in FIELD_SPECS], "normalization_version": 1, "normalization_sha256": hashlib.sha256((output / "normalization.json").read_bytes()).hexdigest(), "training_families": sorted({str(episode.family_id) for episode in splits["train"]}), "excluded_families": sorted(set(SCHEMAS) - {str(episode.family_id) for episode in splits["train"]}), } config = { "history_steps": args.history_steps, "future_steps": 8, "d_model": args.d_model, "layers": args.layers, "heads": args.heads, "ff_dim": args.ff_dim, "dropout": args.dropout, "use_history": not args.no_history, } model = build_model(args.model, config).float() train_data = WindowDataset( splits["train"], normalization, history_steps=args.history_steps, future_steps=8, samples=args.samples, seed=seed, ) validation_data = WindowDataset( selection, normalization, history_steps=args.history_steps, future_steps=8, samples=args.validation_samples, seed=args.split_seed + 1000, ) generator = torch.Generator().manual_seed(seed) train_loader = DataLoader( train_data, batch_size=args.batch_size, shuffle=True, num_workers=0, generator=generator ) validation_loader = DataLoader(validation_data, batch_size=args.batch_size, shuffle=False, num_workers=0) device = resolve_device(args.device) fallback_reason = None model.to(device) try: backend_smoke(model, next(iter(train_loader)), device) except RuntimeError as exc: if device.type != "mps" or not args.allow_cpu_fallback: raise RuntimeError( "Device smoke failed. Explicit --allow-cpu-fallback permits CPU retry, or use --device cpu." ) from exc fallback_reason = str(exc) device = torch.device("cpu") model.to(device) backend_smoke(model, next(iter(train_loader)), device) # Smoke must not advance the training shuffle stream. generator.manual_seed(seed) model.zero_grad(set_to_none=True) optimizer = torch.optim.AdamW(model.parameters(), lr=args.learning_rate, weight_decay=args.weight_decay) total_steps = max(1, args.epochs * len(train_loader)) warmup_steps = min(max(1, round(total_steps * args.warmup_fraction)), total_steps) def lr_factor(step: int) -> float: if step < warmup_steps: return (step + 1) / warmup_steps progress = (step - warmup_steps) / max(1, total_steps - warmup_steps) return 0.5 * (1 + math.cos(math.pi * min(progress, 1.0))) scheduler = torch.optim.lr_scheduler.LambdaLR(optimizer, lr_factor) best_loss, stale, records = float("inf"), 0, [] for epoch in range(args.epochs): model.train() train_total, train_absolute_error, train_squared_error, train_count = 0.0, 0.0, 0.0, 0 for batch in train_loader: batch = to_device(batch, device) optimizer.zero_grad(set_to_none=True) mean, log_std = model(batch) loss = gaussian_nll(mean, log_std, batch["target"], batch["target_mask"]) if not torch.isfinite(loss): raise RuntimeError(f"Nonfinite training loss at epoch {epoch + 1}") loss.backward() gradient_norm = torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0, error_if_nonfinite=True) optimizer.step() scheduler.step() valid = batch["target_mask"].bool() & torch.isfinite(batch["target"]) count = int(valid.sum().item()) train_total += loss.item() * count errors = (mean.detach() - torch.where(valid, batch["target"], 0.0))[valid] train_absolute_error += errors.abs().sum().item() train_squared_error += errors.square().sum().item() train_count += count validation_metrics = evaluate_metrics(model, validation_loader, device) validation_loss = validation_metrics["nll"] record = { "epoch": epoch + 1, "train_nll": train_total / max(1, train_count), "train_mae": train_absolute_error / max(1, train_count), "train_rmse": math.sqrt(train_squared_error / max(1, train_count)), "validation_nll": validation_loss, "validation_mae": validation_metrics["mae"], "validation_rmse": validation_metrics["rmse"], "learning_rate": optimizer.param_groups[0]["lr"], "last_gradient_norm": float(gradient_norm), } records.append(record) print(json.dumps(record), flush=True) if validation_loss < best_loss - args.min_delta: best_loss, stale = validation_loss, 0 model_state = {key: value.detach().cpu() for key, value in model.state_dict().items()} torch.save( { **artifact_metadata, "model_kind": args.model, "model_config": model.config, "model_state": model_state, "model_state_sha256": state_dict_sha256(model_state), "seed": seed, "epoch": epoch + 1, "validation_nll": validation_loss, "training_system_ids": _ids(splits["train"]), "validation_selection_system_ids": _ids(selection), "calibration_system_ids": _ids(calibration), "test_system_ids": _ids(splits["test"]), "split_seed": args.split_seed, }, output / "checkpoint.pt", ) else: stale += 1 if stale >= args.patience: break manifest_path = Path(args.data) if manifest_path.is_dir(): manifest_path = manifest_path / "manifest.json" run = { **artifact_metadata, "model_kind": args.model, "model_config": model.config, "seed": seed, "split_seed": args.split_seed, "device": str(device), "requested_device": args.device, "cpu_fallback_reason": fallback_reason, "dtype": "float32", "backend_smoke_passed": True, "parameter_count": sum(p.numel() for p in model.parameters()), "training_system_ids": _ids(splits["train"]), "validation_system_ids": _ids(splits["validation"]), "validation_selection_system_ids": _ids(selection), "calibration_system_ids": _ids(calibration), "test_system_ids": _ids(splits["test"]), "normalization_fit_partition": "train", "manifest_path": str(manifest_path.resolve()), "manifest_sha256": hashlib.sha256(manifest_path.read_bytes()).hexdigest(), "generator_version": manifest.get("generator_version"), "best_validation_nll": best_loss, "epochs_completed": len(records), "early_stopping_patience": args.patience, "elapsed_seconds": time.time() - started, "arguments": vars(args), "epochs": records, "checkpoint_sha256": hashlib.sha256((output / "checkpoint.pt").read_bytes()).hexdigest(), } (output / "run.json").write_text(json.dumps(run, indent=2) + "\n") return run def parser() -> argparse.ArgumentParser: result = argparse.ArgumentParser(description=__doc__) result.add_argument("--data", required=True) result.add_argument("--output", required=True) result.add_argument("--model", choices=["transformer", "mlp", "gru"], default="transformer") result.add_argument("--epochs", type=int, default=50) result.add_argument("--batch-size", type=int, default=256) result.add_argument("--samples", type=int, default=150000) result.add_argument("--validation-samples", type=int, default=1024) result.add_argument("--history-steps", type=int, default=24) result.add_argument("--no-history", action="store_true") result.add_argument("--seed", type=int, default=42) result.add_argument("--split-seed", type=int, default=42) result.add_argument("--device", choices=["auto", "cpu", "mps"], default="auto") result.add_argument("--allow-cpu-fallback", action="store_true") result.add_argument("--ensemble-size", type=int, default=1) result.add_argument("--seeds", help="Comma-separated independent seeds; overrides --ensemble-size") result.add_argument("--d-model", type=int, default=128) result.add_argument("--layers", type=int, default=4) result.add_argument("--heads", type=int, default=4) result.add_argument("--ff-dim", type=int, default=512) result.add_argument("--dropout", type=float, default=0.0) result.add_argument("--learning-rate", type=float, default=3e-4) result.add_argument("--weight-decay", type=float, default=1e-2) result.add_argument("--warmup-fraction", type=float, default=0.05) result.add_argument("--patience", type=int, default=8) result.add_argument("--min-delta", type=float, default=1e-4) return result def main(argv: list[str] | None = None) -> None: args = parser().parse_args(argv) if ( min( args.epochs, args.batch_size, args.samples, args.validation_samples, args.history_steps, args.patience, args.ensemble_size, ) < 1 ): raise ValueError("Epochs, sample counts, dimensions and patience must be positive") seeds = ( [int(s) for s in args.seeds.split(",")] if args.seeds else [args.seed + i for i in range(args.ensemble_size)] ) if not seeds or len(set(seeds)) != len(seeds): raise ValueError("An ensemble requires distinct seeds") output = Path(args.output) if len(seeds) == 1: train_one(args, seeds[0], output) else: runs = [] for seed in seeds: train_one(args, seed, output / f"seed-{seed}") runs.append( { "seed": seed, "checkpoint": f"seed-{seed}/checkpoint.pt", "normalization": f"seed-{seed}/normalization.json", "run": f"seed-{seed}/run.json", } ) (output / "ensemble.json").write_text(json.dumps({"version": 1, "members": runs}, indent=2) + "\n") if __name__ == "__main__": main()