maxffarrell's picture
Release experimental Transition Foundation Model v0.1 with reproducible source and validation
1baf215 verified
Raw History Blame Contribute Delete
18.4 kB
"""Reproducible FP32 training with system-disjoint model selection/calibration."""
from __future__ import annotations
import argparse
import hashlib
import json
import math
import random
import time
from pathlib import Path
from typing import Any
import numpy as np
import torch
from torch.utils.data import DataLoader
from tfm.models import build_model, gaussian_nll
CHECKPOINT_VERSION = "tfm-checkpoint-v1"
SCHEMA_VERSION = "tfm-schema-v1"
def state_dict_sha256(state: dict[str, torch.Tensor]) -> str:
"""Deterministic digest independent of torch.save's container encoding."""
digest = hashlib.sha256()
for name in sorted(state):
value = state[name].detach().cpu().contiguous()
descriptor = json.dumps([name, str(value.dtype), list(value.shape)], separators=(",", ":"))
digest.update(descriptor.encode() + b"\0")
digest.update(value.numpy().tobytes())
return digest.hexdigest()
def seed_everything(seed: int) -> None:
random.seed(seed)
np.random.seed(seed)
torch.manual_seed(seed)
if torch.backends.mps.is_available():
torch.mps.manual_seed(seed)
# Determinism is requested where supported. FP32 is used on every backend.
torch.use_deterministic_algorithms(True, warn_only=True)
def resolve_device(device: str = "auto") -> torch.device:
if device == "auto":
return torch.device("mps" if torch.backends.mps.is_available() else "cpu")
if device == "mps" and not torch.backends.mps.is_available():
raise RuntimeError("MPS is unavailable; choose --device cpu explicitly.")
return torch.device(device)
def load_manifest_episodes(data: str | Path) -> tuple[dict[str, Any], dict[str, list[Any]]]:
from tfm.data import load_episode
data = Path(data)
manifest_path = data if data.is_file() else data / "manifest.json"
manifest = json.loads(manifest_path.read_text())
splits: dict[str, list[Any]] = {"train": [], "validation": [], "test": []}
systems: dict[str, str] = {}
for entry in manifest["episodes"]:
split = entry["split"]
if split not in splits:
raise ValueError(f"Unrecognized manifest split: {split}")
sid = str(entry["system_id"])
if sid in systems and systems[sid] != split:
raise ValueError(f"System {sid} occurs in more than one manifest split")
systems[sid] = split
episode = load_episode(manifest_path.parent / entry["path"])
if str(episode.system_id) != sid:
raise ValueError(f"Episode system ID differs from manifest: {entry['path']}")
splits[split].append(episode)
if not splits["train"] or not splits["validation"]:
raise ValueError("Training requires nonempty train and validation system partitions")
return manifest, splits
def split_validation_systems(episodes: list[Any], seed: int = 42) -> tuple[list[Any], list[Any]]:
"""Stable system split, stratified by family where at least two systems exist.
A singleton validation partition is reserved for selection; it cannot support
an independent calibration claim. No test episodes enter this function.
"""
grouped: dict[str, set[str]] = {}
for episode in episodes:
grouped.setdefault(str(episode.family_id), set()).add(str(episode.system_id))
rng = np.random.default_rng(seed)
calibration: set[str] = set()
selection: set[str] = set()
for family in sorted(grouped):
ids = sorted(grouped[family])
rng.shuffle(ids)
count = max(1, len(ids) // 2) if len(ids) > 1 else 0
calibration.update(ids[:count])
selection.update(ids[count:])
# Several singleton families still allow a disjoint global split.
if not calibration and len(selection) > 1:
ids = sorted(selection)
rng.shuffle(ids)
calibration.update(ids[: max(1, len(ids) // 2)])
selection.difference_update(calibration)
return (
[e for e in episodes if str(e.system_id) in selection],
[e for e in episodes if str(e.system_id) in calibration],
)
def to_device(batch: dict[str, torch.Tensor], device: torch.device | str) -> dict[str, torch.Tensor]:
return {key: value.to(device) for key, value in batch.items()}
def load_checkpoint(path: str | Path, device: str | torch.device = "cpu") -> tuple[torch.nn.Module, dict]:
from tfm.data.schema import FIELD_SPECS
checkpoint = torch.load(Path(path), map_location="cpu", weights_only=True)
if checkpoint.get("checkpoint_version", CHECKPOINT_VERSION) != CHECKPOINT_VERSION:
raise ValueError("Unsupported checkpoint version")
if checkpoint.get("schema_version", SCHEMA_VERSION) != SCHEMA_VERSION:
raise ValueError("Unsupported checkpoint schema version")
if "field_names" in checkpoint and checkpoint["field_names"] != [field.name for field in FIELD_SPECS]:
raise ValueError("Checkpoint field vocabulary differs from the active schema")
if (
"model_state_sha256" in checkpoint
and state_dict_sha256(checkpoint["model_state"]) != checkpoint["model_state_sha256"]
):
raise ValueError("Checkpoint model state digest mismatch")
model = build_model(checkpoint["model_kind"], checkpoint["model_config"])
model.load_state_dict(checkpoint["model_state"])
model.to(device).float().eval()
return model, checkpoint
def _ids(episodes: list[Any]) -> list[str]:
return sorted({str(episode.system_id) for episode in episodes})
def evaluate_metrics(model: torch.nn.Module, loader: DataLoader, device: torch.device) -> dict[str, float]:
model.eval()
total, absolute_error, squared_error, count = 0.0, 0.0, 0.0, 0
with torch.no_grad():
for batch in loader:
batch = to_device(batch, device)
mean, log_std = model(batch)
valid = batch["target_mask"].bool() & torch.isfinite(batch["target"])
valid_count = int(valid.sum().item())
total += gaussian_nll(mean, log_std, batch["target"], batch["target_mask"]).item() * valid_count
errors = (mean - torch.where(valid, batch["target"], 0.0))[valid]
absolute_error += errors.abs().sum().item()
squared_error += errors.square().sum().item()
count += valid_count
if not count:
raise ValueError("Validation windows contain no observed targets")
return {"nll": total / count, "mae": absolute_error / count, "rmse": math.sqrt(squared_error / count)}
def evaluate_loss(model: torch.nn.Module, loader: DataLoader, device: torch.device) -> float:
return evaluate_metrics(model, loader, device)["nll"]
def backend_smoke(model: torch.nn.Module, batch: dict[str, torch.Tensor], device: torch.device) -> None:
model.train()
batch = to_device(batch, device)
mean, log_std = model(batch)
loss = gaussian_nll(mean, log_std, batch["target"], batch["target_mask"])
if not torch.isfinite(loss):
raise RuntimeError("Device smoke produced a nonfinite loss")
loss.backward()
if not all(p.grad is None or torch.isfinite(p.grad).all() for p in model.parameters()):
raise RuntimeError("Device smoke produced nonfinite gradients")
model.zero_grad(set_to_none=True)
if device.type == "mps":
torch.mps.synchronize()
def train_one(args: argparse.Namespace, seed: int, output: Path) -> dict[str, Any]:
from tfm.data import FIELD_SPECS, SCHEMAS, Normalization, WindowDataset
started = time.time()
seed_everything(seed)
manifest, splits = load_manifest_episodes(args.data)
selection, calibration = split_validation_systems(splits["validation"], args.split_seed)
normalization = Normalization.fit(splits["train"])
output.mkdir(parents=True, exist_ok=True)
normalization.save(output / "normalization.json")
artifact_metadata = {
"checkpoint_version": CHECKPOINT_VERSION,
"schema_version": SCHEMA_VERSION,
"field_names": [field.name for field in FIELD_SPECS],
"normalization_version": 1,
"normalization_sha256": hashlib.sha256((output / "normalization.json").read_bytes()).hexdigest(),
"training_families": sorted({str(episode.family_id) for episode in splits["train"]}),
"excluded_families": sorted(set(SCHEMAS) - {str(episode.family_id) for episode in splits["train"]}),
}
config = {
"history_steps": args.history_steps,
"future_steps": 8,
"d_model": args.d_model,
"layers": args.layers,
"heads": args.heads,
"ff_dim": args.ff_dim,
"dropout": args.dropout,
"use_history": not args.no_history,
}
model = build_model(args.model, config).float()
train_data = WindowDataset(
splits["train"],
normalization,
history_steps=args.history_steps,
future_steps=8,
samples=args.samples,
seed=seed,
)
validation_data = WindowDataset(
selection,
normalization,
history_steps=args.history_steps,
future_steps=8,
samples=args.validation_samples,
seed=args.split_seed + 1000,
)
generator = torch.Generator().manual_seed(seed)
train_loader = DataLoader(
train_data, batch_size=args.batch_size, shuffle=True, num_workers=0, generator=generator
)
validation_loader = DataLoader(validation_data, batch_size=args.batch_size, shuffle=False, num_workers=0)
device = resolve_device(args.device)
fallback_reason = None
model.to(device)
try:
backend_smoke(model, next(iter(train_loader)), device)
except RuntimeError as exc:
if device.type != "mps" or not args.allow_cpu_fallback:
raise RuntimeError(
"Device smoke failed. Explicit --allow-cpu-fallback permits CPU retry, or use --device cpu."
) from exc
fallback_reason = str(exc)
device = torch.device("cpu")
model.to(device)
backend_smoke(model, next(iter(train_loader)), device)
# Smoke must not advance the training shuffle stream.
generator.manual_seed(seed)
model.zero_grad(set_to_none=True)
optimizer = torch.optim.AdamW(model.parameters(), lr=args.learning_rate, weight_decay=args.weight_decay)
total_steps = max(1, args.epochs * len(train_loader))
warmup_steps = min(max(1, round(total_steps * args.warmup_fraction)), total_steps)
def lr_factor(step: int) -> float:
if step < warmup_steps:
return (step + 1) / warmup_steps
progress = (step - warmup_steps) / max(1, total_steps - warmup_steps)
return 0.5 * (1 + math.cos(math.pi * min(progress, 1.0)))
scheduler = torch.optim.lr_scheduler.LambdaLR(optimizer, lr_factor)
best_loss, stale, records = float("inf"), 0, []
for epoch in range(args.epochs):
model.train()
train_total, train_absolute_error, train_squared_error, train_count = 0.0, 0.0, 0.0, 0
for batch in train_loader:
batch = to_device(batch, device)
optimizer.zero_grad(set_to_none=True)
mean, log_std = model(batch)
loss = gaussian_nll(mean, log_std, batch["target"], batch["target_mask"])
if not torch.isfinite(loss):
raise RuntimeError(f"Nonfinite training loss at epoch {epoch + 1}")
loss.backward()
gradient_norm = torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0, error_if_nonfinite=True)
optimizer.step()
scheduler.step()
valid = batch["target_mask"].bool() & torch.isfinite(batch["target"])
count = int(valid.sum().item())
train_total += loss.item() * count
errors = (mean.detach() - torch.where(valid, batch["target"], 0.0))[valid]
train_absolute_error += errors.abs().sum().item()
train_squared_error += errors.square().sum().item()
train_count += count
validation_metrics = evaluate_metrics(model, validation_loader, device)
validation_loss = validation_metrics["nll"]
record = {
"epoch": epoch + 1,
"train_nll": train_total / max(1, train_count),
"train_mae": train_absolute_error / max(1, train_count),
"train_rmse": math.sqrt(train_squared_error / max(1, train_count)),
"validation_nll": validation_loss,
"validation_mae": validation_metrics["mae"],
"validation_rmse": validation_metrics["rmse"],
"learning_rate": optimizer.param_groups[0]["lr"],
"last_gradient_norm": float(gradient_norm),
}
records.append(record)
print(json.dumps(record), flush=True)
if validation_loss < best_loss - args.min_delta:
best_loss, stale = validation_loss, 0
model_state = {key: value.detach().cpu() for key, value in model.state_dict().items()}
torch.save(
{
**artifact_metadata,
"model_kind": args.model,
"model_config": model.config,
"model_state": model_state,
"model_state_sha256": state_dict_sha256(model_state),
"seed": seed,
"epoch": epoch + 1,
"validation_nll": validation_loss,
"training_system_ids": _ids(splits["train"]),
"validation_selection_system_ids": _ids(selection),
"calibration_system_ids": _ids(calibration),
"test_system_ids": _ids(splits["test"]),
"split_seed": args.split_seed,
},
output / "checkpoint.pt",
)
else:
stale += 1
if stale >= args.patience:
break
manifest_path = Path(args.data)
if manifest_path.is_dir():
manifest_path = manifest_path / "manifest.json"
run = {
**artifact_metadata,
"model_kind": args.model,
"model_config": model.config,
"seed": seed,
"split_seed": args.split_seed,
"device": str(device),
"requested_device": args.device,
"cpu_fallback_reason": fallback_reason,
"dtype": "float32",
"backend_smoke_passed": True,
"parameter_count": sum(p.numel() for p in model.parameters()),
"training_system_ids": _ids(splits["train"]),
"validation_system_ids": _ids(splits["validation"]),
"validation_selection_system_ids": _ids(selection),
"calibration_system_ids": _ids(calibration),
"test_system_ids": _ids(splits["test"]),
"normalization_fit_partition": "train",
"manifest_path": str(manifest_path.resolve()),
"manifest_sha256": hashlib.sha256(manifest_path.read_bytes()).hexdigest(),
"generator_version": manifest.get("generator_version"),
"best_validation_nll": best_loss,
"epochs_completed": len(records),
"early_stopping_patience": args.patience,
"elapsed_seconds": time.time() - started,
"arguments": vars(args),
"epochs": records,
"checkpoint_sha256": hashlib.sha256((output / "checkpoint.pt").read_bytes()).hexdigest(),
}
(output / "run.json").write_text(json.dumps(run, indent=2) + "\n")
return run
def parser() -> argparse.ArgumentParser:
result = argparse.ArgumentParser(description=__doc__)
result.add_argument("--data", required=True)
result.add_argument("--output", required=True)
result.add_argument("--model", choices=["transformer", "mlp", "gru"], default="transformer")
result.add_argument("--epochs", type=int, default=50)
result.add_argument("--batch-size", type=int, default=256)
result.add_argument("--samples", type=int, default=150000)
result.add_argument("--validation-samples", type=int, default=1024)
result.add_argument("--history-steps", type=int, default=24)
result.add_argument("--no-history", action="store_true")
result.add_argument("--seed", type=int, default=42)
result.add_argument("--split-seed", type=int, default=42)
result.add_argument("--device", choices=["auto", "cpu", "mps"], default="auto")
result.add_argument("--allow-cpu-fallback", action="store_true")
result.add_argument("--ensemble-size", type=int, default=1)
result.add_argument("--seeds", help="Comma-separated independent seeds; overrides --ensemble-size")
result.add_argument("--d-model", type=int, default=128)
result.add_argument("--layers", type=int, default=4)
result.add_argument("--heads", type=int, default=4)
result.add_argument("--ff-dim", type=int, default=512)
result.add_argument("--dropout", type=float, default=0.0)
result.add_argument("--learning-rate", type=float, default=3e-4)
result.add_argument("--weight-decay", type=float, default=1e-2)
result.add_argument("--warmup-fraction", type=float, default=0.05)
result.add_argument("--patience", type=int, default=8)
result.add_argument("--min-delta", type=float, default=1e-4)
return result
def main(argv: list[str] | None = None) -> None:
args = parser().parse_args(argv)
if (
min(
args.epochs,
args.batch_size,
args.samples,
args.validation_samples,
args.history_steps,
args.patience,
args.ensemble_size,
)
< 1
):
raise ValueError("Epochs, sample counts, dimensions and patience must be positive")
seeds = (
[int(s) for s in args.seeds.split(",")]
if args.seeds
else [args.seed + i for i in range(args.ensemble_size)]
)
if not seeds or len(set(seeds)) != len(seeds):
raise ValueError("An ensemble requires distinct seeds")
output = Path(args.output)
if len(seeds) == 1:
train_one(args, seeds[0], output)
else:
runs = []
for seed in seeds:
train_one(args, seed, output / f"seed-{seed}")
runs.append(
{
"seed": seed,
"checkpoint": f"seed-{seed}/checkpoint.pt",
"normalization": f"seed-{seed}/normalization.json",
"run": f"seed-{seed}/run.json",
}
)
(output / "ensemble.json").write_text(json.dumps({"version": 1, "members": runs}, indent=2) + "\n")
if __name__ == "__main__":
main()