maxffarrell's picture
Release experimental Transition Foundation Model v0.1 with reproducible source and validation
1baf215 verified
Raw History Blame Contribute Delete
10.3 kB
"""Shared per-field temporal encoders and a masked field/query transformer.
Slot order never supplies identity: semantic field, unit and role IDs travel
with each slot. Planned controls are prefix-masked separately for each query.
"""
from __future__ import annotations
import math
from typing import Any
import torch
from torch import nn
def default_config() -> dict[str, Any]:
from tfm.data.schema import FIELD_SPECS
return {
"history_steps": 24,
"future_steps": 8,
"num_fields": len(FIELD_SPECS),
"num_units": len({field.unit for field in FIELD_SPECS}),
"num_roles": 3,
"d_model": 128,
"layers": 4,
"heads": 4,
"ff_dim": 512,
"dropout": 0.0,
"use_history": True,
}
def _mean(tokens: torch.Tensor, mask: torch.Tensor, dim: int) -> torch.Tensor:
weights = mask.to(tokens.dtype).unsqueeze(-1)
return (tokens * weights).sum(dim) / weights.sum(dim).clamp_min(1.0)
class SchemaEncoder(nn.Module):
"""Numerical/mask/time encoders followed by shared per-field temporal nets."""
def __init__(self, kind: str, config: dict[str, Any]):
super().__init__()
d = config["d_model"]
self.use_history = config["use_history"]
self.history_steps = config["history_steps"]
self.future_steps = config["future_steps"]
self.kind = kind
self.field = nn.Embedding(config["num_fields"], d)
self.unit = nn.Embedding(config["num_units"], d)
self.role = nn.Embedding(config["num_roles"], d)
self.numeric = nn.Sequential(nn.Linear(3, d), nn.SiLU(), nn.Linear(d, d))
self.current_numeric = nn.Sequential(nn.Linear(2, d), nn.SiLU(), nn.Linear(d, d))
self.time = nn.Sequential(nn.Linear(3, d), nn.SiLU(), nn.Linear(d, d))
if kind == "gru":
self.history_temporal = nn.GRU(d, d, num_layers=2, batch_first=True, dropout=config["dropout"])
else:
self.history_temporal = nn.Sequential(
nn.Linear(self.history_steps * d, d), nn.GELU(), nn.Linear(d, d)
)
self.future_temporal = nn.Sequential(
nn.Linear(self.future_steps * d, 2 * d), nn.GELU(), nn.Linear(2 * d, d)
)
self.norm = nn.LayerNorm(d)
def forward(self, batch: dict[str, torch.Tensor]) -> dict[str, torch.Tensor]:
active = batch["field_mask"].bool()
identity = (
self.field(batch["field_ids"]) + self.unit(batch["unit_ids"]) + self.role(batch["role_ids"])
)
current_mask = batch["current_mask"].bool() & active & torch.isfinite(batch["current"])
current = torch.where(current_mask, batch["current"], torch.zeros_like(batch["current"]))
current_token = self.current_numeric(torch.stack((current, current_mask.to(current.dtype)), -1))
if self.use_history:
history = batch["history"][:, -self.history_steps :]
history_mask = batch["history_mask"][:, -self.history_steps :].bool() & active[:, None, :]
history_mask = history_mask & torch.isfinite(history)
history = torch.where(history_mask, history, torch.zeros_like(history))
hdt = batch["history_dt"][:, -self.history_steps :, None].expand_as(history) / 120.0
htoken = self.numeric(torch.stack((history, history_mask.to(history.dtype), hdt), -1))
if history.shape[1] < self.history_steps:
# Left pad only the temporal dimension; padding adds no observations.
pad = htoken.new_zeros(
htoken.shape[0], self.history_steps - history.shape[1], htoken.shape[2], htoken.shape[3]
)
htoken = torch.cat((pad, htoken), 1)
sequence = htoken.transpose(1, 2)
if self.kind == "gru":
temporal, _ = self.history_temporal(sequence.flatten(0, 1))
history_token = temporal[:, -1].reshape_as(current_token)
else:
history_token = self.history_temporal(sequence.flatten(2))
else:
# Current sensors and future controls remain available in this ablation.
history_token = torch.zeros_like(current_token)
field_context = self.norm(current_token + history_token + identity)
# Mask future values *before* the temporal MLP. Every query sees only
# controls up to its horizon, and no field is discarded by pooling.
roles = batch["role_ids"]
action_mask = active & (roles == 1)
exogenous_mask = active & (roles == 2)
future = torch.where(action_mask[:, None, :], batch["future_actions"], 0.0) + torch.where(
exogenous_mask[:, None, :], batch["future_exogenous"], 0.0
)
prefix = batch["future_dt"][:, None, :] <= batch["query_horizons"][:, :, None]
future_mask = (action_mask | exogenous_mask)[:, None, :] & torch.isfinite(future)
future = torch.where(future_mask, future, 0.0)
dt = batch["future_dt"][:, :, None].expand_as(future) / 120.0
dt = torch.where(future_mask, dt, 0.0)
ftoken = self.numeric(torch.stack((future, future_mask.to(future.dtype), dt), -1))
ftoken = ftoken * future_mask[..., None]
if future.shape[1] != self.future_steps:
raise ValueError("future_steps differs from model configuration")
# Factor the temporal linear map into step contributions, so the
# numerical encoder is evaluated once instead of once per query.
linear = self.future_temporal[0]
weights = linear.weight.reshape(linear.out_features, self.future_steps, ftoken.shape[-1])
contributions = torch.einsum("bkfd,okd->bkfo", ftoken, weights)
future_token = torch.einsum("bqk,bkfo->bqfo", prefix.to(ftoken.dtype), contributions) + linear.bias
future_token = self.future_temporal[2](self.future_temporal[1](future_token))
fields = self.norm(field_context[:, None] + future_token)
# Queries carry semantic vocabulary IDs, never slot offsets.
matching = (batch["query_fields"][:, :, None] == batch["field_ids"][:, None, :]) & active[:, None, :]
query_current = torch.einsum("bqf,bfd->bqd", matching.to(current.dtype), field_context)
t = batch["query_horizons"] / 120.0
qtime = self.time(torch.stack((t, torch.sin(t), torch.cos(t)), -1))
query = self.field(batch["query_fields"]) + query_current + qtime
return {"fields": fields, "query": query, "active": active, "query_active": matching.any(-1)}
class PreNormBlock(nn.Module):
def __init__(self, config: dict[str, Any]):
super().__init__()
d, drop = config["d_model"], config["dropout"]
self.norm1 = nn.LayerNorm(d)
self.attention = nn.MultiheadAttention(d, config["heads"], dropout=drop, batch_first=True)
self.norm2 = nn.LayerNorm(d)
self.ff = nn.Sequential(
nn.Linear(d, config["ff_dim"]),
nn.GELU(),
nn.Dropout(drop),
nn.Linear(config["ff_dim"], d),
nn.Dropout(drop),
)
def forward(self, x: torch.Tensor, padding: torch.Tensor) -> torch.Tensor:
z = self.norm1(x)
x = x + self.attention(z, z, z, key_padding_mask=padding, need_weights=False)[0]
return x + self.ff(self.norm2(x))
class TransitionModel(nn.Module):
def __init__(self, kind: str, config: dict[str, Any]):
super().__init__()
self.kind, self.config = kind, config
self.encoder = SchemaEncoder(kind, config)
d = config["d_model"]
if kind == "transformer":
self.blocks = nn.ModuleList(PreNormBlock(config) for _ in range(config["layers"]))
elif kind in ("gru", "mlp"):
self.fusion = nn.Sequential(nn.Linear(2 * d, 2 * d), nn.GELU(), nn.Linear(2 * d, d))
else:
raise ValueError(f"Unknown model kind: {kind}")
self.output = nn.Sequential(nn.LayerNorm(d), nn.Linear(d, d), nn.GELU(), nn.Linear(d, 2))
def forward(self, batch: dict[str, torch.Tensor]) -> tuple[torch.Tensor, torch.Tensor]:
encoded = self.encoder(batch)
fields, query = encoded["fields"], encoded["query"]
active = encoded["active"][:, None, :].expand(-1, query.shape[1], -1)
if self.kind == "transformer":
# Each query has a horizon-specific field backbone. Flattening B,Q
# keeps the attention over fields + one query, never H*F tokens.
tokens = torch.cat((fields, query[:, :, None]), 2).flatten(0, 1)
padding = torch.cat((~active, torch.zeros_like(active[:, :, :1])), 2).flatten(0, 1)
for block in self.blocks:
tokens = block(tokens, padding)
query = tokens[:, -1].reshape_as(query)
else:
context = _mean(fields, active, 2)
query = self.fusion(torch.cat((query, context), -1))
output = self.output(query)
query_active = encoded["query_active"]
return (
torch.where(query_active, output[..., 0], 0.0),
torch.where(query_active, output[..., 1].clamp(-7.0, 3.0), 0.0),
)
def build_model(kind: str, config: dict[str, Any] | None = None) -> TransitionModel:
resolved = default_config()
resolved.update(config or {})
if resolved["d_model"] % resolved["heads"]:
raise ValueError("d_model must be divisible by heads")
if resolved["layers"] < 1 or resolved["history_steps"] < 1:
raise ValueError("layers and history_steps must be positive")
return TransitionModel(kind, resolved)
def gaussian_nll(
mean: torch.Tensor, log_std: torch.Tensor, target: torch.Tensor, mask: torch.Tensor
) -> torch.Tensor:
"""Mean Gaussian NLL over observed targets; safely excludes padded NaNs."""
valid = mask.bool() & torch.isfinite(target)
safe_target = torch.where(valid, target, 0.0)
safe_mean = torch.where(valid, mean, 0.0)
safe_log = torch.where(valid, log_std, 0.0).clamp(-7.0, 3.0)
nll = 0.5 * ((safe_target - safe_mean) * torch.exp(-safe_log)).square() + safe_log
nll = nll + 0.5 * math.log(2.0 * math.pi)
return torch.where(valid, nll, 0.0).sum() / valid.sum().clamp_min(1)