Release experimental Transition Foundation Model v0.1 with reproducible source and validation
1baf215 verified Download tfm/models/core.py from maxffarrell/transition-foundation-model: direct link, hf CLI and curl.
- Browser
- Download file 10.3 kB
-
https://huggingface.co/maxffarrell/transition-foundation-model/resolve/main/tfm/models/core.py
- Command line
-
hf download hf://maxffarrell/transition-foundation-model/tfm/models/core.py
-
curl -L -o core.py https://huggingface.co/maxffarrell/transition-foundation-model/resolve/main/tfm/models/core.py
10.3 kB
| """Shared per-field temporal encoders and a masked field/query transformer. | |
| Slot order never supplies identity: semantic field, unit and role IDs travel | |
| with each slot. Planned controls are prefix-masked separately for each query. | |
| """ | |
| from __future__ import annotations | |
| import math | |
| from typing import Any | |
| import torch | |
| from torch import nn | |
| def default_config() -> dict[str, Any]: | |
| from tfm.data.schema import FIELD_SPECS | |
| return { | |
| "history_steps": 24, | |
| "future_steps": 8, | |
| "num_fields": len(FIELD_SPECS), | |
| "num_units": len({field.unit for field in FIELD_SPECS}), | |
| "num_roles": 3, | |
| "d_model": 128, | |
| "layers": 4, | |
| "heads": 4, | |
| "ff_dim": 512, | |
| "dropout": 0.0, | |
| "use_history": True, | |
| } | |
| def _mean(tokens: torch.Tensor, mask: torch.Tensor, dim: int) -> torch.Tensor: | |
| weights = mask.to(tokens.dtype).unsqueeze(-1) | |
| return (tokens * weights).sum(dim) / weights.sum(dim).clamp_min(1.0) | |
| class SchemaEncoder(nn.Module): | |
| """Numerical/mask/time encoders followed by shared per-field temporal nets.""" | |
| def __init__(self, kind: str, config: dict[str, Any]): | |
| super().__init__() | |
| d = config["d_model"] | |
| self.use_history = config["use_history"] | |
| self.history_steps = config["history_steps"] | |
| self.future_steps = config["future_steps"] | |
| self.kind = kind | |
| self.field = nn.Embedding(config["num_fields"], d) | |
| self.unit = nn.Embedding(config["num_units"], d) | |
| self.role = nn.Embedding(config["num_roles"], d) | |
| self.numeric = nn.Sequential(nn.Linear(3, d), nn.SiLU(), nn.Linear(d, d)) | |
| self.current_numeric = nn.Sequential(nn.Linear(2, d), nn.SiLU(), nn.Linear(d, d)) | |
| self.time = nn.Sequential(nn.Linear(3, d), nn.SiLU(), nn.Linear(d, d)) | |
| if kind == "gru": | |
| self.history_temporal = nn.GRU(d, d, num_layers=2, batch_first=True, dropout=config["dropout"]) | |
| else: | |
| self.history_temporal = nn.Sequential( | |
| nn.Linear(self.history_steps * d, d), nn.GELU(), nn.Linear(d, d) | |
| ) | |
| self.future_temporal = nn.Sequential( | |
| nn.Linear(self.future_steps * d, 2 * d), nn.GELU(), nn.Linear(2 * d, d) | |
| ) | |
| self.norm = nn.LayerNorm(d) | |
| def forward(self, batch: dict[str, torch.Tensor]) -> dict[str, torch.Tensor]: | |
| active = batch["field_mask"].bool() | |
| identity = ( | |
| self.field(batch["field_ids"]) + self.unit(batch["unit_ids"]) + self.role(batch["role_ids"]) | |
| ) | |
| current_mask = batch["current_mask"].bool() & active & torch.isfinite(batch["current"]) | |
| current = torch.where(current_mask, batch["current"], torch.zeros_like(batch["current"])) | |
| current_token = self.current_numeric(torch.stack((current, current_mask.to(current.dtype)), -1)) | |
| if self.use_history: | |
| history = batch["history"][:, -self.history_steps :] | |
| history_mask = batch["history_mask"][:, -self.history_steps :].bool() & active[:, None, :] | |
| history_mask = history_mask & torch.isfinite(history) | |
| history = torch.where(history_mask, history, torch.zeros_like(history)) | |
| hdt = batch["history_dt"][:, -self.history_steps :, None].expand_as(history) / 120.0 | |
| htoken = self.numeric(torch.stack((history, history_mask.to(history.dtype), hdt), -1)) | |
| if history.shape[1] < self.history_steps: | |
| # Left pad only the temporal dimension; padding adds no observations. | |
| pad = htoken.new_zeros( | |
| htoken.shape[0], self.history_steps - history.shape[1], htoken.shape[2], htoken.shape[3] | |
| ) | |
| htoken = torch.cat((pad, htoken), 1) | |
| sequence = htoken.transpose(1, 2) | |
| if self.kind == "gru": | |
| temporal, _ = self.history_temporal(sequence.flatten(0, 1)) | |
| history_token = temporal[:, -1].reshape_as(current_token) | |
| else: | |
| history_token = self.history_temporal(sequence.flatten(2)) | |
| else: | |
| # Current sensors and future controls remain available in this ablation. | |
| history_token = torch.zeros_like(current_token) | |
| field_context = self.norm(current_token + history_token + identity) | |
| # Mask future values *before* the temporal MLP. Every query sees only | |
| # controls up to its horizon, and no field is discarded by pooling. | |
| roles = batch["role_ids"] | |
| action_mask = active & (roles == 1) | |
| exogenous_mask = active & (roles == 2) | |
| future = torch.where(action_mask[:, None, :], batch["future_actions"], 0.0) + torch.where( | |
| exogenous_mask[:, None, :], batch["future_exogenous"], 0.0 | |
| ) | |
| prefix = batch["future_dt"][:, None, :] <= batch["query_horizons"][:, :, None] | |
| future_mask = (action_mask | exogenous_mask)[:, None, :] & torch.isfinite(future) | |
| future = torch.where(future_mask, future, 0.0) | |
| dt = batch["future_dt"][:, :, None].expand_as(future) / 120.0 | |
| dt = torch.where(future_mask, dt, 0.0) | |
| ftoken = self.numeric(torch.stack((future, future_mask.to(future.dtype), dt), -1)) | |
| ftoken = ftoken * future_mask[..., None] | |
| if future.shape[1] != self.future_steps: | |
| raise ValueError("future_steps differs from model configuration") | |
| # Factor the temporal linear map into step contributions, so the | |
| # numerical encoder is evaluated once instead of once per query. | |
| linear = self.future_temporal[0] | |
| weights = linear.weight.reshape(linear.out_features, self.future_steps, ftoken.shape[-1]) | |
| contributions = torch.einsum("bkfd,okd->bkfo", ftoken, weights) | |
| future_token = torch.einsum("bqk,bkfo->bqfo", prefix.to(ftoken.dtype), contributions) + linear.bias | |
| future_token = self.future_temporal[2](self.future_temporal[1](future_token)) | |
| fields = self.norm(field_context[:, None] + future_token) | |
| # Queries carry semantic vocabulary IDs, never slot offsets. | |
| matching = (batch["query_fields"][:, :, None] == batch["field_ids"][:, None, :]) & active[:, None, :] | |
| query_current = torch.einsum("bqf,bfd->bqd", matching.to(current.dtype), field_context) | |
| t = batch["query_horizons"] / 120.0 | |
| qtime = self.time(torch.stack((t, torch.sin(t), torch.cos(t)), -1)) | |
| query = self.field(batch["query_fields"]) + query_current + qtime | |
| return {"fields": fields, "query": query, "active": active, "query_active": matching.any(-1)} | |
| class PreNormBlock(nn.Module): | |
| def __init__(self, config: dict[str, Any]): | |
| super().__init__() | |
| d, drop = config["d_model"], config["dropout"] | |
| self.norm1 = nn.LayerNorm(d) | |
| self.attention = nn.MultiheadAttention(d, config["heads"], dropout=drop, batch_first=True) | |
| self.norm2 = nn.LayerNorm(d) | |
| self.ff = nn.Sequential( | |
| nn.Linear(d, config["ff_dim"]), | |
| nn.GELU(), | |
| nn.Dropout(drop), | |
| nn.Linear(config["ff_dim"], d), | |
| nn.Dropout(drop), | |
| ) | |
| def forward(self, x: torch.Tensor, padding: torch.Tensor) -> torch.Tensor: | |
| z = self.norm1(x) | |
| x = x + self.attention(z, z, z, key_padding_mask=padding, need_weights=False)[0] | |
| return x + self.ff(self.norm2(x)) | |
| class TransitionModel(nn.Module): | |
| def __init__(self, kind: str, config: dict[str, Any]): | |
| super().__init__() | |
| self.kind, self.config = kind, config | |
| self.encoder = SchemaEncoder(kind, config) | |
| d = config["d_model"] | |
| if kind == "transformer": | |
| self.blocks = nn.ModuleList(PreNormBlock(config) for _ in range(config["layers"])) | |
| elif kind in ("gru", "mlp"): | |
| self.fusion = nn.Sequential(nn.Linear(2 * d, 2 * d), nn.GELU(), nn.Linear(2 * d, d)) | |
| else: | |
| raise ValueError(f"Unknown model kind: {kind}") | |
| self.output = nn.Sequential(nn.LayerNorm(d), nn.Linear(d, d), nn.GELU(), nn.Linear(d, 2)) | |
| def forward(self, batch: dict[str, torch.Tensor]) -> tuple[torch.Tensor, torch.Tensor]: | |
| encoded = self.encoder(batch) | |
| fields, query = encoded["fields"], encoded["query"] | |
| active = encoded["active"][:, None, :].expand(-1, query.shape[1], -1) | |
| if self.kind == "transformer": | |
| # Each query has a horizon-specific field backbone. Flattening B,Q | |
| # keeps the attention over fields + one query, never H*F tokens. | |
| tokens = torch.cat((fields, query[:, :, None]), 2).flatten(0, 1) | |
| padding = torch.cat((~active, torch.zeros_like(active[:, :, :1])), 2).flatten(0, 1) | |
| for block in self.blocks: | |
| tokens = block(tokens, padding) | |
| query = tokens[:, -1].reshape_as(query) | |
| else: | |
| context = _mean(fields, active, 2) | |
| query = self.fusion(torch.cat((query, context), -1)) | |
| output = self.output(query) | |
| query_active = encoded["query_active"] | |
| return ( | |
| torch.where(query_active, output[..., 0], 0.0), | |
| torch.where(query_active, output[..., 1].clamp(-7.0, 3.0), 0.0), | |
| ) | |
| def build_model(kind: str, config: dict[str, Any] | None = None) -> TransitionModel: | |
| resolved = default_config() | |
| resolved.update(config or {}) | |
| if resolved["d_model"] % resolved["heads"]: | |
| raise ValueError("d_model must be divisible by heads") | |
| if resolved["layers"] < 1 or resolved["history_steps"] < 1: | |
| raise ValueError("layers and history_steps must be positive") | |
| return TransitionModel(kind, resolved) | |
| def gaussian_nll( | |
| mean: torch.Tensor, log_std: torch.Tensor, target: torch.Tensor, mask: torch.Tensor | |
| ) -> torch.Tensor: | |
| """Mean Gaussian NLL over observed targets; safely excludes padded NaNs.""" | |
| valid = mask.bool() & torch.isfinite(target) | |
| safe_target = torch.where(valid, target, 0.0) | |
| safe_mean = torch.where(valid, mean, 0.0) | |
| safe_log = torch.where(valid, log_std, 0.0).clamp(-7.0, 3.0) | |
| nll = 0.5 * ((safe_target - safe_mean) * torch.exp(-safe_log)).square() + safe_log | |
| nll = nll + 0.5 * math.log(2.0 * math.pi) | |
| return torch.where(valid, nll, 0.0).sum() / valid.sum().clamp_min(1) | |