"""TabPFN-3.5 internal nuisance adapters; no target-table optimization. """ from collections import defaultdict import numpy as np import torch from torch import nn from sklearn.model_selection import StratifiedKFold from .score import validate_table class TaskBottleneck(nn.Module): def __init__(self, width, hidden_dim=8, dropout=0.05): super().__init__() self.net = nn.Sequential(nn.LayerNorm(width), nn.Linear(width, hidden_dim), nn.GELU(), nn.Dropout(dropout), nn.Linear(hidden_dim, width)) nn.init.zeros_(self.net[-1].weight) nn.init.zeros_(self.net[-1].bias) def forward(self, hidden): return hidden + self.net(hidden.float()).to(hidden.dtype) class PFNDiD(nn.Module): def __init__(self, seed=39007, hidden_dim=8, dropout=0.05, weight_path=None, backbone=None, criterion=None): super().__init__() torch.manual_seed(seed) if backbone is None: from tabpfn.model_loading import load_model from pathlib import Path backbone, criterion, _, _ = load_model(path=Path(weight_path), estimator_type="regressor") if criterion is None or len(backbone.icl_blocks) < 4: raise ValueError("A regression criterion and at least four ICL blocks are required") self.backbone = backbone.requires_grad_(False) self.criterion = criterion if isinstance(criterion, nn.Module): criterion.requires_grad_(False) self.adapters = nn.ModuleDict({task: nn.ModuleList([ TaskBottleneck(backbone.embedding_dim, hidden_dim, dropout) for _ in range(4) ]) for task in ("ps", "or")}) self._active_task = None for index, block in enumerate(backbone.icl_blocks[-4:]): block.register_forward_hook(self._hook(index)) self.train(False) def _hook(self, index): def apply(module, args, output): if self._active_task is None: return output hidden, cache = output return self.adapters[self._active_task][index](hidden), cache return apply def train(self, mode=True): super().train(mode) self.backbone.eval() self.adapters.train(mode) return self def forward(self, x, d, y, n_splits=5, fold_seed=39007): """Batch equally sized contexts on an independent backbone batch axis.""" x, d, y = validate_table(x, d, y) if min(np.bincount(d)) < n_splits or n_splits < 2: raise ValueError("Insufficient arm counts for cross-fitting") device = next(self.adapters.parameters()).device # Keep the original float32 conversion before constructing fold contexts. xt, dt, yt = [torch.tensor(v, dtype=torch.float32, device=device) for v in (x,d,y)] if xt.shape[1] == 0: xt = xt.new_zeros((len(xt), 1)) groups = defaultdict(list) folds = StratifiedKFold(n_splits, shuffle=True, random_state=fold_seed) for fit, query in folds.split(x,d): for task, context in (("ps",fit),("or",fit[d[fit] == 0])): groups[(task,len(context),len(query))].append((context,query)) predictions = {key: torch.empty(len(d), device=device) for key in ("propensity", "outcome")} for (task, _, _), members in groups.items(): # At most five folds in a group, below the evaluation batch limit of eight. features, labels, centers, scales = [], [], [], [] for context, query in members: xc = xt[context] features.append((torch.cat((xc,xt[query]))-xc.mean(0)) / xc.std(0,unbiased=False).clamp_min(.01)) if task == "ps": labels.append(dt[context]) else: center=yt[context].mean() scale=yt[context].std(unbiased=False).clamp_min(.01) centers.append(center); scales.append(scale) labels.append((yt[context]-center)/scale) self._active_task = task try: raw=self.backbone(torch.stack(features,dim=1),torch.stack(labels,dim=1), task_type="multiclass" if task == "ps" else "regression") finally: self._active_task = None if task == "ps": values=(raw[...,1]-raw[...,0]).sigmoid() key="propensity" else: values=self.criterion.mean(raw)*torch.stack(scales)[None,:]+torch.stack(centers)[None,:] key="outcome" for column, (_, query) in enumerate(members): predictions[key][query]=values[:,column] return predictions