File size: 4,861 Bytes
b7ab973 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 | """TabPFN-3.5 internal nuisance adapters; no target-table optimization.
"""
from collections import defaultdict
import numpy as np
import torch
from torch import nn
from sklearn.model_selection import StratifiedKFold
from .score import validate_table
class TaskBottleneck(nn.Module):
def __init__(self, width, hidden_dim=8, dropout=0.05):
super().__init__()
self.net = nn.Sequential(nn.LayerNorm(width), nn.Linear(width, hidden_dim),
nn.GELU(), nn.Dropout(dropout), nn.Linear(hidden_dim, width))
nn.init.zeros_(self.net[-1].weight)
nn.init.zeros_(self.net[-1].bias)
def forward(self, hidden):
return hidden + self.net(hidden.float()).to(hidden.dtype)
class PFNDiD(nn.Module):
def __init__(self, seed=39007, hidden_dim=8, dropout=0.05,
weight_path=None,
backbone=None, criterion=None):
super().__init__()
torch.manual_seed(seed)
if backbone is None:
from tabpfn.model_loading import load_model
from pathlib import Path
backbone, criterion, _, _ = load_model(path=Path(weight_path), estimator_type="regressor")
if criterion is None or len(backbone.icl_blocks) < 4:
raise ValueError("A regression criterion and at least four ICL blocks are required")
self.backbone = backbone.requires_grad_(False)
self.criterion = criterion
if isinstance(criterion, nn.Module):
criterion.requires_grad_(False)
self.adapters = nn.ModuleDict({task: nn.ModuleList([
TaskBottleneck(backbone.embedding_dim, hidden_dim, dropout) for _ in range(4)
]) for task in ("ps", "or")})
self._active_task = None
for index, block in enumerate(backbone.icl_blocks[-4:]):
block.register_forward_hook(self._hook(index))
self.train(False)
def _hook(self, index):
def apply(module, args, output):
if self._active_task is None:
return output
hidden, cache = output
return self.adapters[self._active_task][index](hidden), cache
return apply
def train(self, mode=True):
super().train(mode)
self.backbone.eval()
self.adapters.train(mode)
return self
def forward(self, x, d, y, n_splits=5, fold_seed=39007):
"""Batch equally sized contexts on an independent backbone batch axis."""
x, d, y = validate_table(x, d, y)
if min(np.bincount(d)) < n_splits or n_splits < 2:
raise ValueError("Insufficient arm counts for cross-fitting")
device = next(self.adapters.parameters()).device
# Keep the original float32 conversion before constructing fold contexts.
xt, dt, yt = [torch.tensor(v, dtype=torch.float32, device=device) for v in (x,d,y)]
if xt.shape[1] == 0:
xt = xt.new_zeros((len(xt), 1))
groups = defaultdict(list)
folds = StratifiedKFold(n_splits, shuffle=True, random_state=fold_seed)
for fit, query in folds.split(x,d):
for task, context in (("ps",fit),("or",fit[d[fit] == 0])):
groups[(task,len(context),len(query))].append((context,query))
predictions = {key: torch.empty(len(d), device=device) for key in
("propensity", "outcome")}
for (task, _, _), members in groups.items():
# At most five folds in a group, below the evaluation batch limit of eight.
features, labels, centers, scales = [], [], [], []
for context, query in members:
xc = xt[context]
features.append((torch.cat((xc,xt[query]))-xc.mean(0)) /
xc.std(0,unbiased=False).clamp_min(.01))
if task == "ps":
labels.append(dt[context])
else:
center=yt[context].mean()
scale=yt[context].std(unbiased=False).clamp_min(.01)
centers.append(center); scales.append(scale)
labels.append((yt[context]-center)/scale)
self._active_task = task
try:
raw=self.backbone(torch.stack(features,dim=1),torch.stack(labels,dim=1),
task_type="multiclass" if task == "ps" else "regression")
finally:
self._active_task = None
if task == "ps":
values=(raw[...,1]-raw[...,0]).sigmoid()
key="propensity"
else:
values=self.criterion.mean(raw)*torch.stack(scales)[None,:]+torch.stack(centers)[None,:]
key="outcome"
for column, (_, query) in enumerate(members):
predictions[key][query]=values[:,column]
return predictions
|