Download arena-methods/runtime/model.py from lisuo/PFN-DiD: direct link, hf CLI and curl.
- Browser
- Download file 4.86 kB
-
https://huggingface.co/lisuo/PFN-DiD/resolve/main/arena-methods/runtime/model.py
- Command line
-
hf download hf://lisuo/PFN-DiD/arena-methods/runtime/model.py
-
curl -L -o model.py https://huggingface.co/lisuo/PFN-DiD/resolve/main/arena-methods/runtime/model.py
4.86 kB
| """TabPFN-3.5 internal nuisance adapters; no target-table optimization. | |
| """ | |
| from collections import defaultdict | |
| import numpy as np | |
| import torch | |
| from torch import nn | |
| from sklearn.model_selection import StratifiedKFold | |
| from .score import validate_table | |
| class TaskBottleneck(nn.Module): | |
| def __init__(self, width, hidden_dim=8, dropout=0.05): | |
| super().__init__() | |
| self.net = nn.Sequential(nn.LayerNorm(width), nn.Linear(width, hidden_dim), | |
| nn.GELU(), nn.Dropout(dropout), nn.Linear(hidden_dim, width)) | |
| nn.init.zeros_(self.net[-1].weight) | |
| nn.init.zeros_(self.net[-1].bias) | |
| def forward(self, hidden): | |
| return hidden + self.net(hidden.float()).to(hidden.dtype) | |
| class PFNDiD(nn.Module): | |
| def __init__(self, seed=39007, hidden_dim=8, dropout=0.05, | |
| weight_path=None, | |
| backbone=None, criterion=None): | |
| super().__init__() | |
| torch.manual_seed(seed) | |
| if backbone is None: | |
| from tabpfn.model_loading import load_model | |
| from pathlib import Path | |
| backbone, criterion, _, _ = load_model(path=Path(weight_path), estimator_type="regressor") | |
| if criterion is None or len(backbone.icl_blocks) < 4: | |
| raise ValueError("A regression criterion and at least four ICL blocks are required") | |
| self.backbone = backbone.requires_grad_(False) | |
| self.criterion = criterion | |
| if isinstance(criterion, nn.Module): | |
| criterion.requires_grad_(False) | |
| self.adapters = nn.ModuleDict({task: nn.ModuleList([ | |
| TaskBottleneck(backbone.embedding_dim, hidden_dim, dropout) for _ in range(4) | |
| ]) for task in ("ps", "or")}) | |
| self._active_task = None | |
| for index, block in enumerate(backbone.icl_blocks[-4:]): | |
| block.register_forward_hook(self._hook(index)) | |
| self.train(False) | |
| def _hook(self, index): | |
| def apply(module, args, output): | |
| if self._active_task is None: | |
| return output | |
| hidden, cache = output | |
| return self.adapters[self._active_task][index](hidden), cache | |
| return apply | |
| def train(self, mode=True): | |
| super().train(mode) | |
| self.backbone.eval() | |
| self.adapters.train(mode) | |
| return self | |
| def forward(self, x, d, y, n_splits=5, fold_seed=39007): | |
| """Batch equally sized contexts on an independent backbone batch axis.""" | |
| x, d, y = validate_table(x, d, y) | |
| if min(np.bincount(d)) < n_splits or n_splits < 2: | |
| raise ValueError("Insufficient arm counts for cross-fitting") | |
| device = next(self.adapters.parameters()).device | |
| # Keep the original float32 conversion before constructing fold contexts. | |
| xt, dt, yt = [torch.tensor(v, dtype=torch.float32, device=device) for v in (x,d,y)] | |
| if xt.shape[1] == 0: | |
| xt = xt.new_zeros((len(xt), 1)) | |
| groups = defaultdict(list) | |
| folds = StratifiedKFold(n_splits, shuffle=True, random_state=fold_seed) | |
| for fit, query in folds.split(x,d): | |
| for task, context in (("ps",fit),("or",fit[d[fit] == 0])): | |
| groups[(task,len(context),len(query))].append((context,query)) | |
| predictions = {key: torch.empty(len(d), device=device) for key in | |
| ("propensity", "outcome")} | |
| for (task, _, _), members in groups.items(): | |
| # At most five folds in a group, below the evaluation batch limit of eight. | |
| features, labels, centers, scales = [], [], [], [] | |
| for context, query in members: | |
| xc = xt[context] | |
| features.append((torch.cat((xc,xt[query]))-xc.mean(0)) / | |
| xc.std(0,unbiased=False).clamp_min(.01)) | |
| if task == "ps": | |
| labels.append(dt[context]) | |
| else: | |
| center=yt[context].mean() | |
| scale=yt[context].std(unbiased=False).clamp_min(.01) | |
| centers.append(center); scales.append(scale) | |
| labels.append((yt[context]-center)/scale) | |
| self._active_task = task | |
| try: | |
| raw=self.backbone(torch.stack(features,dim=1),torch.stack(labels,dim=1), | |
| task_type="multiclass" if task == "ps" else "regression") | |
| finally: | |
| self._active_task = None | |
| if task == "ps": | |
| values=(raw[...,1]-raw[...,0]).sigmoid() | |
| key="propensity" | |
| else: | |
| values=self.criterion.mean(raw)*torch.stack(scales)[None,:]+torch.stack(centers)[None,:] | |
| key="outcome" | |
| for column, (_, query) in enumerate(members): | |
| predictions[key][query]=values[:,column] | |
| return predictions | |