File size: 4,861 Bytes
b7ab973
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
"""TabPFN-3.5 internal nuisance adapters; no target-table optimization.
"""
from collections import defaultdict
import numpy as np
import torch
from torch import nn
from sklearn.model_selection import StratifiedKFold
from .score import validate_table



class TaskBottleneck(nn.Module):
    def __init__(self, width, hidden_dim=8, dropout=0.05):
        super().__init__()
        self.net = nn.Sequential(nn.LayerNorm(width), nn.Linear(width, hidden_dim),
                                 nn.GELU(), nn.Dropout(dropout), nn.Linear(hidden_dim, width))
        nn.init.zeros_(self.net[-1].weight)
        nn.init.zeros_(self.net[-1].bias)

    def forward(self, hidden):
        return hidden + self.net(hidden.float()).to(hidden.dtype)


class PFNDiD(nn.Module):
    def __init__(self, seed=39007, hidden_dim=8, dropout=0.05,
                 weight_path=None,
                 backbone=None, criterion=None):
        super().__init__()
        torch.manual_seed(seed)
        if backbone is None:
            from tabpfn.model_loading import load_model
            from pathlib import Path
            backbone, criterion, _, _ = load_model(path=Path(weight_path), estimator_type="regressor")
        if criterion is None or len(backbone.icl_blocks) < 4:
            raise ValueError("A regression criterion and at least four ICL blocks are required")
        self.backbone = backbone.requires_grad_(False)
        self.criterion = criterion
        if isinstance(criterion, nn.Module):
            criterion.requires_grad_(False)
        self.adapters = nn.ModuleDict({task: nn.ModuleList([
            TaskBottleneck(backbone.embedding_dim, hidden_dim, dropout) for _ in range(4)
        ]) for task in ("ps", "or")})
        self._active_task = None
        for index, block in enumerate(backbone.icl_blocks[-4:]):
            block.register_forward_hook(self._hook(index))
        self.train(False)

    def _hook(self, index):
        def apply(module, args, output):
            if self._active_task is None:
                return output
            hidden, cache = output
            return self.adapters[self._active_task][index](hidden), cache
        return apply

    def train(self, mode=True):
        super().train(mode)
        self.backbone.eval()
        self.adapters.train(mode)
        return self

    def forward(self, x, d, y, n_splits=5, fold_seed=39007):
        """Batch equally sized contexts on an independent backbone batch axis."""
        x, d, y = validate_table(x, d, y)
        if min(np.bincount(d)) < n_splits or n_splits < 2:
            raise ValueError("Insufficient arm counts for cross-fitting")
        device = next(self.adapters.parameters()).device
        # Keep the original float32 conversion before constructing fold contexts.
        xt, dt, yt = [torch.tensor(v, dtype=torch.float32, device=device) for v in (x,d,y)]
        if xt.shape[1] == 0:
            xt = xt.new_zeros((len(xt), 1))
        groups = defaultdict(list)
        folds = StratifiedKFold(n_splits, shuffle=True, random_state=fold_seed)
        for fit, query in folds.split(x,d):
            for task, context in (("ps",fit),("or",fit[d[fit] == 0])):
                groups[(task,len(context),len(query))].append((context,query))
        predictions = {key: torch.empty(len(d), device=device) for key in
                       ("propensity", "outcome")}
        for (task, _, _), members in groups.items():
            # At most five folds in a group, below the evaluation batch limit of eight.
            features, labels, centers, scales = [], [], [], []
            for context, query in members:
                xc = xt[context]
                features.append((torch.cat((xc,xt[query]))-xc.mean(0)) /
                                xc.std(0,unbiased=False).clamp_min(.01))
                if task == "ps":
                    labels.append(dt[context])
                else:
                    center=yt[context].mean()
                    scale=yt[context].std(unbiased=False).clamp_min(.01)
                    centers.append(center); scales.append(scale)
                    labels.append((yt[context]-center)/scale)
            self._active_task = task
            try:
                raw=self.backbone(torch.stack(features,dim=1),torch.stack(labels,dim=1),
                                  task_type="multiclass" if task == "ps" else "regression")
            finally:
                self._active_task = None
            if task == "ps":
                values=(raw[...,1]-raw[...,0]).sigmoid()
                key="propensity"
            else:
                values=self.criterion.mean(raw)*torch.stack(scales)[None,:]+torch.stack(centers)[None,:]
                key="outcome"
            for column, (_, query) in enumerate(members):
                predictions[key][query]=values[:,column]
        return predictions