File size: 2,866 Bytes
04ae9bb
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
"""Configurable MLP / linear model for multi-output regression or distribution heads."""

from __future__ import annotations

from typing import Iterable, List

import torch
import torch.nn as nn

from .registry import register_model


def _activation(name: str) -> nn.Module:
    name = (name or "relu").lower()
    if name == "relu":
        return nn.ReLU()
    if name == "gelu":
        return nn.GELU()
    if name == "silu" or name == "swish":
        return nn.SiLU()
    if name == "tanh":
        return nn.Tanh()
    if name == "leaky_relu":
        return nn.LeakyReLU(0.01)
    raise ValueError(f"Unknown activation: {name}")


def _output_activation(name: str | None) -> nn.Module:
    name = (name or "none").lower()
    if name in {"none", "identity"}:
        return nn.Identity()
    if name == "softplus":
        return nn.Softplus()
    if name == "relu":
        return nn.ReLU()
    if name == "sigmoid":
        return nn.Sigmoid()
    raise ValueError(f"Unknown output activation: {name}")


@register_model("mlp")
class MLP(nn.Module):
    def __init__(
        self,
        input_dim: int,
        output_dim: int,
        hidden_dims: Iterable[int] = (64, 64),
        dropout: float = 0.1,
        activation: str = "relu",
        output_activation: str = "softplus",
        output_type: str = "regression",
        n_classes: int = 87,
        input_dropout: float = 0.0,
    ):
        super().__init__()
        self.input_dim = int(input_dim)
        self.output_dim = int(output_dim)
        self.output_type = output_type
        self.n_classes = int(n_classes)
        dims: List[int] = [self.input_dim] + [int(x) for x in hidden_dims]
        layers: List[nn.Module] = []
        if input_dropout and input_dropout > 0:
            layers.append(nn.Dropout(float(input_dropout)))
        for i in range(len(dims) - 1):
            layers.append(nn.Linear(dims[i], dims[i + 1]))
            layers.append(_activation(activation))
            if dropout and dropout > 0:
                layers.append(nn.Dropout(float(dropout)))
        self.trunk = nn.Sequential(*layers) if layers else nn.Identity()
        final_dim = self.output_dim * self.n_classes if output_type == "distribution" else self.output_dim
        self.head = nn.Linear(dims[-1], final_dim)
        self.out_act = _output_activation(output_activation) if output_type == "regression" else nn.Identity()

    def forward(self, x: torch.Tensor) -> torch.Tensor:
        h = self.trunk(x)
        out = self.head(h)
        if self.output_type == "distribution":
            return out.view(-1, self.output_dim, self.n_classes)
        return self.out_act(out)


@register_model("linear")
class LinearModel(MLP):
    def __init__(self, *args, **kwargs):
        kwargs["hidden_dims"] = []
        kwargs["dropout"] = 0.0
        super().__init__(*args, **kwargs)