fryabcfry's picture
Publish AMP-PD checkpoints, portable loaders and verified metrics (part 3)
04ae9bb verified
Raw History Blame Contribute Delete
2.87 kB
"""Configurable MLP / linear model for multi-output regression or distribution heads."""
from __future__ import annotations
from typing import Iterable, List
import torch
import torch.nn as nn
from .registry import register_model
def _activation(name: str) -> nn.Module:
name = (name or "relu").lower()
if name == "relu":
return nn.ReLU()
if name == "gelu":
return nn.GELU()
if name == "silu" or name == "swish":
return nn.SiLU()
if name == "tanh":
return nn.Tanh()
if name == "leaky_relu":
return nn.LeakyReLU(0.01)
raise ValueError(f"Unknown activation: {name}")
def _output_activation(name: str | None) -> nn.Module:
name = (name or "none").lower()
if name in {"none", "identity"}:
return nn.Identity()
if name == "softplus":
return nn.Softplus()
if name == "relu":
return nn.ReLU()
if name == "sigmoid":
return nn.Sigmoid()
raise ValueError(f"Unknown output activation: {name}")
@register_model("mlp")
class MLP(nn.Module):
def __init__(
self,
input_dim: int,
output_dim: int,
hidden_dims: Iterable[int] = (64, 64),
dropout: float = 0.1,
activation: str = "relu",
output_activation: str = "softplus",
output_type: str = "regression",
n_classes: int = 87,
input_dropout: float = 0.0,
):
super().__init__()
self.input_dim = int(input_dim)
self.output_dim = int(output_dim)
self.output_type = output_type
self.n_classes = int(n_classes)
dims: List[int] = [self.input_dim] + [int(x) for x in hidden_dims]
layers: List[nn.Module] = []
if input_dropout and input_dropout > 0:
layers.append(nn.Dropout(float(input_dropout)))
for i in range(len(dims) - 1):
layers.append(nn.Linear(dims[i], dims[i + 1]))
layers.append(_activation(activation))
if dropout and dropout > 0:
layers.append(nn.Dropout(float(dropout)))
self.trunk = nn.Sequential(*layers) if layers else nn.Identity()
final_dim = self.output_dim * self.n_classes if output_type == "distribution" else self.output_dim
self.head = nn.Linear(dims[-1], final_dim)
self.out_act = _output_activation(output_activation) if output_type == "regression" else nn.Identity()
def forward(self, x: torch.Tensor) -> torch.Tensor:
h = self.trunk(x)
out = self.head(h)
if self.output_type == "distribution":
return out.view(-1, self.output_dim, self.n_classes)
return self.out_act(out)
@register_model("linear")
class LinearModel(MLP):
def __init__(self, *args, **kwargs):
kwargs["hidden_dims"] = []
kwargs["dropout"] = 0.0
super().__init__(*args, **kwargs)