AI4deeperScience commited on
Commit
d97c640
·
verified ·
1 Parent(s): 64fcdda

Upload src/neural_models.py with huggingface_hub

Browse files
Files changed (1) hide show
  1. src/neural_models.py +158 -0
src/neural_models.py ADDED
@@ -0,0 +1,158 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Neural network architectures for descriptor and fingerprint baselines."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from typing import List, Optional, Tuple, cast
6
+
7
+ import torch
8
+ import torch.nn as nn
9
+
10
+
11
+ class DescriptorNN(nn.Module):
12
+ """Feed-forward network for molecular descriptors with optional lab embedding."""
13
+
14
+ def __init__(
15
+ self,
16
+ input_dim: int,
17
+ hidden_dims: List[int] | Tuple[int, ...] = (192, 96, 48),
18
+ dropout: float = 0.2,
19
+ use_batch_norm: bool = True,
20
+ num_labs: Optional[int] = None,
21
+ lab_embed_dim: int = 12,
22
+ input_dropout: float = 0.05,
23
+ ) -> None:
24
+ super().__init__()
25
+
26
+ self.use_lab_embedding = num_labs is not None and num_labs > 0
27
+ lab_count = cast(int, num_labs) if self.use_lab_embedding else 0
28
+ self.lab_embedding = (
29
+ nn.Embedding(lab_count, lab_embed_dim)
30
+ if self.use_lab_embedding
31
+ else None
32
+ )
33
+
34
+ initial_dim = input_dim + (lab_embed_dim if self.use_lab_embedding else 0)
35
+ self.input_dropout = nn.Dropout(input_dropout) if input_dropout > 0 else None
36
+
37
+ layers: List[nn.Module] = []
38
+ prev_dim = initial_dim
39
+ for hidden_dim in hidden_dims:
40
+ layers.append(nn.Linear(prev_dim, hidden_dim))
41
+ if use_batch_norm:
42
+ layers.append(nn.BatchNorm1d(hidden_dim))
43
+ layers.append(nn.GELU())
44
+ layers.append(nn.Dropout(dropout))
45
+ prev_dim = hidden_dim
46
+
47
+ bottleneck_dim = max(32, prev_dim // 2)
48
+ layers.extend(
49
+ [
50
+ nn.Linear(prev_dim, bottleneck_dim),
51
+ nn.GELU(),
52
+ nn.Dropout(dropout * 0.5),
53
+ nn.Linear(bottleneck_dim, 1),
54
+ ]
55
+ )
56
+
57
+ self.network = nn.Sequential(*layers)
58
+ self.apply(self._init_weights)
59
+
60
+ # placeholders populated during training for inverse scaling
61
+ self.target_mean: float = 0.0
62
+ self.target_std: float = 1.0
63
+
64
+ @staticmethod
65
+ def _init_weights(module: nn.Module) -> None:
66
+ if isinstance(module, nn.Linear):
67
+ nn.init.xavier_uniform_(module.weight)
68
+ if module.bias is not None:
69
+ nn.init.zeros_(module.bias)
70
+ elif isinstance(module, nn.BatchNorm1d):
71
+ nn.init.ones_(module.weight)
72
+ nn.init.zeros_(module.bias)
73
+
74
+ def forward(self, x: torch.Tensor, lab_indices: Optional[torch.Tensor] = None) -> torch.Tensor:
75
+ if self.input_dropout is not None:
76
+ x = self.input_dropout(x)
77
+
78
+ if self.lab_embedding is not None:
79
+ if lab_indices is None:
80
+ raise ValueError("lab_indices must be provided when lab embedding is enabled.")
81
+ if lab_indices.dim() > 1:
82
+ lab_indices = lab_indices.squeeze(-1)
83
+ lab_embeddings = self.lab_embedding(lab_indices.long())
84
+ x = torch.cat([x, lab_embeddings], dim=-1)
85
+
86
+ return self.network(x).squeeze(-1)
87
+
88
+
89
+ class FingerprintNN(nn.Module):
90
+ """Feed-forward network tailored for high-dimensional sparse fingerprints."""
91
+
92
+ def __init__(
93
+ self,
94
+ input_dim: int,
95
+ hidden_dims: List[int] | Tuple[int, ...] = (768, 384, 192, 96),
96
+ dropout: float = 0.15,
97
+ use_batch_norm: bool = True,
98
+ num_labs: Optional[int] = None,
99
+ lab_embed_dim: int = 16,
100
+ input_dropout: float = 0.1,
101
+ ) -> None:
102
+ super().__init__()
103
+
104
+ self.use_lab_embedding = num_labs is not None and num_labs > 0
105
+ lab_count = cast(int, num_labs) if self.use_lab_embedding else 0
106
+ self.lab_embedding = (
107
+ nn.Embedding(lab_count, lab_embed_dim)
108
+ if self.use_lab_embedding
109
+ else None
110
+ )
111
+
112
+ initial_dim = input_dim + (lab_embed_dim if self.use_lab_embedding else 0)
113
+ self.input_dropout = nn.Dropout(input_dropout) if input_dropout > 0 else None
114
+
115
+ layers: List[nn.Module] = []
116
+ prev_dim = initial_dim
117
+ for idx, hidden_dim in enumerate(hidden_dims):
118
+ layers.append(nn.Linear(prev_dim, hidden_dim))
119
+ if use_batch_norm:
120
+ layers.append(nn.BatchNorm1d(hidden_dim))
121
+ layers.append(nn.GELU())
122
+ layers.append(nn.Dropout(dropout if idx < len(hidden_dims) - 1 else dropout * 0.5))
123
+ prev_dim = hidden_dim
124
+
125
+ layers.append(nn.Linear(prev_dim, 1))
126
+
127
+ self.network = nn.Sequential(*layers)
128
+ self.apply(self._init_weights)
129
+
130
+ self.target_mean: float = 0.0
131
+ self.target_std: float = 1.0
132
+
133
+ @staticmethod
134
+ def _init_weights(module: nn.Module) -> None:
135
+ if isinstance(module, nn.Linear):
136
+ nn.init.xavier_uniform_(module.weight)
137
+ if module.bias is not None:
138
+ nn.init.zeros_(module.bias)
139
+ elif isinstance(module, nn.BatchNorm1d):
140
+ nn.init.ones_(module.weight)
141
+ nn.init.zeros_(module.bias)
142
+
143
+ def forward(self, x: torch.Tensor, lab_indices: Optional[torch.Tensor] = None) -> torch.Tensor:
144
+ if self.input_dropout is not None:
145
+ x = self.input_dropout(x)
146
+
147
+ if self.lab_embedding is not None:
148
+ if lab_indices is None:
149
+ raise ValueError("lab_indices must be provided when lab embedding is enabled.")
150
+ if lab_indices.dim() > 1:
151
+ lab_indices = lab_indices.squeeze(-1)
152
+ lab_embeddings = self.lab_embedding(lab_indices.long())
153
+ x = torch.cat([x, lab_embeddings], dim=-1)
154
+
155
+ return self.network(x).squeeze(-1)
156
+
157
+
158
+ __all__ = ["DescriptorNN", "FingerprintNN"]