AI4deeperScience commited on
Commit
00f7a8e
·
verified ·
1 Parent(s): 044bd78

Upload src/ensemble.py with huggingface_hub

Browse files
Files changed (1) hide show
  1. src/ensemble.py +460 -0
src/ensemble.py ADDED
@@ -0,0 +1,460 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Ensemble and stacking module for retention time prediction.
3
+ Combines predictions from multiple base models using second-level models.
4
+ """
5
+
6
+ import numpy as np
7
+ import pandas as pd
8
+ import torch
9
+ import torch.nn as nn
10
+ import torch.optim as optim
11
+ from sklearn.linear_model import Ridge, Lasso, ElasticNet
12
+ from sklearn.ensemble import RandomForestRegressor
13
+ from sklearn.model_selection import cross_val_score
14
+ from sklearn.metrics import mean_absolute_error, mean_squared_error, r2_score
15
+ from sklearn.preprocessing import StandardScaler
16
+ import lightgbm as lgb
17
+ import xgboost as xgb
18
+ import joblib
19
+ import os
20
+ from typing import Dict, List, Tuple, Any, Optional
21
+ import warnings
22
+ warnings.filterwarnings('ignore')
23
+
24
+ from .data import RTDataset, create_graph_dataloader
25
+ from .train import calculate_metrics, GNNTrainer, DescriptorTrainer
26
+
27
+
28
+ class MLPStacker(nn.Module):
29
+ """Multi-Layer Perceptron for stacking predictions."""
30
+
31
+ def __init__(self, input_dim: int, hidden_dims: List[int] = [64, 32],
32
+ dropout: float = 0.3, output_dim: int = 1):
33
+ super().__init__()
34
+
35
+ layers = []
36
+ prev_dim = input_dim
37
+
38
+ for hidden_dim in hidden_dims:
39
+ layers.extend([
40
+ nn.Linear(prev_dim, hidden_dim),
41
+ nn.ReLU(),
42
+ nn.Dropout(dropout),
43
+ nn.BatchNorm1d(hidden_dim)
44
+ ])
45
+ prev_dim = hidden_dim
46
+
47
+ layers.append(nn.Linear(prev_dim, output_dim))
48
+
49
+ self.network = nn.Sequential(*layers)
50
+
51
+ def forward(self, x):
52
+ return self.network(x).squeeze()
53
+
54
+
55
+ class EnsemblePredictor:
56
+ """Ensemble predictor class for combining multiple models."""
57
+
58
+ def __init__(self, save_dir: str = "ensemble_models"):
59
+ self.save_dir = save_dir
60
+ self.meta_model = None
61
+ self.meta_scaler = StandardScaler()
62
+ self.base_models = {}
63
+ self.model_weights = {}
64
+
65
+ os.makedirs(save_dir, exist_ok=True)
66
+
67
+ def simple_average(self, predictions: Dict[str, np.ndarray],
68
+ weights: Optional[Dict[str, float]] = None) -> np.ndarray:
69
+ """Simple weighted average of predictions."""
70
+ if weights is None:
71
+ weights = {name: 1.0 for name in predictions.keys()}
72
+
73
+ # Normalize weights
74
+ total_weight = sum(weights.values())
75
+ weights = {name: w / total_weight for name, w in weights.items()}
76
+
77
+ avg_pred = np.zeros(len(next(iter(predictions.values()))))
78
+
79
+ for name, pred in predictions.items():
80
+ avg_pred += weights[name] * pred
81
+
82
+ return avg_pred
83
+
84
+ def rank_average(self, predictions: Dict[str, np.ndarray]) -> np.ndarray:
85
+ """Rank-based averaging of predictions."""
86
+ # Convert predictions to ranks
87
+ ranked_predictions = {}
88
+ for name, pred in predictions.items():
89
+ ranked_predictions[name] = np.argsort(np.argsort(pred))
90
+
91
+ # Average ranks
92
+ avg_ranks = self.simple_average(ranked_predictions)
93
+
94
+ # Convert back to values using first model's sorted values
95
+ first_model_pred = next(iter(predictions.values()))
96
+ sorted_values = np.sort(first_model_pred)
97
+
98
+ # Map average ranks to values
99
+ final_pred = np.zeros_like(avg_ranks)
100
+ for i, rank in enumerate(avg_ranks):
101
+ idx = int(np.clip(rank, 0, len(sorted_values) - 1))
102
+ final_pred[i] = sorted_values[idx]
103
+
104
+ return final_pred
105
+
106
+ def train_meta_model(self, base_predictions: Dict[str, np.ndarray],
107
+ targets: np.ndarray, meta_model_type: str = 'lgb',
108
+ cv_folds: int = 5) -> Any:
109
+ """Train meta-model for stacking."""
110
+
111
+ # Prepare meta-features
112
+ X_meta = np.column_stack([pred for pred in base_predictions.values()])
113
+ X_meta = self.meta_scaler.fit_transform(X_meta)
114
+
115
+ print(f"Training meta-model ({meta_model_type}) with {X_meta.shape[1]} base predictions...")
116
+
117
+ if meta_model_type == 'lgb':
118
+ self.meta_model = lgb.LGBMRegressor(
119
+ objective='regression',
120
+ metric='rmse',
121
+ boosting_type='gbdt',
122
+ num_leaves=15,
123
+ learning_rate=0.1,
124
+ feature_fraction=0.8,
125
+ bagging_fraction=0.8,
126
+ bagging_freq=5,
127
+ min_child_samples=20,
128
+ verbose=-1,
129
+ random_state=42
130
+ )
131
+
132
+ elif meta_model_type == 'xgb':
133
+ self.meta_model = xgb.XGBRegressor(
134
+ objective='reg:squarederror',
135
+ n_estimators=500,
136
+ max_depth=4,
137
+ learning_rate=0.1,
138
+ subsample=0.8,
139
+ colsample_bytree=0.8,
140
+ random_state=42
141
+ )
142
+
143
+ elif meta_model_type == 'ridge':
144
+ self.meta_model = Ridge(alpha=1.0, random_state=42)
145
+
146
+ elif meta_model_type == 'elastic':
147
+ self.meta_model = ElasticNet(alpha=0.5, l1_ratio=0.5, random_state=42)
148
+
149
+ elif meta_model_type == 'rf':
150
+ self.meta_model = RandomForestRegressor(
151
+ n_estimators=200,
152
+ max_depth=10,
153
+ min_samples_split=10,
154
+ random_state=42,
155
+ n_jobs=-1
156
+ )
157
+
158
+ elif meta_model_type == 'mlp':
159
+ # Use PyTorch MLP
160
+ device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
161
+ self.meta_model = MLPStacker(X_meta.shape[1], hidden_dims=[32, 16])
162
+ self.meta_model = self.meta_model.to(device)
163
+
164
+ # Train MLP
165
+ self._train_mlp_meta_model(X_meta, targets, device)
166
+ return self.meta_model
167
+
168
+ else:
169
+ raise ValueError(f"Unknown meta-model type: {meta_model_type}")
170
+
171
+ # Train meta-model
172
+ self.meta_model.fit(X_meta, targets)
173
+
174
+ # Evaluate with cross-validation
175
+ cv_scores = cross_val_score(self.meta_model, X_meta, targets,
176
+ cv=cv_folds, scoring='neg_mean_absolute_error')
177
+ print(f"Meta-model CV MAE: {-cv_scores.mean():.4f} (+/- {cv_scores.std() * 2:.4f})")
178
+
179
+ # Save meta-model
180
+ model_path = os.path.join(self.save_dir, f"meta_model_{meta_model_type}.joblib")
181
+ joblib.dump(self.meta_model, model_path)
182
+
183
+ scaler_path = os.path.join(self.save_dir, "meta_scaler.joblib")
184
+ joblib.dump(self.meta_scaler, scaler_path)
185
+
186
+ return self.meta_model
187
+
188
+ def _train_mlp_meta_model(self, X: np.ndarray, y: np.ndarray, device: torch.device):
189
+ """Train MLP meta-model."""
190
+ X_tensor = torch.FloatTensor(X).to(device)
191
+ y_tensor = torch.FloatTensor(y).to(device)
192
+
193
+ optimizer = optim.AdamW(self.meta_model.parameters(), lr=0.001, weight_decay=1e-4)
194
+ criterion = nn.MSELoss()
195
+ scheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer, patience=10)
196
+
197
+ # Training loop
198
+ self.meta_model.train()
199
+ for epoch in range(500):
200
+ optimizer.zero_grad()
201
+ pred = self.meta_model(X_tensor)
202
+ loss = criterion(pred, y_tensor)
203
+ loss.backward()
204
+ optimizer.step()
205
+ scheduler.step(loss)
206
+
207
+ if epoch % 100 == 0:
208
+ print(f"MLP Meta-model Epoch {epoch}: Loss = {loss.item():.6f}")
209
+
210
+ def predict_meta(self, base_predictions: Dict[str, np.ndarray]) -> np.ndarray:
211
+ """Make predictions using trained meta-model."""
212
+ if self.meta_model is None:
213
+ raise ValueError("Meta-model not trained yet!")
214
+
215
+ # Prepare meta-features
216
+ X_meta = np.column_stack([pred for pred in base_predictions.values()])
217
+ X_meta = self.meta_scaler.transform(X_meta)
218
+
219
+ if isinstance(self.meta_model, nn.Module):
220
+ # PyTorch MLP
221
+ device = next(self.meta_model.parameters()).device
222
+ X_tensor = torch.FloatTensor(X_meta).to(device)
223
+
224
+ self.meta_model.eval()
225
+ with torch.no_grad():
226
+ predictions = self.meta_model(X_tensor).cpu().numpy()
227
+ else:
228
+ # Scikit-learn model
229
+ predictions = self.meta_model.predict(X_meta)
230
+
231
+ return predictions
232
+
233
+ def optimize_weights(self, predictions: Dict[str, np.ndarray],
234
+ targets: np.ndarray, method: str = 'nelder-mead') -> Dict[str, float]:
235
+ """Optimize ensemble weights using scipy optimization."""
236
+ from scipy.optimize import minimize
237
+
238
+ model_names = list(predictions.keys())
239
+ n_models = len(model_names)
240
+
241
+ def objective(weights):
242
+ weights = weights / weights.sum() # Normalize
243
+ ensemble_pred = np.zeros_like(targets)
244
+ for i, name in enumerate(model_names):
245
+ ensemble_pred += weights[i] * predictions[name]
246
+ return mean_squared_error(targets, ensemble_pred)
247
+
248
+ # Constraints: weights sum to 1 and are non-negative
249
+ constraints = {'type': 'eq', 'fun': lambda w: w.sum() - 1}
250
+ bounds = [(0, 1) for _ in range(n_models)]
251
+
252
+ # Initial guess (equal weights)
253
+ x0 = np.ones(n_models) / n_models
254
+
255
+ # Optimize
256
+ result = minimize(objective, x0, method=method, bounds=bounds, constraints=constraints)
257
+
258
+ if result.success:
259
+ optimal_weights = dict(zip(model_names, result.x))
260
+ print(f"Optimized weights: {optimal_weights}")
261
+ return optimal_weights
262
+ else:
263
+ print("Weight optimization failed, using equal weights")
264
+ return {name: 1.0 / n_models for name in model_names}
265
+
266
+
267
+ def create_ensemble_predictions(train_results: Dict[str, Any],
268
+ test_dataset: RTDataset,
269
+ ensemble_methods: List[str] = ['average', 'stacking'],
270
+ device: torch.device = None) -> Dict[str, np.ndarray]:
271
+ """Create ensemble predictions for test set."""
272
+
273
+ if device is None:
274
+ device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
275
+
276
+ print("Generating test predictions from base models...")
277
+
278
+ # Get test data
279
+ test_graphs, test_lab_features, _ = test_dataset.preprocess_data(
280
+ lab_encoder=train_results.get('lab_encoder')
281
+ )
282
+ test_desc_df = test_dataset.get_molecular_descriptors_df()
283
+
284
+ # Generate predictions from each base model
285
+ test_predictions = {}
286
+
287
+ for model_name, trained_models in train_results['models'].items():
288
+ print(f"Generating predictions for {model_name}...")
289
+
290
+ # Get model type from train results
291
+ if 'GCN' in model_name or 'GIN' in model_name or 'GAT' in model_name:
292
+ # Graph Neural Network
293
+ model_type = model_name.lower().replace('_', '')
294
+
295
+ fold_predictions = []
296
+ for fold, model in enumerate(trained_models):
297
+ test_loader = create_graph_dataloader(
298
+ test_graphs, test_lab_features, batch_size=64, shuffle=False
299
+ )
300
+
301
+ trainer = GNNTrainer(model_type, {}, device)
302
+ pred = trainer.predict(model, test_loader)
303
+ fold_predictions.append(pred)
304
+
305
+ # Average fold predictions
306
+ test_predictions[model_name] = np.mean(fold_predictions, axis=0)
307
+
308
+ else:
309
+ # Traditional ML with descriptors
310
+ model_type = model_name.lower()
311
+ trainer = DescriptorTrainer(model_type)
312
+
313
+ fold_predictions = []
314
+ for fold, model in enumerate(trained_models):
315
+ X_test = trainer.prepare_features(test_desc_df, is_train=False)
316
+ pred = trainer.predict(model, X_test)
317
+ fold_predictions.append(pred)
318
+
319
+ # Average fold predictions
320
+ test_predictions[model_name] = np.mean(fold_predictions, axis=0)
321
+
322
+ # Create ensemble predictions
323
+ ensemble_predictor = EnsemblePredictor()
324
+ ensemble_predictions = {}
325
+
326
+ if 'average' in ensemble_methods:
327
+ # Simple average
328
+ ensemble_predictions['simple_average'] = ensemble_predictor.simple_average(test_predictions)
329
+
330
+ # Weighted average (use validation scores as weights)
331
+ val_scores = {}
332
+ for name in test_predictions.keys():
333
+ if name in train_results['cv_scores']:
334
+ # Use R2 score as weight (higher is better)
335
+ val_scores[name] = max(0.1, train_results['cv_scores'][name]['overall']['R2'])
336
+
337
+ ensemble_predictions['weighted_average'] = ensemble_predictor.simple_average(
338
+ test_predictions, val_scores
339
+ )
340
+
341
+ # Rank average
342
+ ensemble_predictions['rank_average'] = ensemble_predictor.rank_average(test_predictions)
343
+
344
+ if 'stacking' in ensemble_methods:
345
+ # Train meta-model on OOF predictions
346
+ oof_predictions = train_results['oof_predictions']
347
+ targets = train_results.get('targets') # Should be added to train_results
348
+
349
+ if targets is not None:
350
+ # Try different meta-models
351
+ meta_models = ['lgb', 'ridge', 'elastic']
352
+
353
+ for meta_type in meta_models:
354
+ try:
355
+ ensemble_predictor.train_meta_model(oof_predictions, targets, meta_type)
356
+ stack_pred = ensemble_predictor.predict_meta(test_predictions)
357
+ ensemble_predictions[f'stacking_{meta_type}'] = stack_pred
358
+ except Exception as e:
359
+ print(f"Failed to train {meta_type} meta-model: {e}")
360
+
361
+ print(f"Generated {len(ensemble_predictions)} ensemble predictions")
362
+
363
+ return ensemble_predictions
364
+
365
+
366
+ def evaluate_ensembles(train_results: Dict[str, Any],
367
+ ensemble_methods: List[str] = ['average', 'stacking']) -> Dict[str, Dict[str, float]]:
368
+ """Evaluate different ensemble methods using OOF predictions."""
369
+
370
+ oof_predictions = train_results['oof_predictions']
371
+ targets = train_results.get('targets')
372
+
373
+ if targets is None:
374
+ print("No targets available for evaluation")
375
+ return {}
376
+
377
+ print("Evaluating ensemble methods on out-of-fold predictions...")
378
+
379
+ ensemble_predictor = EnsemblePredictor()
380
+ ensemble_results = {}
381
+
382
+ # Evaluate base models
383
+ print("\nBase Model Performance:")
384
+ for name, pred in oof_predictions.items():
385
+ metrics = calculate_metrics(targets, pred)
386
+ ensemble_results[name] = metrics
387
+ print(f"{name:15s} - MAE: {metrics['MAE']:.4f}, RMSE: {metrics['RMSE']:.4f}, R2: {metrics['R2']:.4f}")
388
+
389
+ print("\nEnsemble Performance:")
390
+
391
+ if 'average' in ensemble_methods:
392
+ # Simple average
393
+ simple_avg = ensemble_predictor.simple_average(oof_predictions)
394
+ metrics = calculate_metrics(targets, simple_avg)
395
+ ensemble_results['Simple Average'] = metrics
396
+ print(f"{'Simple Average':15s} - MAE: {metrics['MAE']:.4f}, RMSE: {metrics['RMSE']:.4f}, R2: {metrics['R2']:.4f}")
397
+
398
+ # Weighted average by validation scores
399
+ val_scores = {}
400
+ for name in oof_predictions.keys():
401
+ if name in train_results['cv_scores']:
402
+ val_scores[name] = max(0.1, train_results['cv_scores'][name]['overall']['R2'])
403
+
404
+ weighted_avg = ensemble_predictor.simple_average(oof_predictions, val_scores)
405
+ metrics = calculate_metrics(targets, weighted_avg)
406
+ ensemble_results['Weighted Average'] = metrics
407
+ print(f"{'Weighted Average':15s} - MAE: {metrics['MAE']:.4f}, RMSE: {metrics['RMSE']:.4f}, R2: {metrics['R2']:.4f}")
408
+
409
+ # Optimized weights
410
+ opt_weights = ensemble_predictor.optimize_weights(oof_predictions, targets)
411
+ opt_avg = ensemble_predictor.simple_average(oof_predictions, opt_weights)
412
+ metrics = calculate_metrics(targets, opt_avg)
413
+ ensemble_results['Optimized Average'] = metrics
414
+ print(f"{'Optimized Average':15s} - MAE: {metrics['MAE']:.4f}, RMSE: {metrics['RMSE']:.4f}, R2: {metrics['R2']:.4f}")
415
+
416
+ if 'stacking' in ensemble_methods:
417
+ # Stacking with different meta-models
418
+ meta_models = ['lgb', 'ridge', 'elastic']
419
+
420
+ for meta_type in meta_models:
421
+ try:
422
+ meta_model = ensemble_predictor.train_meta_model(oof_predictions, targets, meta_type)
423
+ stack_pred = ensemble_predictor.predict_meta(oof_predictions)
424
+ metrics = calculate_metrics(targets, stack_pred)
425
+ ensemble_results[f'Stacking ({meta_type})'] = metrics
426
+ print(f"{'Stacking (' + meta_type + ')':15s} - MAE: {metrics['MAE']:.4f}, RMSE: {metrics['RMSE']:.4f}, R2: {metrics['R2']:.4f}")
427
+ except Exception as e:
428
+ print(f"Failed to evaluate {meta_type} stacking: {e}")
429
+
430
+ return ensemble_results
431
+
432
+
433
+ if __name__ == "__main__":
434
+ # Example usage
435
+ print("Ensemble module test - ready for integration")
436
+
437
+ # Test simple ensemble functions
438
+ dummy_predictions = {
439
+ 'Model1': np.random.randn(100) + 5,
440
+ 'Model2': np.random.randn(100) + 5.2,
441
+ 'Model3': np.random.randn(100) + 4.8
442
+ }
443
+
444
+ dummy_targets = np.random.randn(100) + 5
445
+
446
+ ensemble_predictor = EnsemblePredictor()
447
+
448
+ # Test simple average
449
+ avg_pred = ensemble_predictor.simple_average(dummy_predictions)
450
+ print(f"Average prediction shape: {avg_pred.shape}")
451
+
452
+ # Test meta-model training
453
+ meta_model = ensemble_predictor.train_meta_model(dummy_predictions, dummy_targets, 'ridge')
454
+ print(f"Meta-model trained: {type(meta_model)}")
455
+
456
+ # Test meta predictions
457
+ meta_pred = ensemble_predictor.predict_meta(dummy_predictions)
458
+ print(f"Meta prediction shape: {meta_pred.shape}")
459
+
460
+ print("Ensemble module tests completed successfully!")