import torch import pandas as pd import numpy as np import torch.nn as nn from torch.utils.data import TensorDataset, DataLoader from tab_transformer_pytorch import TabTransformer from typing import List, Dict from utils import UncertaintyEstimator class EnhancedTabTransformerWithImprovements: def __init__(self, categories: List[int], num_continuous: int, num_classes: int, device: torch.device, best_params: Dict): self.categories = categories self.num_continuous = num_continuous self.num_classes = num_classes self.device = device self.best_params = best_params self.models = [] self.uncertainty_estimator = UncertaintyEstimator() self.interpreter = None def create_model(self) -> TabTransformer: model = TabTransformer( categories=tuple(self.categories), num_continuous=self.num_continuous, dim=self.best_params['dim'], dim_out=self.num_classes, depth=self.best_params['depth'], heads=self.best_params['heads'], attn_dropout=self.best_params['attn_dropout'], ff_dropout=self.best_params['ff_dropout'], mlp_hidden_mults=( self.best_params['mlp_hidden_mult_1'], self.best_params['mlp_hidden_mult_2'] ), mlp_act=nn.ReLU() ).to(self.device) return model def load_models(self, model_path: str) -> None: """ Load trained model(s) from a file. Args: model_path: Path to the saved model file (.pkl or .pth) """ import pickle import os if not os.path.exists(model_path): raise FileNotFoundError(f"Model file not found: {model_path}") class _ModelUnpickler(pickle.Unpickler): def find_class(self, module, name): if name == 'EnhancedTabTransformerWithImprovements': return EnhancedTabTransformerWithImprovements if name == 'UncertaintyEstimator': return UncertaintyEstimator return super().find_class(module, name) # Load the saved model(s) with open(model_path, 'rb') as f: saved_data = _ModelUnpickler(f).load() print(f"Loaded pickle object of type: {type(saved_data)}") # Handle different save formats if isinstance(saved_data, EnhancedTabTransformerWithImprovements): # If saved as an EnhancedTabTransformerWithImprovements object, # extract its internal models and configuration if hasattr(saved_data, 'models') and saved_data.models: self.models = saved_data.models # Use the configuration from the saved model self.categories = saved_data.categories self.num_continuous = saved_data.num_continuous self.num_classes = saved_data.num_classes print(f"Using saved config: {len(self.categories)} categorical features, " f"{self.num_continuous} continuous features, {self.num_classes} classes") else: raise ValueError("Loaded EnhancedTabTransformerWithImprovements has no models") elif isinstance(saved_data, list): # Check if it's a list of EnhancedTabTransformerWithImprovements or raw models if saved_data and isinstance(saved_data[0], EnhancedTabTransformerWithImprovements): # Extract models from each wrapper self.models = [] for wrapper in saved_data: if hasattr(wrapper, 'models') and wrapper.models: self.models.extend(wrapper.models) # Use configuration from first wrapper if hasattr(saved_data[0], 'categories'): self.categories = saved_data[0].categories self.num_continuous = saved_data[0].num_continuous self.num_classes = saved_data[0].num_classes print(f"Using saved config: {len(self.categories)} categorical features, " f"{self.num_continuous} continuous features, {self.num_classes} classes") else: # List of raw models - keep original configuration self.models = saved_data elif isinstance(saved_data, dict): if 'models' in saved_data and isinstance(saved_data['models'], list) and saved_data['models']: self.models = saved_data['models'] self.categories = saved_data.get('categories', self.categories) self.num_continuous = saved_data.get('num_continuous', self.num_continuous) self.num_classes = saved_data.get('num_classes', self.num_classes) elif 'models_state_dicts' in saved_data: self.models = [] state_dicts = saved_data['models_state_dicts'] for state in state_dicts: model = self.create_model() model.load_state_dict(state) self.models.append(model) self.categories = saved_data.get('categories', self.categories) self.num_continuous = saved_data.get('num_continuous', self.num_continuous) self.num_classes = saved_data.get('num_classes', self.num_classes) else: raise ValueError(f"Unsupported pickle structure keys: {list(saved_data.keys())}") else: # Single raw model - wrap in list for consistency self.models = [saved_data] # Move models to device and set to eval mode for model in self.models: if hasattr(model, 'to'): model.to(self.device) if hasattr(model, 'eval'): model.eval() print(f"Loaded {len(self.models)} model(s) from {model_path}") def batch_predict_with_uncertainty(self, test_df: pd.DataFrame) -> Dict: if not self.models: raise ValueError( "No models loaded. Please call load_models() before making predictions." ) test_df_enhanced = test_df.copy() X_test = test_df_enhanced.drop(columns=['class'], errors='ignore') test_tensor = torch.FloatTensor(X_test.values) test_dataset = TensorDataset(test_tensor, torch.zeros(len(test_tensor))) test_loader = DataLoader(test_dataset, batch_size=256, shuffle=False) all_predictions = [] all_probabilities = [] for model in self.models: model.eval() predictions = [] probabilities = [] with torch.no_grad(): for batch_x, _ in test_loader: batch_x = batch_x.to(self.device) cat_features = batch_x[:, :len(self.categories)].long() cont_features = batch_x[:, len(self.categories):] if self.num_continuous > 0 else None outputs = model(cat_features, cont_features) probs = torch.softmax(outputs, dim=1) _, preds = torch.max(outputs, 1) predictions.extend(preds.cpu().numpy()) probabilities.extend(probs.cpu().numpy()) all_predictions.append(predictions) all_probabilities.append(probabilities) all_predictions = np.array(all_predictions) all_probabilities = np.array(all_probabilities) ensemble_probs = np.mean(all_probabilities, axis=0) ensemble_predictions = np.argmax(ensemble_probs, axis=1) uncertainties = self.uncertainty_estimator.estimate_uncertainty( all_predictions, all_probabilities ) return { 'predictions': ensemble_predictions, 'probabilities': ensemble_probs, 'uncertainties': uncertainties, 'individual_predictions': all_predictions, 'individual_probabilities': all_probabilities } def predict_single(self, sample: pd.DataFrame) -> Dict: """ Make prediction for a single sample with uncertainty estimation. Args: sample: A single sample DataFrame with the same features as training data Returns: Dictionary containing: - prediction: Predicted class index - probability: Probability distribution over classes - uncertainty: Uncertainty score for the prediction - individual_predictions: Predictions from each model in the ensemble - individual_probabilities: Probability distributions from each model """ # Convert single sample to batch format with batch size 1 batch_df = sample.copy() if isinstance(batch_df, pd.Series): batch_df = batch_df.to_frame().T # Get batch prediction result = self.batch_predict_with_uncertainty(batch_df) # Extract single sample results return { 'prediction': result['predictions'][0], 'probability': result['probabilities'][0], 'uncertainty': float(result['uncertainties'][0]), 'individual_predictions': [pred[0] for pred in result['individual_predictions']], 'individual_probabilities': [prob[0] for prob in result['individual_probabilities']] } def get_feature_importance(self, test_loader: DataLoader) -> Dict[str, float]: return self.interpreter.permutation_importance(test_loader)