from __future__ import annotations import copy import math import random from dataclasses import asdict, dataclass from typing import Callable import numpy as np import pandas as pd import torch from sklearn.metrics import ( accuracy_score, confusion_matrix, f1_score, precision_score, recall_score, ) from sklearn.model_selection import train_test_split from torch import nn from torch.utils.data import DataLoader, TensorDataset from .data import PreparedWorkspace from .model import BioLMNet ProgressCallback = Callable[[float, str], None] # 80% training / 20% test, as in the BioLM-NET paper. The test partition is never # used during fitting; every reported score comes from it. DEFAULT_TEST_FRACTION = 0.2 # Share of the training partition held out to choose the early-stopping checkpoint # (1/8 of 80% = 10% of all samples), so checkpoint selection never sees the test set. EARLY_STOPPING_FRACTION = 0.125 # Activations BioLMNet accepts. ACTIVATIONS = ("relu", "sigmoid", "tanh") # Defaults follow the Bozdag Lab grid (J. Rifat, 2026-10-17): the configuration that stayed # stable across 8 seeds on BRCA (macro F1 0.749 +/- 0.018). The published activations # (relu/sigmoid/tanh) collapse at this learning rate and L2; see # diagnostics/2026-10-30-lab-hyperparameter-grid. @dataclass(frozen=True) class Hyperparameters: epochs: int = 200 batch_size: int = 32 learning_rate: float = 0.01 weight_decay: float = 0.01 dropout: float = 0.3 projection_dim: int = 64 fusion_dim: int = 12 test_fraction: float = DEFAULT_TEST_FRACTION early_stopping_fraction: float = EARLY_STOPPING_FRACTION optimizer: str = "Adam" class_weighting: bool = True early_stopping_patience: int = 12 seed: int = 42 biological_activation: str = "tanh" projection_activation: str = "relu" fusion_activation: str = "tanh" sgd_momentum: float = 0.03 def validate(self) -> None: if not 1 <= self.epochs <= 1000: raise ValueError("Epochs must be between 1 and 1,000.") if not 2 <= self.batch_size <= 1024: raise ValueError("Batch size must be between 2 and 1,024.") if not 0 < self.learning_rate <= 1: raise ValueError("Learning rate must be in (0, 1].") if not 0 <= self.dropout < 1: raise ValueError("Dropout must be in [0, 1).") if not 0.05 <= self.test_fraction <= 0.5: raise ValueError("Test fraction must be between 0.05 and 0.5.") if not 0.05 <= self.early_stopping_fraction <= 0.5: raise ValueError("Early-stopping fraction must be between 0.05 and 0.5.") if self.optimizer.lower() not in {"adam", "sgd"}: raise ValueError("Optimizer must be Adam or SGD.") for name in ("biological_activation", "projection_activation", "fusion_activation"): if getattr(self, name) not in ACTIVATIONS: raise ValueError(f"{name.replace('_', ' ').capitalize()} must be one of {', '.join(ACTIVATIONS)}.") if not 0 <= self.sgd_momentum < 1: raise ValueError("SGD momentum must be in [0, 1).") @dataclass class ModelBundle: model: BioLMNet gene_features: list[str] dna_features: list[str] label_names: list[str] gene_mean: np.ndarray gene_scale: np.ndarray dna_mean: np.ndarray dna_scale: np.ndarray config: dict metrics: dict history: list[dict[str, float]] @dataclass class TrainingResult: bundle: ModelBundle test_predictions: pd.DataFrame confusion: np.ndarray def set_reproducible_seed(seed: int) -> None: random.seed(seed) np.random.seed(seed) torch.manual_seed(seed) if torch.cuda.is_available(): torch.cuda.manual_seed_all(seed) def stratified_split( labels: np.ndarray, holdout_fraction: float, seed: int, holdout_name: str = "test" ) -> tuple[np.ndarray, np.ndarray]: """Stratified (kept, held-out) index split shared by both omics and the labels.""" classes, class_counts = np.unique(labels, return_counts=True) if class_counts.min() < 2: raise ValueError( f"Each class needs at least two samples for a stratified {holdout_name} split." ) sample_count = len(labels) holdout_count = math.ceil(holdout_fraction * sample_count) kept_count = sample_count - holdout_count if min(holdout_count, kept_count) < len(classes): raise ValueError( f"A {holdout_name} fraction of {holdout_fraction:.2f} on {sample_count} samples " f"gives {kept_count} kept and {holdout_count} {holdout_name} samples; " f"each partition needs at least one sample per class ({len(classes)} classes) " f"for a stratified split. Adjust the {holdout_name} fraction or add samples." ) indices = np.arange(sample_count) kept_index, holdout_index = train_test_split( indices, test_size=holdout_fraction, random_state=seed, stratify=labels, ) return kept_index, holdout_index def split_train_validation_test( labels: np.ndarray, test_fraction: float, early_stopping_fraction: float, seed: int ) -> tuple[np.ndarray, np.ndarray, np.ndarray]: """Training, early-stopping validation and untouched test indices. The test set is split off first; the validation set is then taken from the training part only, so neither checkpoint selection nor fitting sees the test set. """ development_index, test_index = stratified_split(labels, test_fraction, seed, "test") fit_position, validation_position = stratified_split( labels[development_index], early_stopping_fraction, seed, "early-stopping validation" ) return development_index[fit_position], development_index[validation_position], test_index def _fit_scaler(values: np.ndarray) -> tuple[np.ndarray, np.ndarray]: mean = values.mean(axis=0, dtype=np.float64).astype(np.float32) scale = values.std(axis=0, dtype=np.float64).astype(np.float32) scale[scale < 1e-8] = 1.0 return mean, scale def _scale(values: np.ndarray, mean: np.ndarray, scale: np.ndarray) -> np.ndarray: return ((values.astype(np.float32) - mean) / scale).astype(np.float32) def _make_model( workspace: PreparedWorkspace, hyperparameters: Hyperparameters ) -> BioLMNet: gene = workspace.gene_branch dna = workspace.dna_branch if ( gene.embeddings is None or dna.embeddings is None or gene.pathway_mask is None or dna.pathway_mask is None ): raise ValueError("Workspace priors are incomplete; run data preparation first.") return BioLMNet( gene_biological_mask=torch.from_numpy(gene.biological_mask), dna_biological_mask=torch.from_numpy(dna.biological_mask), gene_embeddings=torch.from_numpy(gene.embeddings), dna_embeddings=torch.from_numpy(dna.embeddings), gene_pathway_mask=torch.from_numpy(gene.pathway_mask), dna_pathway_mask=torch.from_numpy(dna.pathway_mask), n_classes=len(workspace.label_names), projection_dim=hyperparameters.projection_dim, fusion_dim=hyperparameters.fusion_dim, dropout=hyperparameters.dropout, biological_activation=hyperparameters.biological_activation, projection_activation=hyperparameters.projection_activation, fusion_activation=hyperparameters.fusion_activation, ) def _evaluate( model: BioLMNet, gene_values: np.ndarray, dna_values: np.ndarray, labels: np.ndarray, device: torch.device, ) -> tuple[float, np.ndarray, np.ndarray]: model.eval() with torch.no_grad(): logits = model( torch.from_numpy(gene_values).to(device), torch.from_numpy(dna_values).to(device), ) loss = nn.functional.cross_entropy( logits, torch.from_numpy(labels).to(device) ).item() probabilities = torch.softmax(logits, dim=1).cpu().numpy() predictions = probabilities.argmax(axis=1) return float(loss), probabilities, predictions def train( workspace: PreparedWorkspace, hyperparameters: Hyperparameters, progress: ProgressCallback | None = None, ) -> TrainingResult: hyperparameters.validate() set_reproducible_seed(hyperparameters.seed) labels = workspace.labels train_index, validation_index, test_index = split_train_validation_test( labels, hyperparameters.test_fraction, hyperparameters.early_stopping_fraction, hyperparameters.seed, ) gene_mean, gene_scale = _fit_scaler( workspace.gene_expression[train_index] ) dna_mean, dna_scale = _fit_scaler(workspace.dna_methylation[train_index]) gene_train = _scale( workspace.gene_expression[train_index], gene_mean, gene_scale ) gene_validation = _scale( workspace.gene_expression[validation_index], gene_mean, gene_scale ) dna_train = _scale( workspace.dna_methylation[train_index], dna_mean, dna_scale ) dna_validation = _scale( workspace.dna_methylation[validation_index], dna_mean, dna_scale ) gene_test = _scale(workspace.gene_expression[test_index], gene_mean, gene_scale) dna_test = _scale(workspace.dna_methylation[test_index], dna_mean, dna_scale) y_train = labels[train_index] y_validation = labels[validation_index] y_test = labels[test_index] dataset = TensorDataset( torch.from_numpy(gene_train), torch.from_numpy(dna_train), torch.from_numpy(y_train), ) generator = torch.Generator().manual_seed(hyperparameters.seed) loader = DataLoader( dataset, batch_size=min(hyperparameters.batch_size, len(dataset)), shuffle=True, generator=generator, ) device = torch.device("cuda" if torch.cuda.is_available() else "cpu") model = _make_model(workspace, hyperparameters).to(device) if hyperparameters.optimizer.lower() == "sgd": optimizer = torch.optim.SGD( model.parameters(), lr=hyperparameters.learning_rate, momentum=hyperparameters.sgd_momentum, weight_decay=hyperparameters.weight_decay, ) else: optimizer = torch.optim.Adam( model.parameters(), lr=hyperparameters.learning_rate, weight_decay=hyperparameters.weight_decay, ) class_weights: torch.Tensor | None = None if hyperparameters.class_weighting: count_by_class = np.bincount( y_train, minlength=len(workspace.label_names) ).astype(np.float32) weights = len(y_train) / (len(count_by_class) * count_by_class) class_weights = torch.from_numpy(weights).to(device) criterion = nn.CrossEntropyLoss(weight=class_weights) history: list[dict[str, float]] = [] best_state: dict[str, torch.Tensor] | None = None best_loss = float("inf") patience = 0 for epoch in range(hyperparameters.epochs): model.train() running_loss = 0.0 seen = 0 for gene_batch, dna_batch, label_batch in loader: gene_batch = gene_batch.to(device) dna_batch = dna_batch.to(device) label_batch = label_batch.to(device) optimizer.zero_grad(set_to_none=True) logits = model(gene_batch, dna_batch) loss = criterion(logits, label_batch) loss.backward() optimizer.step() running_loss += loss.item() * len(label_batch) seen += len(label_batch) validation_loss, _, validation_predictions = _evaluate( model, gene_validation, dna_validation, y_validation, device, ) epoch_row = { "epoch": float(epoch + 1), "training_loss": float(running_loss / max(seen, 1)), "validation_loss": validation_loss, "validation_accuracy": float( accuracy_score(y_validation, validation_predictions) ), "validation_f1_macro": float( f1_score( y_validation, validation_predictions, average="macro", zero_division=0, ) ), } history.append(epoch_row) if progress: progress( (epoch + 1) / hyperparameters.epochs, ( f"Epoch {epoch + 1}/{hyperparameters.epochs} ยท " f"validation F1 {epoch_row['validation_f1_macro']:.3f}" ), ) if validation_loss < best_loss - 1e-5: best_loss = validation_loss best_state = copy.deepcopy(model.state_dict()) patience = 0 else: patience += 1 if patience >= hyperparameters.early_stopping_patience: break if best_state is not None: model.load_state_dict(best_state) # Scores come only from the untouched test set. test_loss, probabilities, predictions = _evaluate( model, gene_test, dna_test, y_test, device, ) confusion = confusion_matrix( y_test, predictions, labels=np.arange(len(workspace.label_names)), ) metrics = { "evaluation_split": "test", "test_loss": test_loss, "best_validation_loss": best_loss, "accuracy": float(accuracy_score(y_test, predictions)), "f1_macro": float( f1_score(y_test, predictions, average="macro", zero_division=0) ), "f1_weighted": float( f1_score(y_test, predictions, average="weighted", zero_division=0) ), "precision_macro": float( precision_score(y_test, predictions, average="macro", zero_division=0) ), "recall_macro": float( recall_score(y_test, predictions, average="macro", zero_division=0) ), "epochs_completed": len(history), "training_samples": int(len(train_index)), "validation_samples": int(len(validation_index)), "test_samples": int(len(test_index)), "device": str(device), } predictions_frame = pd.DataFrame( { "sample_row": test_index, "observed": [workspace.label_names[value] for value in y_test], "predicted": [workspace.label_names[value] for value in predictions], } ) for index, label in enumerate(workspace.label_names): predictions_frame[f"P({label})"] = probabilities[:, index] gene = workspace.gene_branch dna = workspace.dna_branch config = { "format_version": 1, "source_name": workspace.source_name, "gene_features": gene.input_genes, "dna_features": dna.input_genes, "label_names": workspace.label_names, "gene_hidden_genes": gene.hidden_genes, "dna_hidden_genes": dna.hidden_genes, "gene_pathways": gene.pathways, "dna_pathways": dna.pathways, "hyperparameters": asdict(hyperparameters), "architecture": { "projection_dim": hyperparameters.projection_dim, "fusion_dim": hyperparameters.fusion_dim, "dropout": hyperparameters.dropout, "biological_activation": hyperparameters.biological_activation, "projection_activation": hyperparameters.projection_activation, "fusion_activation": hyperparameters.fusion_activation, }, } bundle = ModelBundle( model=model.cpu(), gene_features=gene.input_genes, dna_features=dna.input_genes, label_names=workspace.label_names, gene_mean=gene_mean, gene_scale=gene_scale, dna_mean=dna_mean, dna_scale=dna_scale, config=config, metrics=metrics, history=history, ) return TrainingResult( bundle=bundle, test_predictions=predictions_frame, confusion=confusion, ) def validate_prediction_frames( gene_frame: pd.DataFrame, dna_frame: pd.DataFrame, bundle: ModelBundle, ) -> tuple[np.ndarray, np.ndarray]: if len(gene_frame) != len(dna_frame): raise ValueError( "Prediction gene-expression and DNA-methylation files must have " "the same number of rows." ) missing_gene = sorted(set(bundle.gene_features) - set(gene_frame.columns)) missing_dna = sorted(set(bundle.dna_features) - set(dna_frame.columns)) if missing_gene or missing_dna: details = [] if missing_gene: details.append( "gene-expression: " + ", ".join(missing_gene[:8]) ) if missing_dna: details.append("DNA-methylation: " + ", ".join(missing_dna[:8])) raise ValueError( "Prediction files are missing trained features (" + "; ".join(details) + ")." ) gene_values = gene_frame.loc[:, bundle.gene_features].apply( pd.to_numeric, errors="coerce" ) dna_values = dna_frame.loc[:, bundle.dna_features].apply( pd.to_numeric, errors="coerce" ) if gene_values.isna().any().any() or dna_values.isna().any().any(): raise ValueError("Prediction inputs contain missing or non-numeric values.") return ( _scale(gene_values.to_numpy(), bundle.gene_mean, bundle.gene_scale), _scale(dna_values.to_numpy(), bundle.dna_mean, bundle.dna_scale), ) def predict( gene_frame: pd.DataFrame, dna_frame: pd.DataFrame, bundle: ModelBundle, ) -> pd.DataFrame: gene_values, dna_values = validate_prediction_frames( gene_frame, dna_frame, bundle ) bundle.model.eval() with torch.no_grad(): logits = bundle.model( torch.from_numpy(gene_values), torch.from_numpy(dna_values) ) probabilities = torch.softmax(logits, dim=1).numpy() predicted = probabilities.argmax(axis=1) output = pd.DataFrame( { "sample_row": np.arange(len(gene_frame)), "predicted_class": [ bundle.label_names[index] for index in predicted ], "confidence": probabilities.max(axis=1), } ) for index, label in enumerate(bundle.label_names): output[f"P({label})"] = probabilities[:, index] return output def pathway_importance(bundle: ModelBundle) -> pd.DataFrame: bundle.model.eval() with torch.no_grad(): attention = bundle.model.pathway_attention() records: list[dict[str, float | str]] = [] for branch_name, config_key in ( ("Gene expression", "gene_pathways"), ("DNA methylation", "dna_pathways"), ): key = "gene_expression" if branch_name == "Gene expression" else "dna_methylation" weights = attention[key].cpu().numpy() pathways = bundle.config[config_key] peak_attention = weights.max(axis=0) entropy = -( weights * np.log(np.clip(weights, 1e-12, None)) ).sum(axis=0) for pathway, peak, entropy_value in zip( pathways, peak_attention, entropy, strict=True ): records.append( { "branch": branch_name, "pathway": pathway, "peak_gene_attention": float(peak), "attention_entropy": float(entropy_value), } ) return ( pd.DataFrame(records) .sort_values("peak_gene_attention", ascending=False) .reset_index(drop=True) )