Download models/admet_ft/_modules/model.py from Hattie/PharmAI-models: direct link, hf CLI and curl.
- Browser
- Download file 31.9 kB
-
https://huggingface.co/Hattie/PharmAI-models/resolve/main/models/admet_ft/_modules/model.py
- Command line
-
hf download hf://Hattie/PharmAI-models/models/admet_ft/_modules/model.py
-
curl -L -o model.py https://huggingface.co/Hattie/PharmAI-models/resolve/main/models/admet_ft/_modules/model.py
31.9 kB
| """ | |
| model.py - ChemBERTa ๊ธฐ๋ฐ Multi-Task ๋ชจ๋ธ ์ ์ | |
| """ | |
| import os, sys | |
| import pandas as pd | |
| import numpy as np | |
| import torch | |
| import torch.nn as nn | |
| import torch.nn.functional as F | |
| import pytorch_lightning as pl | |
| from transformers.models.auto.modeling_auto import AutoModel | |
| from transformers.models.auto.configuration_auto import AutoConfig | |
| from transformers.optimization import get_linear_schedule_with_warmup | |
| from sklearn.metrics import ( | |
| accuracy_score, precision_recall_fscore_support, roc_auc_score, | |
| mean_squared_error, mean_absolute_error, r2_score | |
| ) | |
| from typing import Dict, List, Optional | |
| # ํ๋ก์ ํธ ๋ฃจํธ ๊ฒฝ๋ก ์ถ๊ฐ | |
| _current_dir = os.path.dirname(os.path.abspath(__file__)) | |
| _project_root = os.path.dirname(os.path.dirname(_current_dir)) | |
| if _project_root not in sys.path: | |
| sys.path.insert(0, _project_root) | |
| from admet_ft._modules.scaler import (reverse_scaling, | |
| reverse_scaling_power, | |
| reverse_scaling_minmax, | |
| reverse_scaling_adaptive) | |
| class ChemBERTaMultiTask(nn.Module): | |
| """ChemBERTa ๊ธฐ๋ฐ Multi-Task Learning ๋ชจ๋ธ""" | |
| def __init__( | |
| self, | |
| model_name: Optional[str], | |
| filter_cols: List[str], | |
| task_list: List[str], | |
| task_types: Dict[str, str], | |
| num_classes: Optional[Dict[str, int]] = None, | |
| hidden_dim: List[int] = [128, 256, 128, 64], | |
| data_type: str = 'normal' # data_type ํ๋ผ๋ฏธํฐ ์ถ๊ฐ | |
| ): | |
| """ | |
| Args: | |
| model_name: ChemBERTa ๋ชจ๋ธ ์ด๋ฆ | |
| filter_cols: ๋ฒ์ฃผํ ํน์ฑ ์ปฌ๋ผ ๋ชฉ๋ก | |
| task_list: ์์ธกํ ํ์คํฌ ๋ชฉ๋ก | |
| task_types: ๊ฐ ํ์คํฌ์ ์ ํ (classification ๋๋ regression / multi_layer_regression) | |
| num_classes: ๊ฐ ๋ถ๋ฅ ํ์คํฌ์ ํด๋์ค ์ (dict) | |
| hidden_dim: ์๋์ธต ์ฐจ์ ๋ฆฌ์คํธ (์: [128, 256, 128, 64]) | |
| use_attention: ์ดํ ์ ๋ฉ์ปค๋์ฆ ์ฌ์ฉ ์ฌ๋ถ | |
| data_type: ๋ฐ์ดํฐ ์ ํ (normal, didb_reduce ๋ฑ) | |
| """ | |
| super().__init__() | |
| self.model_name = model_name | |
| self.filter_cols = filter_cols | |
| self.task_list = task_list | |
| self.task_types = task_types | |
| self.hidden_dim_list = hidden_dim if isinstance(hidden_dim, list) else [hidden_dim] | |
| self.final_hidden_dim = self.hidden_dim_list[-1] # ๋ง์ง๋ง ์ฐจ์์ด ์ต์ข ์ถ๋ ฅ ์ฐจ์ | |
| self.data_type = data_type # data_type ์ ์ฅ | |
| # ChemBERTa ๋ชจ๋ธ ์ค์ | |
| self.config = AutoConfig.from_pretrained(model_name) | |
| self.encoder = AutoModel.from_pretrained(model_name, config=self.config) | |
| self.encoder_hidden = self.config.hidden_size | |
| # Task type๋ณ๋ก output layers ๊ตฌ์ฑ ๊ฒฐ์ | |
| self.reg_tasks = [] | |
| self.multi_reg_tasks = [] | |
| self.cls_tasks = [] | |
| for task in task_list: | |
| if task_types[task] == 'regression': | |
| self.reg_tasks.append(task) | |
| elif task_types[task] == 'multi_layer_regression': | |
| self.multi_reg_tasks.append(task) | |
| elif task_types[task] == 'classification': | |
| self.cls_tasks.append(task) | |
| # Task heads ๊ตฌ์ฑ (๋ชจ๋ layers๋ฅผ task_heads์ ํตํฉ) | |
| self.task_heads = nn.ModuleDict() | |
| # ===== Regression: ๊ณต์ layers [128, 256, 128, 64, filter_cols] ===== | |
| if len(self.reg_tasks) > 0: | |
| merged_layers = nn.ModuleList() | |
| prev_dim = self.encoder_hidden | |
| # Hidden layers | |
| for curr_dim in self.hidden_dim_list: | |
| merged_layers.append(nn.Linear(prev_dim, curr_dim)) | |
| prev_dim = curr_dim | |
| # Final output layer | |
| merged_layers.append(nn.Linear(self.final_hidden_dim, len(self.filter_cols))) | |
| self.task_heads['merged'] = merged_layers | |
| # ===== Multi-layer Regression: ๊ฐ task๋ณ ๋ ๋ฆฝ layers [128, 256, 128, 64, 1] ===== | |
| if len(self.multi_reg_tasks) > 0: | |
| for task in self.multi_reg_tasks: | |
| safe_task_name = task.replace('.', '__') | |
| task_layers = nn.ModuleList() | |
| prev_dim = self.encoder_hidden | |
| # Hidden layers | |
| for curr_dim in self.hidden_dim_list: | |
| task_layers.append(nn.Linear(prev_dim, curr_dim)) | |
| prev_dim = curr_dim | |
| # Final output layer | |
| task_layers.append(nn.Linear(self.final_hidden_dim, 1)) | |
| self.task_heads[safe_task_name] = task_layers | |
| # ===== Classification: ๊ฐ task๋ณ ๋ ๋ฆฝ layers [128, 256, 128, 64, num_classes] ===== | |
| if len(self.cls_tasks) > 0: | |
| for task in self.cls_tasks: | |
| safe_task_name = task.replace('.', '__') | |
| cls_layers = nn.ModuleList() | |
| prev_dim = self.encoder_hidden | |
| # Hidden layers | |
| for curr_dim in self.hidden_dim_list: | |
| cls_layers.append(nn.Linear(prev_dim, curr_dim)) | |
| prev_dim = curr_dim | |
| # Final output layer | |
| cls_layers.append(nn.Linear( | |
| self.final_hidden_dim, | |
| num_classes[task] if num_classes else 2 | |
| )) | |
| self.task_heads[safe_task_name] = cls_layers | |
| def forward(self, input_ids, attention_mask): | |
| """์์ ํ""" | |
| encoder_output = self.encoder(input_ids, attention_mask=attention_mask).pooler_output | |
| task_outputs = {} | |
| # ===== Regression: merged layers ํต๊ณผ ===== | |
| if len(self.reg_tasks) > 0: | |
| x = encoder_output | |
| for layer in self.task_heads['merged']: | |
| x = layer(x) | |
| # ์ต์ข ์ถ๋ ฅ์ ๊ฐ task๋ณ๋ก ๋ถ๋ฆฌ | |
| for i, task in enumerate(self.reg_tasks): | |
| task_outputs[task] = x[:, i] | |
| # ===== Multi-layer Regression: ๊ฐ task๋ณ ๋ ๋ฆฝ layers ํต๊ณผ ===== | |
| if len(self.multi_reg_tasks) > 0: | |
| for task in self.multi_reg_tasks: | |
| safe_task_name = task.replace('.', '__') | |
| x = encoder_output | |
| # Task๋ณ ์ ์ฒด layers ํต๊ณผ | |
| for layer in self.task_heads[safe_task_name]: | |
| x = layer(x) | |
| task_outputs[task] = x.squeeze(-1) | |
| # ===== Classification: ๊ฐ task๋ณ ๋ ๋ฆฝ layers ํต๊ณผ ===== | |
| if len(self.cls_tasks) > 0: | |
| for task in self.cls_tasks: | |
| safe_task_name = task.replace('.', '__') | |
| x = encoder_output | |
| # Task๋ณ ์ ์ฒด layers ํต๊ณผ | |
| for layer in self.task_heads[safe_task_name]: | |
| x = layer(x) | |
| task_outputs[task] = x | |
| return task_outputs | |
| class ChemBERTaMultiTaskLightning(pl.LightningModule): | |
| """PyTorch Lightning ๊ธฐ๋ฐ ChemBERTa Multi-Task Learning ๋ชจ๋ธ""" | |
| def __init__( | |
| self, | |
| model_name: Optional[str], | |
| filter_cols: List[str], | |
| task_list: List[str], | |
| task_types: Dict[str, str], | |
| num_classes: Optional[Dict[str, int]] = None, | |
| hidden_dim: List[int] = [128, 256, 128, 64], | |
| learning_rate: float = 2e-5, | |
| weight_decay: float = 0.01, | |
| warmup_steps: int = 500, | |
| task_weights: Optional[Dict[str, float]] = None, | |
| data_type: str = 'normal', # data_type ํ๋ผ๋ฏธํฐ ์ถ๊ฐ | |
| scaling: bool = True, | |
| scaler_path: Optional[str] = None, | |
| scaler_type: str = 'power', | |
| loss_type: str = 'mse' # mse or huber | |
| ): | |
| """ | |
| Args: | |
| model_name: ChemBERTa ๋ชจ๋ธ ์ด๋ฆ | |
| filter_cols: ๋ฒ์ฃผํ ํน์ฑ ์ปฌ๋ผ ๋ชฉ๋ก | |
| task_list: ์์ธกํ ํ์คํฌ ๋ชฉ๋ก | |
| task_types: ๊ฐ ํ์คํฌ์ ์ ํ (classification ๋๋ regression) | |
| num_classes: ๊ฐ ๋ถ๋ฅ ํ์คํฌ์ ํด๋์ค ์ (dict) | |
| hidden_dim: ์๋์ธต ์ฐจ์ ๋ฆฌ์คํธ (์: [128, 256, 128, 64]) | |
| learning_rate: ํ์ต๋ฅ | |
| weight_decay: ๊ฐ์ค์น ๊ฐ์ | |
| warmup_steps: ์๋ฐ์ ์คํ ์ | |
| task_weights: ๊ฐ ํ์คํฌ์ ์์ค ๊ฐ์ค์น (dict) | |
| data_type: ๋ฐ์ดํฐ ์ ํ (normal, didb_reduce ๋ฑ) | |
| """ | |
| super().__init__() | |
| # PyTorch Lightning ํ์ดํผํ๋ผ๋ฏธํฐ ์ ์ฅ | |
| self.save_hyperparameters() | |
| self.model = ChemBERTaMultiTask( | |
| model_name=model_name, | |
| filter_cols=filter_cols, | |
| task_list=task_list, | |
| task_types=task_types, | |
| num_classes=num_classes, | |
| hidden_dim=hidden_dim, | |
| data_type=data_type # data_type ์ ๋ฌ | |
| ) | |
| self.task_list = task_list | |
| self.task_types = task_types | |
| self.task_weights = task_weights or {task: 1.0 for task in task_list} | |
| self.learning_rate = learning_rate | |
| self.weight_decay = weight_decay | |
| self.warmup_steps = warmup_steps | |
| self.filter_cols = filter_cols | |
| self.scaling = scaling | |
| self.scaler_path = scaler_path | |
| self.scaler_type = scaler_type | |
| self.loss_type = loss_type | |
| self.validation_step_outputs = [] | |
| self.test_step_outputs = [] | |
| self.test_results_cache = None # ํ ์คํธ ๊ฒฐ๊ณผ ์์ ์ ์ฅ์ฉ | |
| # ํ๊ฐ ์งํ | |
| self.metrics = {} | |
| for task in task_list: | |
| if task_types[task] == 'classification': | |
| self.metrics[task] = { | |
| 'accuracy': accuracy_score, | |
| 'precision_recall_f1': lambda y_true, y_pred: precision_recall_fscore_support( | |
| y_true, y_pred, average='binary', zero_division='0')[:3], | |
| 'auc': lambda y_true, y_pred_proba: roc_auc_score( | |
| y_true, y_pred_proba[:, 1]) if len(np.unique(y_true)) > 1 else 0.5 | |
| } | |
| else: | |
| # Regression metrics: loss_type์ ๋ฐ๋ผ ์ฃผ ์์ค ํจ์๊ฐ ๋ฌ๋ผ์ง์ง๋ง, | |
| # ํ๊ฐ๋ MSE, MAE, R2๋ฅผ ๋ชจ๋ ๊ณ์ฐ | |
| self.metrics[task] = { | |
| 'mse': mean_squared_error, | |
| 'mae': mean_absolute_error, | |
| 'r2': r2_score | |
| } | |
| def _reverse_scale(self, values, column: str) -> np.ndarray: | |
| """ | |
| ์ค์ผ์ผ๋ง๋ ์์ธก๊ฐ์ ์๋ณธ ์ค์ผ์ผ๋ก ์ญ๋ณํ | |
| Args: | |
| values: ์ค์ผ์ผ๋ง๋ ์์ธก๊ฐ (Tensor ๋๋ ndarray) | |
| column: ์ปฌ๋ผ ์ด๋ฆ | |
| Returns: | |
| ์๋ณธ ์ค์ผ์ผ๋ก ์ญ๋ณํ๋ numpy array | |
| """ | |
| if isinstance(values, torch.Tensor): | |
| values_np = values.detach().cpu().numpy() | |
| else: | |
| values_np = np.asarray(values) | |
| if not self.scaling or not self.scaler_path: | |
| return values_np.astype(np.float32) | |
| if os.path.exists(self.scaler_path): | |
| data = {col: np.full_like(values_np, np.nan, dtype=np.float32) for col in self.filter_cols} | |
| data[column] = values_np | |
| df = pd.DataFrame(data) | |
| # Scaler ํ์ ์ ๋ฐ๋ผ ์ ์ ํ ์ญ๋ณํ ํจ์ ์ ํ | |
| if self.scaler_type == 'adapt': | |
| # Adaptive scaler: feature-specific reverse scaling | |
| reverted_values = reverse_scaling_adaptive(values_np, column, self.scaler_path) | |
| return reverted_values.astype(np.float32) | |
| elif self.scaler_type == 'power': | |
| reverted = reverse_scaling_power(df, self.scaler_path) | |
| elif self.scaler_type == 'minmax': | |
| reverted = reverse_scaling_minmax(df, self.scaler_path) | |
| else: # zscore (default) | |
| reverted = reverse_scaling(df, self.scaler_path) | |
| return reverted[column].to_numpy(dtype=np.float32) | |
| return values_np.astype(np.float32) | |
| def _compute_regression_loss(self, preds: torch.Tensor, labels: torch.Tensor) -> torch.Tensor: | |
| """ | |
| Regression loss ๊ณ์ฐ (MSE ๋๋ Huber) | |
| Args: | |
| preds: ์์ธก๊ฐ (์๋ณธ ์ค์ผ์ผ) | |
| labels: ์ค์ ๊ฐ (์๋ณธ ์ค์ผ์ผ) | |
| Returns: | |
| Loss ๊ฐ | |
| """ | |
| if self.loss_type == 'huber': | |
| return F.huber_loss(preds, labels, delta=1.0) | |
| else: # mse (default) | |
| return F.mse_loss(preds, labels) | |
| def forward(self, input_ids, attention_mask): | |
| """๋ชจ๋ธ ์์ ํ""" | |
| return self.model(input_ids, attention_mask) | |
| def training_step(self, batch, batch_idx): | |
| """ํ์ต ๋จ๊ณ - Scaled ๊ณต๊ฐ์์ loss ๊ณ์ฐ""" | |
| labels = batch['labels'] | |
| input_ids, attention_mask = batch['input_ids'], batch['attention_mask'] | |
| task_outputs = self(input_ids, attention_mask) | |
| losses = {} | |
| for i, task in enumerate(self.task_list): | |
| task_labels = labels[:, i] | |
| mask = ~torch.isnan(task_labels) | |
| if mask.sum() > 0: | |
| valid_labels = task_labels[mask] | |
| valid_outputs = task_outputs[task][mask] | |
| if self.task_types[task] == 'classification': | |
| task_loss = F.cross_entropy(valid_outputs, valid_labels.long()) | |
| else: | |
| # Regression: Scaled ๊ณต๊ฐ์์ loss ๊ณ์ฐ (MSE ๋๋ Huber) | |
| task_loss = self._compute_regression_loss(valid_outputs, valid_labels.float()) | |
| losses[task] = task_loss | |
| if len(losses) > 0: | |
| total_loss = sum(losses.values()) | |
| # PyTorch Lightning ๋ก๊น | |
| self.log('train_loss', total_loss, sync_dist=True, on_step=True, on_epoch=True, prog_bar=True) | |
| for task, loss in losses.items(): | |
| self.log(f'train_{task}_loss', loss, sync_dist=True, on_step=True, on_epoch=True) | |
| return total_loss | |
| else: | |
| # ๋ชจ๋ ํ์คํฌ๊ฐ ๊ฒฐ์ธก์ด๋ฉด None ๋ฐํ(์ด step์ ๋ํด ํ์ตX) | |
| return None | |
| def validation_step(self, batch, batch_idx): | |
| """๊ฒ์ฆ ๋จ๊ณ - Scaled ๊ณต๊ฐ์์ loss ๊ณ์ฐ""" | |
| labels = batch['labels'] | |
| input_ids, attention_mask = batch['input_ids'], batch['attention_mask'] | |
| task_outputs = self(input_ids, attention_mask) | |
| losses = {} | |
| predictions = {} | |
| for i, task in enumerate(self.task_list): | |
| task_labels = labels[:, i] | |
| mask = ~torch.isnan(task_labels) | |
| if mask.sum() > 0: | |
| valid_labels = task_labels[mask] | |
| valid_outputs = task_outputs[task][mask] | |
| if self.task_types[task] == 'classification': | |
| task_loss = F.cross_entropy(valid_outputs, valid_labels.long()) | |
| task_preds = torch.argmax(valid_outputs, dim=1) | |
| task_probs = F.softmax(valid_outputs, dim=1) | |
| predictions[task] = { | |
| 'labels': valid_labels.detach().cpu(), | |
| 'preds': task_preds.detach().cpu(), | |
| 'probs': task_probs.detach().cpu() | |
| } | |
| else: | |
| # Regression: Scaled ๊ณต๊ฐ์์ loss ๊ณ์ฐ (MSE ๋๋ Huber) | |
| task_loss = self._compute_regression_loss(valid_outputs, valid_labels.float()) | |
| predictions[task] = { | |
| 'labels': valid_labels.detach().cpu(), | |
| 'preds': valid_outputs.detach().cpu() | |
| } | |
| losses[task] = task_loss | |
| else: | |
| continue | |
| if len(losses) > 0: | |
| total_loss = sum(losses.values()) | |
| # PyTorch Lightning ๋ก๊น | |
| self.log('val_loss', total_loss, prog_bar=True, sync_dist=True, on_step=True, on_epoch=True) | |
| for task, loss in losses.items(): | |
| self.log(f'val_{task}_loss', loss, sync_dist=True, on_step=True, on_epoch=True) | |
| else: | |
| total_loss = None | |
| output = {'val_loss': total_loss, 'predictions': predictions} | |
| self.validation_step_outputs.append(output) | |
| return output | |
| def on_validation_epoch_end(self): | |
| """๊ฒ์ฆ ์ํฌํฌ ์ข ๋ฃ ์ ์ฒ๋ฆฌ""" | |
| # ๋ชจ๋ ๋ฐฐ์น์ ์์ธก ๊ฒฐ๊ณผ ์์ง | |
| all_predictions = {task: {'labels': [], 'preds': [], 'probs': []} | |
| for task in self.task_list if self.task_types[task] == 'classification'} | |
| all_predictions.update({task: {'labels': [], 'preds': []} | |
| for task in self.task_list if self.task_types[task] == 'regression' or self.task_types[task] == 'multi_layer_regression'}) | |
| for output in self.validation_step_outputs: | |
| for task in self.task_list: | |
| if task not in output['predictions']: | |
| continue # ํด๋น ๋ฐฐ์น์ ์์ธก๊ฐ์ด ์์ผ๋ฉด ๊ฑด๋๋ | |
| task_preds = output['predictions'][task] | |
| if self.task_types[task] == 'classification': | |
| all_predictions[task]['labels'].extend(task_preds['labels']) | |
| all_predictions[task]['preds'].extend(task_preds['preds']) | |
| all_predictions[task]['probs'].extend(task_preds['probs']) | |
| else: | |
| all_predictions[task]['labels'].extend(task_preds['labels']) | |
| all_predictions[task]['preds'].extend(task_preds['preds']) | |
| # ๊ฐ ํ์คํฌ๋ณ ํ๊ฐ ์งํ ๊ณ์ฐ | |
| for task in self.task_list: | |
| task_preds = all_predictions[task] | |
| # ๋น์ด์๊ฑฐ๋, ํ ์๊ฐ ์๋ ๊ฐ์ด ๋ค์ด์์ผ๋ฉด metric ๊ณ์ฐ ๊ฑด๋๋ | |
| if not task_preds['labels'] or not task_preds['preds']: | |
| continue | |
| if not all(isinstance(x, torch.Tensor) for x in task_preds['labels']): | |
| continue | |
| if not all(isinstance(x, torch.Tensor) for x in task_preds['preds']): | |
| continue | |
| if self.task_types[task] == 'classification': | |
| # ๋ถ๋ฅ ํ์คํฌ ํ๊ฐ | |
| labels_np = torch.stack(task_preds['labels']).numpy() | |
| preds_np = torch.stack(task_preds['preds']).numpy() | |
| acc = self.metrics[task]['accuracy'](labels_np, preds_np) | |
| prec, rec, f1 = self.metrics[task]['precision_recall_f1'](labels_np, preds_np) | |
| # ๋ก๊น | |
| self.log(f'val_{task}_acc', acc, sync_dist=True) | |
| self.log(f'val_{task}_precision', prec, sync_dist=True) | |
| self.log(f'val_{task}_recall', rec, sync_dist=True) | |
| self.log(f'val_{task}_f1', f1, sync_dist=True) | |
| # AUC ๊ณ์ฐ (๊ฐ๋ฅํ ๊ฒฝ์ฐ) | |
| try: | |
| probs = torch.stack(task_preds['probs']).numpy() | |
| if probs.shape[1] >= 2: # ์ด์ง ๋ถ๋ฅ ์ด์์ธ ๊ฒฝ์ฐ | |
| auc = self.metrics[task]['auc'](labels_np, probs) | |
| self.log(f'val_{task}_auc', auc, sync_dist=True) | |
| except: | |
| pass | |
| else: | |
| # ํ๊ท ํ์คํฌ ํ๊ฐ | |
| labels_np = torch.stack(task_preds['labels']).numpy() | |
| preds_np = torch.stack(task_preds['preds']).numpy() | |
| # === NaN ๋ง์คํน ์ถ๊ฐ === | |
| valid_mask = (~np.isnan(labels_np)) & (~np.isnan(preds_np)) | |
| labels_np_valid = labels_np[valid_mask] | |
| preds_np_valid = preds_np[valid_mask] | |
| if len(preds_np_valid) < 2: # ์ํ์ด ๋ถ์กฑํ ๊ฒฝ์ฐ R2, MSE ๊ณ์ฐ ์๋ต | |
| continue | |
| mse = self.metrics[task]['mse'](labels_np_valid, preds_np_valid) | |
| mae = self.metrics[task]['mae'](labels_np_valid, preds_np_valid) | |
| r2 = self.metrics[task]['r2'](labels_np_valid, preds_np_valid) | |
| # ๋ก๊น | |
| self.log(f'val_{task}_mse', mse, sync_dist=True) | |
| self.log(f'val_{task}_mae', mae, sync_dist=True) | |
| self.log(f'val_{task}_r2', r2, sync_dist=True) | |
| # ๋จ๊ณ๊ฐ ๋๋๋ฉด ์ถ๋ ฅ ๋ชฉ๋ก ์ด๊ธฐํ | |
| self.validation_step_outputs.clear() | |
| def test_step(self, batch, batch_idx): | |
| labels = batch['labels'] | |
| input_ids, attention_mask = batch['input_ids'], batch['attention_mask'] | |
| task_outputs = self(input_ids, attention_mask) | |
| batch_size = labels.shape[0] | |
| losses = {} | |
| predictions = {} | |
| # SMILES ์ ๋ณด ์ถ์ถ | |
| batch_smiles = batch.get('smiles', None) | |
| for i, task in enumerate(self.task_list): | |
| task_labels = labels[:, i] | |
| mask = ~torch.isnan(task_labels) | |
| # ์ ์ฒด ๋ฐฐ์น ํฌ๊ธฐ ์ ์งํ๋ฉด์ NaN์ผ๋ก ์ด๊ธฐํ | |
| full_preds = torch.full((batch_size,), float('nan'), dtype=torch.float32) | |
| full_labels = task_labels.detach().cpu().float() | |
| if mask.sum() > 0: | |
| valid_labels = task_labels[mask] | |
| valid_outputs = task_outputs[task][mask] | |
| if self.task_types[task] == 'classification': | |
| task_loss = F.cross_entropy(valid_outputs, valid_labels.long()) | |
| task_preds = torch.argmax(valid_outputs, dim=1).float() | |
| task_probs = F.softmax(valid_outputs, dim=1) | |
| # ์ ํจํ ์์น์๋ง ์์ธก๊ฐ ์ฑ์ฐ๊ธฐ | |
| full_preds[mask] = task_preds.detach().cpu() | |
| predictions[task] = { | |
| 'labels': full_labels, | |
| 'preds': full_preds, | |
| 'probs': task_probs.detach().cpu() | |
| } | |
| else: | |
| # Regression: ์ญ๋ณํ ํ ์๋ณธ ์ค์ผ์ผ์์ loss ๊ณ์ฐ | |
| preds_orig_np = self._reverse_scale(valid_outputs, task) | |
| preds_orig = torch.from_numpy(preds_orig_np).float() | |
| labels_orig = valid_labels.detach().cpu().float() # Test labels are already in original scale | |
| # ์๋ณธ ์ค์ผ์ผ์์ loss ๊ณ์ฐ (MSE ๋๋ Huber) | |
| task_loss = self._compute_regression_loss( | |
| preds_orig.to(valid_outputs.device), | |
| labels_orig.to(valid_outputs.device) | |
| ) | |
| # ์ ํจํ ์์น์๋ง ์์ธก๊ฐ ์ฑ์ฐ๊ธฐ | |
| full_preds[mask] = preds_orig.cpu() | |
| predictions[task] = { | |
| 'labels': full_labels, | |
| 'preds': full_preds | |
| } | |
| losses[task] = task_loss | |
| else: | |
| # ๋ชจ๋ ๊ฐ์ด NaN์ธ ๊ฒฝ์ฐ์๋ ๊ตฌ์กฐ ์ ์ง | |
| predictions[task] = { | |
| 'labels': full_labels, | |
| 'preds': full_preds | |
| } | |
| if len(losses) > 0: | |
| total_loss = sum(losses.values()) | |
| # PyTorch Lightning ๋ก๊น (WandB ํธํ) | |
| self.log('test_loss', total_loss, sync_dist=True, on_step=True, on_epoch=True, prog_bar=True) | |
| for task, loss in losses.items(): | |
| self.log(f'test_{task}_loss', loss, sync_dist=True, on_step=True, on_epoch=True) | |
| else: | |
| total_loss = None | |
| output = {'test_loss': total_loss, 'predictions': predictions, 'smiles': batch_smiles} # SMILES ์ถ๊ฐ | |
| self.test_step_outputs.append(output) | |
| return output | |
| def test_step_end(self, outputs): | |
| """ํ ์คํธ ๋จ๊ณ ์ข ๋ฃ ์ ์ฒ๋ฆฌ (WandB ํธํ)""" | |
| # step๋ณ ๋ฉํธ๋ฆญ ๊ณ์ฐ ๋ฐ ๋ก๊น | |
| if outputs['test_loss'] is not None: | |
| # step๋ณ loss ๋ก๊น | |
| self.log('test_step_loss', outputs['test_loss'], on_step=True, on_epoch=False) | |
| # ๊ฐ task๋ณ step๋ณ loss ๋ก๊น | |
| for task in self.task_list: | |
| if task in outputs['predictions']: | |
| task_preds = outputs['predictions'][task] | |
| if self.task_types[task] == 'classification': | |
| # ๋ถ๋ฅ ๋ฉํธ๋ฆญ ๊ณ์ฐ | |
| labels = task_preds['labels'] | |
| preds = task_preds['preds'] | |
| if len(labels) > 0 and len(preds) > 0: | |
| acc = (labels == preds).float().mean() | |
| self.log(f'test_{task}_step_acc', acc, on_step=True, on_epoch=False) | |
| else: | |
| # ํ๊ท ๋ฉํธ๋ฆญ ๊ณ์ฐ | |
| labels = task_preds['labels'] | |
| preds = task_preds['preds'] | |
| if len(labels) > 0 and len(preds) > 0: | |
| labels_np = labels.detach().cpu().numpy() | |
| preds_np = preds.detach().cpu().numpy() | |
| valid_mask = (~np.isnan(labels_np)) & (~np.isnan(preds_np)) | |
| labels_np = labels_np[valid_mask] | |
| preds_np = preds_np[valid_mask] | |
| if len(preds_np) == 0: | |
| continue | |
| mse_val = mean_squared_error(labels_np, preds_np) | |
| mse_tensor = torch.tensor(mse_val, device=labels.device, dtype=torch.float32) | |
| self.log(f'test_{task}_step_mse', mse_tensor, on_step=True, on_epoch=False) | |
| return outputs | |
| def on_test_epoch_end(self): | |
| """ํ ์คํธ ์ํฌํฌ ์ข ๋ฃ ์ ์ฒ๋ฆฌ (๊ฒ์ฆ๊ณผ ์ ์ฌ)""" | |
| # ๋ชจ๋ ๋ฐฐ์น์ ์์ธก ๊ฒฐ๊ณผ ์์ง | |
| all_predictions = {task: {'labels': [], 'preds': [], 'probs': []} | |
| for task in self.task_list if self.task_types[task] == 'classification'} | |
| all_predictions.update({task: {'labels': [], 'preds': []} | |
| for task in self.task_list if self.task_types[task] == 'regression' or self.task_types[task] == 'multi_layer_regression'}) | |
| # SMILES ์์ง์ฉ ๋ฆฌ์คํธ ์ถ๊ฐ | |
| all_smiles = [] | |
| for output in self.test_step_outputs: | |
| # SMILES ์์ง | |
| if 'smiles' in output and output['smiles'] is not None: | |
| all_smiles.extend(output['smiles']) | |
| for task in self.task_list: | |
| if task not in output['predictions']: | |
| continue # ํด๋น ๋ฐฐ์น์ ์์ธก๊ฐ์ด ์์ผ๋ฉด ๊ฑด๋๋ | |
| task_preds = output['predictions'][task] | |
| if self.task_types[task] == 'classification': | |
| all_predictions[task]['labels'].extend(task_preds['labels']) | |
| all_predictions[task]['preds'].extend(task_preds['preds']) | |
| all_predictions[task]['probs'].extend(task_preds['probs']) | |
| else: | |
| all_predictions[task]['labels'].extend(task_preds['labels']) | |
| all_predictions[task]['preds'].extend(task_preds['preds']) | |
| # ๊ฐ ํ์คํฌ๋ณ ํ๊ฐ ์งํ ๊ณ์ฐ ๋ฐ ์ ์ฅ | |
| results = {} | |
| for task in self.task_list: | |
| task_preds = all_predictions[task] | |
| # ๋น์ด์๊ฑฐ๋, ํ ์๊ฐ ์๋ ๊ฐ์ด ๋ค์ด์์ผ๋ฉด metric ๊ณ์ฐ ๊ฑด๋๋ | |
| if not task_preds['labels'] or not task_preds['preds']: | |
| continue | |
| if not all(isinstance(x, torch.Tensor) for x in task_preds['labels']): | |
| continue | |
| if not all(isinstance(x, torch.Tensor) for x in task_preds['preds']): | |
| continue | |
| if self.task_types[task] == 'classification': | |
| # ๋ถ๋ฅ ํ์คํฌ ํ๊ฐ | |
| labels_np = torch.stack(task_preds['labels']).numpy() | |
| preds_np = torch.stack(task_preds['preds']).numpy() | |
| acc = self.metrics[task]['accuracy'](labels_np, preds_np) | |
| prec, rec, f1 = self.metrics[task]['precision_recall_f1'](labels_np, preds_np) | |
| # ๊ฒฐ๊ณผ ์ ์ฅ | |
| results[f'{task}_acc'] = acc | |
| results[f'{task}_precision'] = prec | |
| results[f'{task}_recall'] = rec | |
| results[f'{task}_f1'] = f1 | |
| # ๋ก๊น | |
| self.log(f'test_{task}_acc', acc, sync_dist=True) | |
| self.log(f'test_{task}_precision', prec, sync_dist=True) | |
| self.log(f'test_{task}_recall', rec, sync_dist=True) | |
| self.log(f'test_{task}_f1', f1, sync_dist=True) | |
| else: | |
| # ํ๊ท ํ์คํฌ ํ๊ฐ | |
| labels_np = torch.stack(task_preds['labels']).numpy() | |
| preds_np = torch.stack(task_preds['preds']).numpy() | |
| # === NaN ๋ง์คํน ์ถ๊ฐ === | |
| valid_mask = (~np.isnan(labels_np)) & (~np.isnan(preds_np)) | |
| labels_np_valid = labels_np[valid_mask] | |
| preds_np_valid = preds_np[valid_mask] | |
| if len(preds_np_valid) < 2: # ๋ชจ๋ ๊ฒฐ์ธก ๋๋ ์ํ ๋ถ์กฑ | |
| continue | |
| mse = self.metrics[task]['mse'](labels_np_valid, preds_np_valid) | |
| mae = self.metrics[task]['mae'](labels_np_valid, preds_np_valid) | |
| r2 = self.metrics[task]['r2'](labels_np_valid, preds_np_valid) | |
| # ์์ : test metric๋ง ๋ก๊น | |
| self.log(f'test_{task}_mse', mse, sync_dist=True) | |
| self.log(f'test_{task}_mae', mae, sync_dist=True) | |
| self.log(f'test_{task}_r2', r2, sync_dist=True) | |
| # SMILES์ predictions๋ฅผ results์ ์ถ๊ฐ | |
| results['smiles'] = all_smiles | |
| results['predictions'] = all_predictions | |
| # ๊ฒฐ๊ณผ๋ฅผ ์บ์์ ์ ์ฅ (trainer์์ ์ ๊ทผ ๊ฐ๋ฅํ๋๋ก) | |
| self.test_results_cache = results | |
| # ๋จ๊ณ๊ฐ ๋๋๋ฉด ์ถ๋ ฅ ๋ชฉ๋ก ์ด๊ธฐํ | |
| self.test_step_outputs.clear() | |
| # ์ ์ฒด ๊ฒฐ๊ณผ ๋ฐํ (์ซ์ ๋ฉํธ๋ฆญ๋ง ๋ก๊น ) | |
| numeric_results = {k: v for k, v in results.items() if isinstance(v, (int, float))} | |
| self.log_dict(numeric_results, sync_dist=True) | |
| return results | |
| def configure_optimizers(self): | |
| """์ตํฐ๋ง์ด์ ์ค์ """ | |
| # ๊ฐ์ค์น ๊ฐ์ ์ ์ธ ํ๋ผ๋ฏธํฐ ์ค์ | |
| no_decay = ["bias", "LayerNorm.weight"] | |
| optimizer_grouped_parameters = [ | |
| { | |
| "params": [p for n, p in self.named_parameters() if not any(nd in n for nd in no_decay)], | |
| "weight_decay": self.weight_decay, | |
| }, | |
| { | |
| "params": [p for n, p in self.named_parameters() if any(nd in n for nd in no_decay)], | |
| "weight_decay": 0.0, | |
| }, | |
| ] | |
| # ์ตํฐ๋ง์ด์ ์ค์ | |
| optimizer = torch.optim.AdamW(optimizer_grouped_parameters, lr=self.learning_rate) | |
| # ์ค์ผ์ค๋ฌ ์ค์ (์๋ฐ์ ํฌํจ) | |
| # ์ด ์คํ ์๋ Trainer๊ฐ ์ ๊ณตํ๋ ์ถ์ ์น๋ฅผ ์ฌ์ฉ | |
| if self.trainer is not None: | |
| total_steps = max(1, int(self.trainer.estimated_stepping_batches)) | |
| else: | |
| total_steps = 1000 | |
| num_training_steps = total_steps | |
| scheduler = get_linear_schedule_with_warmup( | |
| optimizer, | |
| num_warmup_steps=min(self.warmup_steps, num_training_steps - 1), | |
| num_training_steps=num_training_steps | |
| ) | |
| return { | |
| "optimizer": optimizer, | |
| "lr_scheduler": { | |
| "scheduler": scheduler, | |
| "interval": "step", | |
| }, | |
| } | |