import torch import pandas as pd import numpy as np from sklearn.metrics import f1_score, roc_auc_score, average_precision_score from timm.models import create_model import models_mamba_ecg import ecg_dataset_2021 import ecg_dataset_2020 from main_ecg import collate def evaluate_best_checkpoint(checkpoint_path, data_directory, dataset_file_all_address, threshold, device='cuda'): checkpoint = torch.load(checkpoint_path, map_location=device) args = checkpoint['args'] # Model creation updated to match HWMamba arguments model = create_model( args.model, num_classes=args.nb_classes, drop_rate=args.drop, drop_path_rate=args.drop_path, drop_block_rate=None, ) model.load_state_dict(checkpoint['model']) model.to(device) model.eval() the_testing_address = data_directory + "/testing_group" + data_directory[-1] + ".csv" df = pd.read_csv(the_testing_address) testing_header_files = [] for i in range(len(df['Name'])): each_header_file = dataset_file_all_address + df['Name'][i] testing_header_files.append(each_header_file) if args.challenge_scenario == 2021: test_dataset = ecg_dataset_2021.dataset(testing_header_files, Mixup=0) elif args.challenge_scenario == 2020: test_dataset = ecg_dataset_2020.dataset(testing_header_files, Mixup=0) test_dataset.num_leads = 12 if args.lead == '12Lead' else None test_dataset.sample = True sampler_val = torch.utils.data.SequentialSampler(test_dataset) data_loader_val = torch.utils.data.DataLoader( test_dataset, sampler=sampler_val, batch_size=int(1.5 * args.batch_size), collate_fn=collate, num_workers=0, pin_memory=False, drop_last=False ) num_samples = len(test_dataset) num_classes = args.nb_classes all_targets = np.zeros((num_samples, num_classes), dtype=np.int8) all_preds = np.zeros((num_samples, num_classes), dtype=np.int8) all_probs = np.zeros((num_samples, num_classes), dtype=np.float32) start_idx = 0 with torch.no_grad(): for batch in data_loader_val: inputs = batch[0].to(device).float() targets = batch[1].cpu().numpy() outputs = model(inputs) probs = torch.sigmoid(outputs).cpu().numpy() preds = (probs >= threshold).astype(np.int8) batch_size = inputs.size(0) end_idx = start_idx + batch_size all_targets[start_idx:end_idx] = targets all_preds[start_idx:end_idx] = preds all_probs[start_idx:end_idx] = probs start_idx = end_idx per_class_f1 = f1_score(all_targets, all_preds, average=None) per_class_auroc = roc_auc_score(all_targets, all_probs, average=None) per_class_auprc = average_precision_score(all_targets, all_probs, average=None) return per_class_f1, per_class_auroc, per_class_auprc if __name__ == '__main__': classes = ['AF', 'AFL', 'BBB', 'Brady', 'CLBBB', 'CRBBB', '1AVB', 'IRBBB', 'LAD', 'LAnFB', 'LPR', 'LQRSV', 'LQT', 'NSIVCD', 'NSR', 'PAC', 'PR', 'PRWP', 'PVC', 'QAb', 'RAD', 'SA', 'SB', 'STach', 'TAb', 'TInv'] all_f1_scores = [] all_auroc_scores = [] all_auprc_scores = [] threshold_list = [0.60, 0.64, 0.62, 0.60, 0.62] for i in range(1, 6): print(f"Running inference for group {i}...") # Verify the batchSize and block match the trained HWMamba model paths checkpoint_path = f'./output/Scenario_2021_VMamba_12Lead_batchSize_20_NoDropPathRate_Rectangle(k(3,16)s(1,16))_lossScaler_Gmlp_DownsamplingV3(k(3)_s(1,2)_p(1))_(2,2,2,8)_Adam(Higest(0.001)_decay(13)/lr_min_1e-6)_No(Z-score)_[48,96,192,384]_AmpLoss_ssm_d_state(16)_thresholdSaved_AUROC_group{i}/best_auroc_checkpoint.pth' data_directory = f'./csv-file_2021_challenge/training_validation_testing/group{i}' dataset_file_all_address = '../collection_of_all_datasets/' f1, auroc, auprc = evaluate_best_checkpoint(checkpoint_path, data_directory, dataset_file_all_address, threshold_list[i-1]) all_f1_scores.append(f1) all_auroc_scores.append(auroc) all_auprc_scores.append(auprc) avg_f1_scores = np.mean(all_f1_scores, axis=0) avg_auroc_scores = np.mean(all_auroc_scores, axis=0) avg_auprc_scores = np.mean(all_auprc_scores, axis=0) print("\n--- Average Per-Class Metrics Across 5 Folds ---") for j in range(len(classes)): print(f"Class {classes[j]:<10} | Average F1: {avg_f1_scores[j]:.4f} | Average AUROC: {avg_auroc_scores[j]:.4f} | Average AUPRC: {avg_auprc_scores[j]:.4f}") # --- Average Per-Class Metrics Across 5 Folds --- # Class AF | Average F1: 0.6058 | Average AUROC: 0.9720 | Average AUPRC: 0.7459 # Class AFL | Average F1: 0.7864 | Average AUROC: 0.9841 | Average AUPRC: 0.8306 # Class BBB | Average F1: 0.2118 | Average AUROC: 0.9568 | Average AUPRC: 0.2856 # Class Brady | Average F1: 0.1813 | Average AUROC: 0.9730 | Average AUPRC: 0.3057 # Class CLBBB | Average F1: 0.7818 | Average AUROC: 0.9912 | Average AUPRC: 0.8512 # Class CRBBB | Average F1: 0.8649 | Average AUROC: 0.9930 | Average AUPRC: 0.9300 # Class 1AVB | Average F1: 0.6867 | Average AUROC: 0.9841 | Average AUPRC: 0.7933 # Class IRBBB | Average F1: 0.4917 | Average AUROC: 0.9718 | Average AUPRC: 0.5834 # Class LAD | Average F1: 0.6624 | Average AUROC: 0.9759 | Average AUPRC: 0.7652 # Class LAnFB | Average F1: 0.6622 | Average AUROC: 0.9908 | Average AUPRC: 0.7803 # Class LPR | Average F1: 0.1042 | Average AUROC: 0.9813 | Average AUPRC: 0.2863 # Class LQRSV | Average F1: 0.3557 | Average AUROC: 0.9626 | Average AUPRC: 0.4346 # Class LQT | Average F1: 0.4124 | Average AUROC: 0.9520 | Average AUPRC: 0.5023 # Class NSIVCD | Average F1: 0.2981 | Average AUROC: 0.9195 | Average AUPRC: 0.3698 # Class NSR | Average F1: 0.9174 | Average AUROC: 0.9873 | Average AUPRC: 0.9739 # Class PAC | Average F1: 0.6257 | Average AUROC: 0.9681 | Average AUPRC: 0.7160 # Class PR | Average F1: 0.8787 | Average AUROC: 0.9904 | Average AUPRC: 0.9128 # Class PRWP | Average F1: 0.0639 | Average AUROC: 0.9736 | Average AUPRC: 0.2049 # Class PVC | Average F1: 0.6363 | Average AUROC: 0.9726 | Average AUPRC: 0.7001 # Class QAb | Average F1: 0.1829 | Average AUROC: 0.9177 | Average AUPRC: 0.3107 # Class RAD | Average F1: 0.4682 | Average AUROC: 0.9850 | Average AUPRC: 0.5486 # Class SA | Average F1: 0.7340 | Average AUROC: 0.9864 | Average AUPRC: 0.8258 # Class SB | Average F1: 0.9670 | Average AUROC: 0.9976 | Average AUPRC: 0.9915 # Class STach | Average F1: 0.9350 | Average AUROC: 0.9961 | Average AUPRC: 0.9746 # Class TAb | Average F1: 0.5172 | Average AUROC: 0.9193 | Average AUPRC: 0.6249 # Class TInv | Average F1: 0.2510 | Average AUROC: 0.9313 | Average AUPRC: 0.4148