Hexagonal_Warrior_Mamba_Patient_level / checkpoint_per_class_5groups.py
poult's picture
Upload 50 files
56abf31 verified
Raw History Blame Contribute Delete
7.12 kB
import torch
import pandas as pd
import numpy as np
from sklearn.metrics import f1_score, roc_auc_score, average_precision_score
from timm.models import create_model
import models_mamba_ecg
import ecg_dataset_2021
import ecg_dataset_2020
from main_ecg import collate
def evaluate_best_checkpoint(checkpoint_path, data_directory, dataset_file_all_address, threshold, device='cuda'):
checkpoint = torch.load(checkpoint_path, map_location=device)
args = checkpoint['args']
# Model creation updated to match HWMamba arguments
model = create_model(
args.model,
num_classes=args.nb_classes,
drop_rate=args.drop,
drop_path_rate=args.drop_path,
drop_block_rate=None,
)
model.load_state_dict(checkpoint['model'])
model.to(device)
model.eval()
the_testing_address = data_directory + "/testing_group" + data_directory[-1] + ".csv"
df = pd.read_csv(the_testing_address)
testing_header_files = []
for i in range(len(df['Name'])):
each_header_file = dataset_file_all_address + df['Name'][i]
testing_header_files.append(each_header_file)
if args.challenge_scenario == 2021:
test_dataset = ecg_dataset_2021.dataset(testing_header_files, Mixup=0)
elif args.challenge_scenario == 2020:
test_dataset = ecg_dataset_2020.dataset(testing_header_files, Mixup=0)
test_dataset.num_leads = 12 if args.lead == '12Lead' else None
test_dataset.sample = True
sampler_val = torch.utils.data.SequentialSampler(test_dataset)
data_loader_val = torch.utils.data.DataLoader(
test_dataset, sampler=sampler_val,
batch_size=int(1.5 * args.batch_size),
collate_fn=collate,
num_workers=0,
pin_memory=False,
drop_last=False
)
num_samples = len(test_dataset)
num_classes = args.nb_classes
all_targets = np.zeros((num_samples, num_classes), dtype=np.int8)
all_preds = np.zeros((num_samples, num_classes), dtype=np.int8)
all_probs = np.zeros((num_samples, num_classes), dtype=np.float32)
start_idx = 0
with torch.no_grad():
for batch in data_loader_val:
inputs = batch[0].to(device).float()
targets = batch[1].cpu().numpy()
outputs = model(inputs)
probs = torch.sigmoid(outputs).cpu().numpy()
preds = (probs >= threshold).astype(np.int8)
batch_size = inputs.size(0)
end_idx = start_idx + batch_size
all_targets[start_idx:end_idx] = targets
all_preds[start_idx:end_idx] = preds
all_probs[start_idx:end_idx] = probs
start_idx = end_idx
per_class_f1 = f1_score(all_targets, all_preds, average=None)
per_class_auroc = roc_auc_score(all_targets, all_probs, average=None)
per_class_auprc = average_precision_score(all_targets, all_probs, average=None)
return per_class_f1, per_class_auroc, per_class_auprc
if __name__ == '__main__':
classes = ['AF', 'AFL', 'BBB', 'Brady', 'CLBBB', 'CRBBB', '1AVB', 'IRBBB', 'LAD', 'LAnFB',
'LPR', 'LQRSV', 'LQT', 'NSIVCD', 'NSR', 'PAC', 'PR', 'PRWP', 'PVC', 'QAb',
'RAD', 'SA', 'SB', 'STach', 'TAb', 'TInv']
all_f1_scores = []
all_auroc_scores = []
all_auprc_scores = []
threshold_list = [0.60, 0.64, 0.62, 0.60, 0.62]
for i in range(1, 6):
print(f"Running inference for group {i}...")
# Verify the batchSize and block match the trained HWMamba model paths
checkpoint_path = f'./output/Scenario_2021_VMamba_12Lead_batchSize_20_NoDropPathRate_Rectangle(k(3,16)s(1,16))_lossScaler_Gmlp_DownsamplingV3(k(3)_s(1,2)_p(1))_(2,2,2,8)_Adam(Higest(0.001)_decay(13)/lr_min_1e-6)_No(Z-score)_[48,96,192,384]_AmpLoss_ssm_d_state(16)_thresholdSaved_AUROC_group{i}/best_auroc_checkpoint.pth'
data_directory = f'./csv-file_2021_challenge/training_validation_testing/group{i}'
dataset_file_all_address = '../collection_of_all_datasets/'
f1, auroc, auprc = evaluate_best_checkpoint(checkpoint_path, data_directory, dataset_file_all_address, threshold_list[i-1])
all_f1_scores.append(f1)
all_auroc_scores.append(auroc)
all_auprc_scores.append(auprc)
avg_f1_scores = np.mean(all_f1_scores, axis=0)
avg_auroc_scores = np.mean(all_auroc_scores, axis=0)
avg_auprc_scores = np.mean(all_auprc_scores, axis=0)
print("\n--- Average Per-Class Metrics Across 5 Folds ---")
for j in range(len(classes)):
print(f"Class {classes[j]:<10} | Average F1: {avg_f1_scores[j]:.4f} | Average AUROC: {avg_auroc_scores[j]:.4f} | Average AUPRC: {avg_auprc_scores[j]:.4f}")
# --- Average Per-Class Metrics Across 5 Folds ---
# Class AF | Average F1: 0.6058 | Average AUROC: 0.9720 | Average AUPRC: 0.7459
# Class AFL | Average F1: 0.7864 | Average AUROC: 0.9841 | Average AUPRC: 0.8306
# Class BBB | Average F1: 0.2118 | Average AUROC: 0.9568 | Average AUPRC: 0.2856
# Class Brady | Average F1: 0.1813 | Average AUROC: 0.9730 | Average AUPRC: 0.3057
# Class CLBBB | Average F1: 0.7818 | Average AUROC: 0.9912 | Average AUPRC: 0.8512
# Class CRBBB | Average F1: 0.8649 | Average AUROC: 0.9930 | Average AUPRC: 0.9300
# Class 1AVB | Average F1: 0.6867 | Average AUROC: 0.9841 | Average AUPRC: 0.7933
# Class IRBBB | Average F1: 0.4917 | Average AUROC: 0.9718 | Average AUPRC: 0.5834
# Class LAD | Average F1: 0.6624 | Average AUROC: 0.9759 | Average AUPRC: 0.7652
# Class LAnFB | Average F1: 0.6622 | Average AUROC: 0.9908 | Average AUPRC: 0.7803
# Class LPR | Average F1: 0.1042 | Average AUROC: 0.9813 | Average AUPRC: 0.2863
# Class LQRSV | Average F1: 0.3557 | Average AUROC: 0.9626 | Average AUPRC: 0.4346
# Class LQT | Average F1: 0.4124 | Average AUROC: 0.9520 | Average AUPRC: 0.5023
# Class NSIVCD | Average F1: 0.2981 | Average AUROC: 0.9195 | Average AUPRC: 0.3698
# Class NSR | Average F1: 0.9174 | Average AUROC: 0.9873 | Average AUPRC: 0.9739
# Class PAC | Average F1: 0.6257 | Average AUROC: 0.9681 | Average AUPRC: 0.7160
# Class PR | Average F1: 0.8787 | Average AUROC: 0.9904 | Average AUPRC: 0.9128
# Class PRWP | Average F1: 0.0639 | Average AUROC: 0.9736 | Average AUPRC: 0.2049
# Class PVC | Average F1: 0.6363 | Average AUROC: 0.9726 | Average AUPRC: 0.7001
# Class QAb | Average F1: 0.1829 | Average AUROC: 0.9177 | Average AUPRC: 0.3107
# Class RAD | Average F1: 0.4682 | Average AUROC: 0.9850 | Average AUPRC: 0.5486
# Class SA | Average F1: 0.7340 | Average AUROC: 0.9864 | Average AUPRC: 0.8258
# Class SB | Average F1: 0.9670 | Average AUROC: 0.9976 | Average AUPRC: 0.9915
# Class STach | Average F1: 0.9350 | Average AUROC: 0.9961 | Average AUPRC: 0.9746
# Class TAb | Average F1: 0.5172 | Average AUROC: 0.9193 | Average AUPRC: 0.6249
# Class TInv | Average F1: 0.2510 | Average AUROC: 0.9313 | Average AUPRC: 0.4148