import pandas as pd import numpy as np import torch import os from train_hmoc_ensemble import Spectra1DCNN, pad_or_truncate DATA_PATH = "./data/unknown_data_after_noise_filter.parquet" MODELS_DIR = "./hmoc_binary_classifier/models" OUTPUT_PATH = "./hmoc_binary_classifier/hmmoc_candidates_70k.csv" TARGET_LEN = 1000 BATCH_SIZE = 256 def main(): print("Loading unknown data...") df = pd.read_parquet(DATA_PATH) print("Processing spectra arrays...") X = np.stack(df['spectrum'].apply(lambda s: pad_or_truncate(np.array(s), TARGET_LEN)).values) # Simple Min-Max normalization per spectrum X_min = X.min(axis=1, keepdims=True) X_max = X.max(axis=1, keepdims=True) X = (X - X_min) / (X_max - X_min + 1e-8) device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') print(f"Using device: {device}") # Load ensemble model_files = [f for f in os.listdir(MODELS_DIR) if f.startswith('hmoc_model_seed_')] models = [] for mf in model_files: model = Spectra1DCNN(TARGET_LEN) model.load_state_dict(torch.load(os.path.join(MODELS_DIR, mf), map_location=device)) model.to(device) model.eval() models.append(model) print(f"Loaded {len(models)} models for ensemble.") # Inference X_tensor = torch.tensor(X, dtype=torch.float32).unsqueeze(1) dataset = torch.utils.data.TensorDataset(X_tensor) loader = torch.utils.data.DataLoader(dataset, batch_size=BATCH_SIZE, shuffle=False, num_workers=4) ensemble_preds = np.zeros(len(X)) with torch.no_grad(): for i, (batch_x,) in enumerate(loader): batch_x = batch_x.to(device) batch_preds = torch.zeros(batch_x.size(0)).to(device) for model in models: batch_preds += torch.sigmoid(model(batch_x)).squeeze(-1) batch_preds /= len(models) start_idx = i * BATCH_SIZE end_idx = start_idx + batch_x.size(0) ensemble_preds[start_idx:end_idx] = batch_preds.cpu().numpy() if (i+1) % 10 == 0: print(f"Processed {(i+1)*BATCH_SIZE} / {len(X)} spectra...") # Save results df['hmoc_probability'] = ensemble_preds df_sorted = df.sort_values('hmoc_probability', ascending=False) print("Saving top candidates to CSV...") # Select subset of columns to save cols_to_save = ['event_id', 'hmoc_probability'] if 'event_id' in df.columns else ['hmoc_probability'] if 'sclk' in df_sorted.columns: cols_to_save = ['sclk'] + cols_to_save # Save parquet index to restore mapping later df_sorted.reset_index(names=['original_parquet_index']).to_csv(OUTPUT_PATH, index=False) print(f"Done! Ranked candidates saved to {OUTPUT_PATH}") if __name__ == "__main__": main()