cda-hmoc-classifier / scripts /infer_70k.py
Laibaff7's picture
Upload folder using huggingface_hub
1c51b77 verified
Raw History Blame Contribute Delete
2.92 kB
import pandas as pd
import numpy as np
import torch
import os
from train_hmoc_ensemble import Spectra1DCNN, pad_or_truncate
DATA_PATH = "./data/unknown_data_after_noise_filter.parquet"
MODELS_DIR = "./hmoc_binary_classifier/models"
OUTPUT_PATH = "./hmoc_binary_classifier/hmmoc_candidates_70k.csv"
TARGET_LEN = 1000
BATCH_SIZE = 256
def main():
print("Loading unknown data...")
df = pd.read_parquet(DATA_PATH)
print("Processing spectra arrays...")
X = np.stack(df['spectrum'].apply(lambda s: pad_or_truncate(np.array(s), TARGET_LEN)).values)
# Simple Min-Max normalization per spectrum
X_min = X.min(axis=1, keepdims=True)
X_max = X.max(axis=1, keepdims=True)
X = (X - X_min) / (X_max - X_min + 1e-8)
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
print(f"Using device: {device}")
# Load ensemble
model_files = [f for f in os.listdir(MODELS_DIR) if f.startswith('hmoc_model_seed_')]
models = []
for mf in model_files:
model = Spectra1DCNN(TARGET_LEN)
model.load_state_dict(torch.load(os.path.join(MODELS_DIR, mf), map_location=device))
model.to(device)
model.eval()
models.append(model)
print(f"Loaded {len(models)} models for ensemble.")
# Inference
X_tensor = torch.tensor(X, dtype=torch.float32).unsqueeze(1)
dataset = torch.utils.data.TensorDataset(X_tensor)
loader = torch.utils.data.DataLoader(dataset, batch_size=BATCH_SIZE, shuffle=False, num_workers=4)
ensemble_preds = np.zeros(len(X))
with torch.no_grad():
for i, (batch_x,) in enumerate(loader):
batch_x = batch_x.to(device)
batch_preds = torch.zeros(batch_x.size(0)).to(device)
for model in models:
batch_preds += torch.sigmoid(model(batch_x)).squeeze(-1)
batch_preds /= len(models)
start_idx = i * BATCH_SIZE
end_idx = start_idx + batch_x.size(0)
ensemble_preds[start_idx:end_idx] = batch_preds.cpu().numpy()
if (i+1) % 10 == 0:
print(f"Processed {(i+1)*BATCH_SIZE} / {len(X)} spectra...")
# Save results
df['hmoc_probability'] = ensemble_preds
df_sorted = df.sort_values('hmoc_probability', ascending=False)
print("Saving top candidates to CSV...")
# Select subset of columns to save
cols_to_save = ['event_id', 'hmoc_probability'] if 'event_id' in df.columns else ['hmoc_probability']
if 'sclk' in df_sorted.columns:
cols_to_save = ['sclk'] + cols_to_save
# Save parquet index to restore mapping later
df_sorted.reset_index(names=['original_parquet_index']).to_csv(OUTPUT_PATH, index=False)
print(f"Done! Ranked candidates saved to {OUTPUT_PATH}")
if __name__ == "__main__":
main()