backend/preprocessing.py

import pandas as pd
import numpy as np
import os
import joblib
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler, LabelEncoder
from sklearn.impute import SimpleImputer

def resolve_model_dir():
    base_dir = os.path.dirname(os.path.abspath(__file__))
    candidates = [
        os.path.join(base_dir, '..', 'model'),
        os.path.join(base_dir, 'model'),
        '/app/model'
    ]
    for p in candidates:
        if os.path.exists(p):
            return os.path.abspath(p)
    return os.path.abspath(candidates[0])

def resolve_data_path():
    base_dir = os.path.dirname(os.path.abspath(__file__))
    candidates = [
        os.path.join(base_dir, '..', 'data', 'patient_dataset.csv'),
        os.path.join(base_dir, 'data', 'patient_dataset.csv'),
        os.path.join(base_dir, 'patient_dataset.csv'),
        '/app/data/patient_dataset.csv'
    ]
    for p in candidates:
        if os.path.exists(p):
            return os.path.abspath(p)
    return os.path.abspath(candidates[0])

# Canonical Column Alias Mapping
COLUMN_ALIASES = {
    'heart rate (bpm)': 'Heart Rate (bpm)',
    'heart rate': 'Heart Rate (bpm)',
    'heart_rate': 'Heart Rate (bpm)',
    'heartrate': 'Heart Rate (bpm)',
    'hr': 'Heart Rate (bpm)',

    'spo2 level (%)': 'SpO2 Level (%)',
    'spo2 level': 'SpO2 Level (%)',
    'spo2_level': 'SpO2 Level (%)',
    'spo2': 'SpO2 Level (%)',
    'oxygen': 'SpO2 Level (%)',

    'systolic blood pressure (mmhg)': 'Systolic Blood Pressure (mmHg)',
    'systolic blood pressure': 'Systolic Blood Pressure (mmHg)',
    'systolic_bp': 'Systolic Blood Pressure (mmHg)',
    'sys_bp': 'Systolic Blood Pressure (mmHg)',
    'systolic': 'Systolic Blood Pressure (mmHg)',

    'diastolic blood pressure (mmhg)': 'Diastolic Blood Pressure (mmHg)',
    'diastolic blood pressure': 'Diastolic Blood Pressure (mmHg)',
    'diastolic_bp': 'Diastolic Blood Pressure (mmHg)',
    'dia_bp': 'Diastolic Blood Pressure (mmHg)',
    'diastolic': 'Diastolic Blood Pressure (mmHg)',

    'body temperature (°c)': 'Body Temperature (°C)',
    'body temperature': 'Body Temperature (°C)',
    'body_temp': 'Body Temperature (°C)',
    'temperature': 'Body Temperature (°C)',
    'temp': 'Body Temperature (°C)',

    'predicted disease': 'Predicted Disease',
    'predicted_disease': 'Predicted Disease',
    'disease': 'Predicted Disease',
    'condition': 'Predicted Disease',
    'diagnosis': 'Predicted Disease',
    'label': 'Predicted Disease',
}

def preprocess_data(df, is_training=False, model_dir=None):
    if model_dir is None:
        model_dir = resolve_model_dir()
    os.makedirs(model_dir, exist_ok=True)
    
    scaler_path = os.path.join(model_dir, 'scaler.pkl')
    label_encoder_path = os.path.join(model_dir, 'label_encoder.pkl')
    imputer_path = os.path.join(model_dir, 'imputer.pkl')
    
    # 1. Normalize column names to canonical forms
    col_map = {}
    for col in df.columns:
        norm_key = str(col).strip().lower().replace('\ufffd', '°')
        if norm_key in COLUMN_ALIASES:
            col_map[col] = COLUMN_ALIASES[norm_key]
        elif isinstance(col, str) and '\ufffd' in col:
            col_map[col] = col.replace('\ufffd', '°')
    if col_map:
        df = df.rename(columns=col_map)
        
    columns_to_drop = ['Patient Number', 'Data Accuracy (%)', 'Heart Rate Alert', 'SpO2 Level Alert', 'Blood Pressure Alert', 'Temperature Alert', 'Fall Detection']
    cols_drop = [c for c in columns_to_drop if c in df.columns]
    X_raw = df.drop(columns=cols_drop).copy()
    
    y = None
    if 'Predicted Disease' in X_raw.columns:
        y_raw = X_raw['Predicted Disease']
        X_raw = X_raw.drop(columns=['Predicted Disease'])
    
        if is_training:
            label_encoder = LabelEncoder()
            y = label_encoder.fit_transform(y_raw)
            joblib.dump(label_encoder, label_encoder_path)
            joblib.dump(label_encoder.classes_, os.path.join(model_dir, 'classes.pkl'))
        else:
            if os.path.exists(label_encoder_path):
                try:
                    label_encoder = joblib.load(label_encoder_path)
                    y = label_encoder.transform(y_raw)
                except Exception:
                    y = None
            else:
                y = y_raw 
    
    # Ensure all canonical continuous features exist (defaults for missing inputs)
    defaults = {
        'Heart Rate (bpm)': 75.0,
        'SpO2 Level (%)': 98.0,
        'Systolic Blood Pressure (mmHg)': 120.0,
        'Diastolic Blood Pressure (mmHg)': 80.0,
        'Body Temperature (°C)': 37.0,
    }
    for col, default_val in defaults.items():
        if col not in X_raw.columns:
            X_raw[col] = default_val
            
    # Engineered Feature: Risk_Severity
    X_raw['Risk_Severity'] = (X_raw['Heart Rate (bpm)'] > 105).astype(int) + (X_raw['SpO2 Level (%)'] < 94).astype(int)
    
    continuous_features = ['Heart Rate (bpm)', 'SpO2 Level (%)', 'Systolic Blood Pressure (mmHg)', 'Diastolic Blood Pressure (mmHg)', 'Body Temperature (°C)', 'Risk_Severity']
    
    if is_training:
        imputer = SimpleImputer(strategy='mean')
        X_raw[continuous_features] = imputer.fit_transform(X_raw[continuous_features])
        joblib.dump(imputer, imputer_path)
        
        scaler = StandardScaler()
        X_raw[continuous_features] = scaler.fit_transform(X_raw[continuous_features])
        joblib.dump(scaler, scaler_path)
    else:
        if os.path.exists(imputer_path):
            try:
                imputer = joblib.load(imputer_path)
                X_raw[continuous_features] = imputer.transform(X_raw[continuous_features])
            except Exception:
                pass
            
        if os.path.exists(scaler_path):
            try:
                scaler = joblib.load(scaler_path)
                X_raw[continuous_features] = scaler.transform(X_raw[continuous_features])
            except Exception:
                pass
            
    return X_raw, y

def get_train_test_split(df):
    X, y = preprocess_data(df, is_training=True)
    return train_test_split(X, y, test_size=0.2, random_state=42, stratify=y)