src/Analytics/ModelML.py

import pandas as pd
import numpy as np
import xgboost as xgb
from sklearn.pipeline import Pipeline
from sklearn.compose import ColumnTransformer
from sklearn.preprocessing import OneHotEncoder
from sklearn.impute import SimpleImputer
from datetime import datetime
import unicodedata

# ============================================================================
# FONCTIONS DE NETTOYAGE (PREPROCESSING)
# ============================================================================

def clean_number(x):
    if pd.isna(x) or str(x).strip() == "": return np.nan
    try:
        return float(str(x).replace('XOF', '').replace(' ', '').replace(',', '.'))
    except (ValueError, TypeError):
        return np.nan

def parse_date(date_str):
    if pd.isna(date_str) or str(date_str).strip() == "": return pd.NaT
    try:
        return pd.to_datetime(str(date_str).split(' ')[0], format='%d-%m-%Y', errors='raise')
    except (ValueError, TypeError):
        return pd.to_datetime(str(date_str), dayfirst=True, errors='coerce')

def calcul_age(date_str):
    naissance = parse_date(date_str)
    if pd.isna(naissance): return np.nan
    return (datetime.now() - naissance).days / 365.25

def normalize_text(text):
    if pd.isna(text) or str(text).strip() == "": return "inconnu"
    text = str(text).strip().lower()
    text = unicodedata.normalize('NFD', text).encode('ascii', 'ignore').decode('utf-8')
    return text

# ============================================================================
# PIPELINE MACHINE LEARNING
# ============================================================================

def build_and_train_xgb(df_master, df_update, df_clients, df_garants, df_remb=None):
    if df_remb is None:
        df_remb = pd.DataFrame()
        
    # 1. CONSOLIDATION HISTORIQUE ROBUSTE
    df_loans = df_master.copy()
    if not df_update.empty and 'ID_Pret' in df_update.columns:
        df_combined = pd.concat([df_master, df_update], ignore_index=True)
        date_col = 'Date_Modification' if 'Date_Modification' in df_combined.columns else 'Date_Update'
        df_combined['Date_Tri'] = pd.to_datetime(df_combined[date_col], format='%d-%m-%Y %H:%M:%S', errors='coerce')
        if 'Version' in df_combined.columns:
            df_combined['Version_Num'] = pd.to_numeric(df_combined['Version'], errors='coerce').fillna(1.0)
        else:
            df_combined['Version_Num'] = 1.0
            
        df_loans = df_combined.sort_values(by=['Date_Tri', 'Version_Num'], na_position='first').drop_duplicates(subset=['ID_Pret'], keep='last')

    df_loans['Taux_Hebdo'] = df_loans['Taux_Hebdo'].apply(clean_number)
    df_loans = df_loans.dropna(subset=['Taux_Hebdo'])
    
    # 2. CALCUL DE L'HISTORIQUE CLIENT
    df_loans['Montant_Capital_Num'] = df_loans['Montant_Capital'].apply(clean_number)
    history = df_loans.groupby('ID_Client').agg(
        Nb_Prets_Historique=('ID_Pret', 'count')
    ).reset_index()
    history['Nb_Prets_Historique'] = (history['Nb_Prets_Historique'] - 1).clip(lower=0)

    # 3. JOINTURES DE BASE
    df_merged = df_loans.merge(history, on='ID_Client', how='left')
    df_merged = df_merged.merge(df_clients, on='ID_Client', how='left')
    
    if not df_garants.empty and 'ID_Garant' in df_loans.columns:
        df_merged = df_merged.merge(df_garants, on='ID_Garant', how='left', suffixes=('', '_Garant'))
    else:
        for col in ['Revenus_Mensuels_Garant']:
            df_merged[col] = np.nan

    # 4. INTÉGRATION DES COMPORTEMENTS DE PAIEMENT (Feuille Remboursements)
    if not df_remb.empty and 'ID_Client' in df_remb.columns and 'Jours_Retard' in df_remb.columns:
        df_remb['Jours_Retard_Num'] = df_remb['Jours_Retard'].apply(clean_number).fillna(0.0)
        client_remb = df_remb.groupby('ID_Client').agg(
            Retard_Moyen=('Jours_Retard_Num', 'mean'),
            Ratio_Echeances_Retard=('Jours_Retard_Num', lambda x: (x > 0).mean())
        ).reset_index()
        df_merged = df_merged.merge(client_remb, on='ID_Client', how='left')
    else:
        df_merged['Retard_Moyen'] = 0.0
        df_merged['Ratio_Echeances_Retard'] = 0.0

    df_merged['Retard_Moyen'] = df_merged['Retard_Moyen'].fillna(0.0)
    df_merged['Ratio_Echeances_Retard'] = df_merged['Ratio_Echeances_Retard'].fillna(0.0)

    # 5. FEATURE ENGINEERING GLOBAL
    df_merged['A_Un_Garant'] = df_merged['ID_Garant'].apply(lambda x: 0 if pd.isna(x) or str(x).strip() == "" else 1)
    # Exercice du pret (annee de deblocage) : le modele apprend une eventuelle
    # derive des taux d'une annee a l'autre au lieu de melanger les exercices.
    from core.exercice import annee_de, exercice_courant
    if 'Date_Deblocage' in df_merged.columns:
        df_merged['Annee_Deblocage'] = annee_de(df_merged['Date_Deblocage']).astype('float')
    else:
        df_merged['Annee_Deblocage'] = np.nan
    df_merged['Nb_Versements'] = df_merged['Nb_Versements'].apply(clean_number).fillna(1.0)
    
    # Numériques - Client
    df_merged['Age'] = df_merged['Date_Naissance'].apply(calcul_age)
    df_merged['Anciennete_Emploi'] = df_merged['Anciennete_Emploi'].apply(clean_number)
    df_merged['Anciennete_Revenu'] = df_merged['Anciennete_Revenu'].apply(clean_number)
    df_merged['Revenus_Mensuels'] = df_merged['Revenus_Mensuels'].apply(clean_number).fillna(0)
    df_merged['Autres_Revenus'] = df_merged['Autres_Revenus'].apply(clean_number).fillna(0)
    df_merged['Revenus_Totaux'] = df_merged['Revenus_Mensuels'] + df_merged['Autres_Revenus']
    df_merged['Charges_Estimees'] = df_merged['Charges_Estimees'].apply(clean_number).fillna(0)
    df_merged['Pers_Charge'] = df_merged['Pers_Charge'].apply(clean_number)
    
    df_merged['Endettement_Initial'] = np.where(df_merged['Revenus_Totaux'] > 0, (df_merged['Charges_Estimees'] / df_merged['Revenus_Totaux']) * 100, 0)
    
    # Ingénierie du Garant
    df_merged['Revenus_Mensuels_Garant'] = df_merged['Revenus_Mensuels_Garant'].apply(clean_number).fillna(0)
    df_merged['Ratio_Garant_Client'] = np.where(
        df_merged['Revenus_Mensuels'] > 0, 
        df_merged['Revenus_Mensuels_Garant'] / df_merged['Revenus_Mensuels'], 
        0.0
    )

    # Textes (Normalisation)
    text_cols = ['Offre', 'Type_Pret', 'Motif', 'Statut_Pro', 'Profession', 'Statut_Logement', 'Etat_Civil']
    for col in text_cols:
        if col not in df_merged.columns: df_merged[col] = "inconnu"
        df_merged[col] = df_merged[col].apply(normalize_text)

    # 6. SÉLECTION DES VARIABLES FINALES
    cat_features = text_cols
    num_features = [
        'Montant_Capital_Num', 'Duree_Semaines', 'Nb_Prets_Historique', 
        'Retard_Moyen', 'Ratio_Echeances_Retard', 'Nb_Versements',
        'Age', 'Anciennete_Emploi', 'Anciennete_Revenu', 'Revenus_Totaux', 'Endettement_Initial', 
        'Pers_Charge', 'A_Un_Garant', 'Ratio_Garant_Client', 'Annee_Deblocage'
    ]

    X = df_merged[cat_features + num_features]
    y = df_merged['Taux_Hebdo']

    if len(X) < 3: return None, None

    # Dossier "type" (medianes numeriques / modes categoriels) sur l'exact
    # espace de features utilise a l'entrainement, pour permettre une
    # prediction de sanite (tendance suggeree) sans dupliquer l'ingenierie
    # de features ci-dessus.
    dossier_type = {col: X[col].median() for col in num_features}
    for col in cat_features:
        mode = X[col].mode()
        dossier_type[col] = mode.iloc[0] if not mode.empty else "inconnu"
    # Un dossier "type" est toujours evalue pour l'exercice en cours.
    dossier_type['Annee_Deblocage'] = float(exercice_courant())

    # 7. PIPELINE SCILIKT-LEARN
    numeric_transformer = SimpleImputer(strategy='median')
    categorical_transformer = Pipeline(steps=[
        ('imputer', SimpleImputer(strategy='constant', fill_value='inconnu')),
        ('onehot', OneHotEncoder(handle_unknown='ignore'))
    ])

    preprocessor = ColumnTransformer(transformers=[
        ('num', numeric_transformer, num_features),
        ('cat', categorical_transformer, cat_features)
    ])

    model = Pipeline(steps=[
        ('preprocessor', preprocessor),
        ('regressor', xgb.XGBRegressor(n_estimators=50, max_depth=3, learning_rate=0.1, random_state=42))
    ])

    model.fit(X, y)
    model.vortex_erreur_par_exercice = erreur_par_exercice(model, X, y)
    return model, dossier_type


def erreur_par_exercice(model, X, y, n_plis=3):
    """Erreur absolue moyenne (points de taux hebdo) des predictions HORS
    ECHANTILLON (validation croisee), detaillee par exercice de deblocage :
    {annee: {"nb_prets": n, "mae_taux_hebdo": e}}. Dict vide si trop peu de
    donnees. Sert a verifier que le modele reste fiable d'une annee a l'autre."""
    try:
        from sklearn.base import clone
        from sklearn.model_selection import KFold, cross_val_predict
        if len(X) < max(6, n_plis * 2):
            return {}
        pred = cross_val_predict(clone(model), X, y, cv=KFold(n_splits=n_plis, shuffle=True, random_state=42))
        erreurs = pd.DataFrame({"annee": X['Annee_Deblocage'].values, "err": np.abs(np.asarray(y, dtype=float) - pred)})
        erreurs = erreurs.dropna(subset=["annee"])
        return {int(a): {"nb_prets": int(len(g)), "mae_taux_hebdo": round(float(g["err"].mean()), 3)}
                for a, g in erreurs.groupby("annee")}
    except Exception:
        return {}


def predire_taux_xgboost(modele, dossier_dict):
    """Transforme le dictionnaire en DataFrame avec normalisation des textes à la volée"""
    df_new = pd.DataFrame([dossier_dict])
    if 'Annee_Deblocage' not in df_new.columns:
        # Dossier sans date : il est evalue pour l'exercice en cours.
        from core.exercice import exercice_courant
        df_new['Annee_Deblocage'] = float(exercice_courant())
    
    for col in ['Offre', 'Type_Pret', 'Motif', 'Statut_Pro', 'Profession', 'Statut_Logement', 'Etat_Civil']:
        if col in df_new.columns:
            df_new[col] = df_new[col].apply(normalize_text)
            
    return float(modele.predict(df_new)[0])