Spaces:
Running
Running
| import pandas as pd | |
| import numpy as np | |
| import xgboost as xgb | |
| from sklearn.pipeline import Pipeline | |
| from sklearn.compose import ColumnTransformer | |
| from sklearn.preprocessing import OneHotEncoder | |
| from sklearn.impute import SimpleImputer | |
| from datetime import datetime | |
| import unicodedata | |
| # ============================================================================ | |
| # FONCTIONS DE NETTOYAGE (PREPROCESSING) | |
| # ============================================================================ | |
| def clean_number(x): | |
| if pd.isna(x) or str(x).strip() == "": return np.nan | |
| try: | |
| return float(str(x).replace('XOF', '').replace(' ', '').replace(',', '.')) | |
| except (ValueError, TypeError): | |
| return np.nan | |
| def parse_date(date_str): | |
| if pd.isna(date_str) or str(date_str).strip() == "": return pd.NaT | |
| try: | |
| return pd.to_datetime(str(date_str).split(' ')[0], format='%d-%m-%Y', errors='raise') | |
| except (ValueError, TypeError): | |
| return pd.to_datetime(str(date_str), dayfirst=True, errors='coerce') | |
| def calcul_age(date_str): | |
| naissance = parse_date(date_str) | |
| if pd.isna(naissance): return np.nan | |
| return (datetime.now() - naissance).days / 365.25 | |
| def normalize_text(text): | |
| if pd.isna(text) or str(text).strip() == "": return "inconnu" | |
| text = str(text).strip().lower() | |
| text = unicodedata.normalize('NFD', text).encode('ascii', 'ignore').decode('utf-8') | |
| return text | |
| # ============================================================================ | |
| # PIPELINE MACHINE LEARNING | |
| # ============================================================================ | |
| def build_and_train_xgb(df_master, df_update, df_clients, df_garants, df_remb=None): | |
| if df_remb is None: | |
| df_remb = pd.DataFrame() | |
| # 1. CONSOLIDATION HISTORIQUE ROBUSTE | |
| df_loans = df_master.copy() | |
| if not df_update.empty and 'ID_Pret' in df_update.columns: | |
| df_combined = pd.concat([df_master, df_update], ignore_index=True) | |
| date_col = 'Date_Modification' if 'Date_Modification' in df_combined.columns else 'Date_Update' | |
| df_combined['Date_Tri'] = pd.to_datetime(df_combined[date_col], format='%d-%m-%Y %H:%M:%S', errors='coerce') | |
| if 'Version' in df_combined.columns: | |
| df_combined['Version_Num'] = pd.to_numeric(df_combined['Version'], errors='coerce').fillna(1.0) | |
| else: | |
| df_combined['Version_Num'] = 1.0 | |
| df_loans = df_combined.sort_values(by=['Date_Tri', 'Version_Num'], na_position='first').drop_duplicates(subset=['ID_Pret'], keep='last') | |
| df_loans['Taux_Hebdo'] = df_loans['Taux_Hebdo'].apply(clean_number) | |
| df_loans = df_loans.dropna(subset=['Taux_Hebdo']) | |
| # 2. CALCUL DE L'HISTORIQUE CLIENT | |
| df_loans['Montant_Capital_Num'] = df_loans['Montant_Capital'].apply(clean_number) | |
| history = df_loans.groupby('ID_Client').agg( | |
| Nb_Prets_Historique=('ID_Pret', 'count') | |
| ).reset_index() | |
| history['Nb_Prets_Historique'] = (history['Nb_Prets_Historique'] - 1).clip(lower=0) | |
| # 3. JOINTURES DE BASE | |
| df_merged = df_loans.merge(history, on='ID_Client', how='left') | |
| df_merged = df_merged.merge(df_clients, on='ID_Client', how='left') | |
| if not df_garants.empty and 'ID_Garant' in df_loans.columns: | |
| df_merged = df_merged.merge(df_garants, on='ID_Garant', how='left', suffixes=('', '_Garant')) | |
| else: | |
| for col in ['Revenus_Mensuels_Garant']: | |
| df_merged[col] = np.nan | |
| # 4. INTÉGRATION DES COMPORTEMENTS DE PAIEMENT (Feuille Remboursements) | |
| if not df_remb.empty and 'ID_Client' in df_remb.columns and 'Jours_Retard' in df_remb.columns: | |
| df_remb['Jours_Retard_Num'] = df_remb['Jours_Retard'].apply(clean_number).fillna(0.0) | |
| client_remb = df_remb.groupby('ID_Client').agg( | |
| Retard_Moyen=('Jours_Retard_Num', 'mean'), | |
| Ratio_Echeances_Retard=('Jours_Retard_Num', lambda x: (x > 0).mean()) | |
| ).reset_index() | |
| df_merged = df_merged.merge(client_remb, on='ID_Client', how='left') | |
| else: | |
| df_merged['Retard_Moyen'] = 0.0 | |
| df_merged['Ratio_Echeances_Retard'] = 0.0 | |
| df_merged['Retard_Moyen'] = df_merged['Retard_Moyen'].fillna(0.0) | |
| df_merged['Ratio_Echeances_Retard'] = df_merged['Ratio_Echeances_Retard'].fillna(0.0) | |
| # 5. FEATURE ENGINEERING GLOBAL | |
| df_merged['A_Un_Garant'] = df_merged['ID_Garant'].apply(lambda x: 0 if pd.isna(x) or str(x).strip() == "" else 1) | |
| # Exercice du pret (annee de deblocage) : le modele apprend une eventuelle | |
| # derive des taux d'une annee a l'autre au lieu de melanger les exercices. | |
| from core.exercice import annee_de, exercice_courant | |
| if 'Date_Deblocage' in df_merged.columns: | |
| df_merged['Annee_Deblocage'] = annee_de(df_merged['Date_Deblocage']).astype('float') | |
| else: | |
| df_merged['Annee_Deblocage'] = np.nan | |
| df_merged['Nb_Versements'] = df_merged['Nb_Versements'].apply(clean_number).fillna(1.0) | |
| # Numériques - Client | |
| df_merged['Age'] = df_merged['Date_Naissance'].apply(calcul_age) | |
| df_merged['Anciennete_Emploi'] = df_merged['Anciennete_Emploi'].apply(clean_number) | |
| df_merged['Anciennete_Revenu'] = df_merged['Anciennete_Revenu'].apply(clean_number) | |
| df_merged['Revenus_Mensuels'] = df_merged['Revenus_Mensuels'].apply(clean_number).fillna(0) | |
| df_merged['Autres_Revenus'] = df_merged['Autres_Revenus'].apply(clean_number).fillna(0) | |
| df_merged['Revenus_Totaux'] = df_merged['Revenus_Mensuels'] + df_merged['Autres_Revenus'] | |
| df_merged['Charges_Estimees'] = df_merged['Charges_Estimees'].apply(clean_number).fillna(0) | |
| df_merged['Pers_Charge'] = df_merged['Pers_Charge'].apply(clean_number) | |
| df_merged['Endettement_Initial'] = np.where(df_merged['Revenus_Totaux'] > 0, (df_merged['Charges_Estimees'] / df_merged['Revenus_Totaux']) * 100, 0) | |
| # Ingénierie du Garant | |
| df_merged['Revenus_Mensuels_Garant'] = df_merged['Revenus_Mensuels_Garant'].apply(clean_number).fillna(0) | |
| df_merged['Ratio_Garant_Client'] = np.where( | |
| df_merged['Revenus_Mensuels'] > 0, | |
| df_merged['Revenus_Mensuels_Garant'] / df_merged['Revenus_Mensuels'], | |
| 0.0 | |
| ) | |
| # Textes (Normalisation) | |
| text_cols = ['Offre', 'Type_Pret', 'Motif', 'Statut_Pro', 'Profession', 'Statut_Logement', 'Etat_Civil'] | |
| for col in text_cols: | |
| if col not in df_merged.columns: df_merged[col] = "inconnu" | |
| df_merged[col] = df_merged[col].apply(normalize_text) | |
| # 6. SÉLECTION DES VARIABLES FINALES | |
| cat_features = text_cols | |
| num_features = [ | |
| 'Montant_Capital_Num', 'Duree_Semaines', 'Nb_Prets_Historique', | |
| 'Retard_Moyen', 'Ratio_Echeances_Retard', 'Nb_Versements', | |
| 'Age', 'Anciennete_Emploi', 'Anciennete_Revenu', 'Revenus_Totaux', 'Endettement_Initial', | |
| 'Pers_Charge', 'A_Un_Garant', 'Ratio_Garant_Client', 'Annee_Deblocage' | |
| ] | |
| X = df_merged[cat_features + num_features] | |
| y = df_merged['Taux_Hebdo'] | |
| if len(X) < 3: return None, None | |
| # Dossier "type" (medianes numeriques / modes categoriels) sur l'exact | |
| # espace de features utilise a l'entrainement, pour permettre une | |
| # prediction de sanite (tendance suggeree) sans dupliquer l'ingenierie | |
| # de features ci-dessus. | |
| dossier_type = {col: X[col].median() for col in num_features} | |
| for col in cat_features: | |
| mode = X[col].mode() | |
| dossier_type[col] = mode.iloc[0] if not mode.empty else "inconnu" | |
| # Un dossier "type" est toujours evalue pour l'exercice en cours. | |
| dossier_type['Annee_Deblocage'] = float(exercice_courant()) | |
| # 7. PIPELINE SCILIKT-LEARN | |
| numeric_transformer = SimpleImputer(strategy='median') | |
| categorical_transformer = Pipeline(steps=[ | |
| ('imputer', SimpleImputer(strategy='constant', fill_value='inconnu')), | |
| ('onehot', OneHotEncoder(handle_unknown='ignore')) | |
| ]) | |
| preprocessor = ColumnTransformer(transformers=[ | |
| ('num', numeric_transformer, num_features), | |
| ('cat', categorical_transformer, cat_features) | |
| ]) | |
| model = Pipeline(steps=[ | |
| ('preprocessor', preprocessor), | |
| ('regressor', xgb.XGBRegressor(n_estimators=50, max_depth=3, learning_rate=0.1, random_state=42)) | |
| ]) | |
| model.fit(X, y) | |
| model.vortex_erreur_par_exercice = erreur_par_exercice(model, X, y) | |
| return model, dossier_type | |
| def erreur_par_exercice(model, X, y, n_plis=3): | |
| """Erreur absolue moyenne (points de taux hebdo) des predictions HORS | |
| ECHANTILLON (validation croisee), detaillee par exercice de deblocage : | |
| {annee: {"nb_prets": n, "mae_taux_hebdo": e}}. Dict vide si trop peu de | |
| donnees. Sert a verifier que le modele reste fiable d'une annee a l'autre.""" | |
| try: | |
| from sklearn.base import clone | |
| from sklearn.model_selection import KFold, cross_val_predict | |
| if len(X) < max(6, n_plis * 2): | |
| return {} | |
| pred = cross_val_predict(clone(model), X, y, cv=KFold(n_splits=n_plis, shuffle=True, random_state=42)) | |
| erreurs = pd.DataFrame({"annee": X['Annee_Deblocage'].values, "err": np.abs(np.asarray(y, dtype=float) - pred)}) | |
| erreurs = erreurs.dropna(subset=["annee"]) | |
| return {int(a): {"nb_prets": int(len(g)), "mae_taux_hebdo": round(float(g["err"].mean()), 3)} | |
| for a, g in erreurs.groupby("annee")} | |
| except Exception: | |
| return {} | |
| def predire_taux_xgboost(modele, dossier_dict): | |
| """Transforme le dictionnaire en DataFrame avec normalisation des textes à la volée""" | |
| df_new = pd.DataFrame([dossier_dict]) | |
| if 'Annee_Deblocage' not in df_new.columns: | |
| # Dossier sans date : il est evalue pour l'exercice en cours. | |
| from core.exercice import exercice_courant | |
| df_new['Annee_Deblocage'] = float(exercice_courant()) | |
| for col in ['Offre', 'Type_Pret', 'Motif', 'Statut_Pro', 'Profession', 'Statut_Logement', 'Etat_Civil']: | |
| if col in df_new.columns: | |
| df_new[col] = df_new[col].apply(normalize_text) | |
| return float(modele.predict(df_new)[0]) |