""" ML_FEATURE_STORE_ANALYTICS.PY — Reacteur OAG ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ Module d'ingenierie de features pour le futur moteur d'IA OAG. 23 features FL_* calculees silencieusement, utilisables par tout modele ML. Fonctions publiques : enrich_fl(sheets) -> dict avec colonnes FL_* injectees (silencieux) render_feature_store(sheets) -> UI complète : U0 protocole / U1 ML / U2 export Prefix FL_* (Feature Library) : jamais affiche directement dans les graphiques metier, expose uniquement dans la console export OAG et les diagnostics du present module. Integration Usis.py : elif "U3" in mode: mla.render_feature_store(sheets) """ import streamlit as st import pandas as pd import numpy as np import plotly.graph_objects as go from datetime import datetime # ML from sklearn.preprocessing import StandardScaler from sklearn.decomposition import PCA from sklearn.ensemble import GradientBoostingRegressor # ── Palette ─────────────────────────────────────────────────────────────────── _C = { "bg": "#0a0e12", "card": "#0f141a", "border": "rgba(80,100,120,0.25)", "accent": "#58a6ff", "success": "#2ecc71", "warning": "#f39c12", "critical": "#c0392b", "ml": "#a78bfa", # violet = signature ML "neutral": "#5a6a7a", "text": "#a8b8c8", "subtext": "#5a6a7a", "grid": "rgba(80,100,120,0.12)", } _PAL = ["#58a6ff","#2ecc71","#f39c12","#c0392b","#a78bfa","#38bdf8","#fb7185","#34d399"] _FONT = "JetBrains Mono, Courier New, monospace" _BASE = dict( paper_bgcolor=_C["card"], plot_bgcolor=_C["card"], font=dict(color=_C["text"], family=_FONT, size=11), title_font=dict(size=12, color=_C["text"], family=_FONT), margin=dict(t=44, b=36, l=60, r=24), legend=dict(bgcolor=_C["bg"], bordercolor=_C["border"], borderwidth=1, font=dict(size=10, family=_FONT)), xaxis=dict(gridcolor=_C["grid"], zerolinecolor=_C["grid"], tickfont=dict(family=_FONT, size=10)), yaxis=dict(gridcolor=_C["grid"], zerolinecolor=_C["grid"], tickfont=dict(family=_FONT, size=10)), colorway=_PAL, ) # ── Helpers ─────────────────────────────────────────────────────────────────── def _L(**kw): m = {**_BASE} for ax in ("xaxis", "yaxis"): if ax in kw: m[ax] = {**m[ax], **kw.pop(ax)} m.update(kw) return m def _ef(msg, h=220): fig = go.Figure() fig.add_annotation(text=msg, xref="paper", yref="paper", x=0.5, y=0.5, showarrow=False, font=dict(size=12, color=_C["subtext"], family=_FONT)) fig.update_layout(**_L(height=h)) return fig def _age(v): try: d = pd.to_datetime(str(v), dayfirst=True, errors="coerce") return int((datetime.today() - d).days / 365.25) if not pd.isna(d) else np.nan except Exception: return np.nan def _safe(df, col, default=0): return df[col].fillna(default) if col in df.columns else pd.Series(default, index=df.index) def _norm_statut(v): """Normalise Statut_Pro en cle standard.""" mapping = { "cdi": "CDI", "salarie": "SALARIE", "salarié": "SALARIE", "etudiant": "ETUDIANT", "étudiant": "ETUDIANT", "independant": "INDEPENDANT", "indépendant": "INDEPENDANT", "cdd": "CDD", "informel": "INFORMEL", "sans_emploi": "SANS_EMPLOI", } return mapping.get(str(v).lower().strip(), str(v).upper().strip()) def _title_bar(txt: str, color: str = None): c = color or _C["ml"] st.markdown( f'
' f'{txt}
', unsafe_allow_html=True, ) # ══════════════════════════════════════════════════════════════════════════════ # ENRICHISSEMENT FL_* (silencieux — 23 features) # ══════════════════════════════════════════════════════════════════════════════ def enrich_fl(sheets: dict) -> dict: """ Calcule les 23 features FL_* et les injecte dans Clients_KYC. Toutes les features sont aggregees au niveau client (une ligne par client). Retourne un nouveau dict de DataFrames enrichis. """ s = {k: df.copy() for k, df in sheets.items() if isinstance(df, pd.DataFrame)} cli = s.get("Clients_KYC", pd.DataFrame()) prt = s.get("Prets_Master", pd.DataFrame()) upd = s.get("Prets_Update", pd.DataFrame()) rem = s.get("Remboursements", pd.DataFrame()) adj = s.get("Ajustements_Echeances", pd.DataFrame()) if cli.empty: return s log = [] # log des transformations pour U2 # ── 0.1 DEMOGRAPHIQUES ET STABILITE ────────────────────────────────────── # Age cli["_age"] = cli.get("Date_Naissance", pd.Series(dtype=str)).apply(_age) log.append(f"Age calcule depuis Date_Naissance : {cli['_age'].isna().sum()} null(s) impute(s) par mediane") cli["_age"] = cli["_age"].fillna(cli["_age"].median()) # FL_Age_Risk_Index : age × (capital_total / revenu_annuel) # Detecte si un jeune emprunte des sommes disproportionnees if not prt.empty and "ID_Client" in prt.columns: cap_tot = prt.groupby("ID_Client")["Montant_Capital"].sum().to_dict() else: cap_tot = {} rev_ann = (_safe(cli,"Revenus_Mensuels") + _safe(cli,"Autres_Revenus")) * 12 cli["_cap"] = _safe(cli,"ID_Client").map(cap_tot).fillna(0) cli["FL_Age_Risk_Index"] = ( cli["_age"] * (cli["_cap"] / rev_ann.replace(0, np.nan)) ).round(3) log.append("FL_Age_Risk_Index : age * (capital_total / revenu_annuel)") # FL_Job_Seniority_vs_Age : anciennete emploi / age cli["FL_Job_Seniority_vs_Age"] = ( _safe(cli,"Anciennete_Emploi") / cli["_age"].replace(0, np.nan) ).round(3) log.append("FL_Job_Seniority_vs_Age : anciennete_emploi_mois / age_annees") # FL_Social_Network_Density : compte des reseaux fournis (FB/IG/LI/TW/TT/WA) def _social_count(v): if pd.isna(v) or not str(v).strip(): return 0 prefixes = ["FB:", "IG:", "LI:", "TW:", "TT:", "WA:"] count = 0 for p in prefixes: idx = str(v).find(p) if idx >= 0: after = str(v)[idx+3:idx+30].strip() if after and not after.startswith("|") and len(after) > 2: count += 1 return count cli["FL_Social_Network_Density"] = cli.get("Reseau_sociaux", pd.Series(dtype=str)).apply(_social_count) log.append("FL_Social_Network_Density : nb reseaux sociaux renseignes (0-6)") # FL_Geographic_Stability_Score : sedentarite — score base sur unicite Ville/Quartier if "Ville" in cli.columns and "Quartier" in cli.columns: freq_ville = cli["Ville"].value_counts(normalize=True) freq_qrt = cli["Quartier"].value_counts(normalize=True) cli["FL_Geographic_Stability_Score"] = ( cli["Ville"].map(freq_ville).fillna(0) * 0.4 + cli["Quartier"].map(freq_qrt).fillna(0) * 0.6 ).round(3) else: cli["FL_Geographic_Stability_Score"] = 0.5 log.append("FL_Geographic_Stability_Score : frequence relative Ville(40%) + Quartier(60%)") # FL_Life_Stage_Proxy : etape de vie estimee (0=etudiant, 1=actif junior, 2=actif senior, 3=senior) statut_norm = cli.get("Statut_Pro", pd.Series(dtype=str)).apply(_norm_statut) def _life_stage(row): age = row["_age"] sp = _norm_statut(row.get("Statut_Pro", "")) if sp in ("ETUDIANT",) or age < 25: return 0 elif sp in ("SALARIE","CDI","CDD") and age < 35: return 1 elif sp in ("INDEPENDANT","INFORMEL") or 35 <= age < 50: return 2 else: return 3 cli["FL_Life_Stage_Proxy"] = cli.apply(_life_stage, axis=1) log.append("FL_Life_Stage_Proxy : 0=etudiant, 1=actif jr, 2=actif sr, 3=senior (derive age+statut_pro)") # ── 0.2 PUISSANCE FINANCIERE ────────────────────────────────────────────── rev_tot = _safe(cli,"Revenus_Mensuels") + _safe(cli,"Autres_Revenus") charges = _safe(cli,"Charges_Estimees") # FL_Net_Disposable_Income cli["FL_Net_Disposable_Income"] = (rev_tot - charges).round(0) log.append("FL_Net_Disposable_Income : Revenus_Mensuels + Autres_Revenus - Charges_Estimees") # FL_Theoretical_Debt_Capacity : 33% du revenu net mensuel (seuil bancaire) cli["FL_Theoretical_Debt_Capacity"] = (cli["FL_Net_Disposable_Income"] * 0.33).round(0) log.append("FL_Theoretical_Debt_Capacity : 33% du revenu net mensuel (benchmark bancaire)") # FL_Income_MultiSource_Index : part des autres revenus dans le total cli["FL_Income_MultiSource_Index"] = ( _safe(cli,"Autres_Revenus") / rev_tot.replace(0, np.nan) ).fillna(0).round(3) log.append("FL_Income_MultiSource_Index : Autres_Revenus / Revenus_Totaux") # FL_Wealth_to_Commitment_Ratio # NOTA : Patrimoine_Declare est sur echelle ordinale 0-3 (pas FCFA). # Normalise sur 100 (0->0, 1->33, 2->67, 3->100) puis divise par capital normalise. patr_norm = (_safe(cli,"Patrimoine_Declare").clip(0,3) / 3.0) cap_norm = (cli["_cap"] / cli["_cap"].replace(0,np.nan).max()).fillna(0) if cli["_cap"].max() > 0 else pd.Series(0.5,index=cli.index) cli["FL_Wealth_to_Commitment_Ratio"] = (patr_norm / cap_norm.replace(0, np.nan)).fillna(0).round(3) log.append("FL_Wealth_to_Commitment_Ratio : Patrimoine_norm(0-3→0-1) / Capital_norm — NOTE: Patrimoine_Declare est une echelle ordinale 0-3, pas FCFA") # FL_Financial_Fragility_Index : charges / revenu total (proximite rupture) cli["FL_Financial_Fragility_Index"] = ( charges / rev_tot.replace(0, np.nan) ).fillna(0).round(3) log.append("FL_Financial_Fragility_Index : Charges_Estimees / Revenus_Totaux — seuil critique > 0.6") # FL_Loan_to_Income_Ratio (ajout strategique) cli["FL_Loan_to_Income_Ratio"] = ( cli["_cap"] / (rev_tot.replace(0, np.nan) * 12) ).fillna(0).round(3) log.append("FL_Loan_to_Income_Ratio : Capital_total / Revenus_annuels (benchmark < 3.0)") # FL_Debt_Service_Coverage (ajout strategique) if not prt.empty and "ID_Client" in prt.columns: versement_map = prt.groupby("ID_Client")["Montant_Versement"].sum().to_dict() else: versement_map = {} cli["_versement"] = _safe(cli,"ID_Client").map(versement_map).fillna(0) cli["FL_Debt_Service_Coverage"] = ( cli["FL_Net_Disposable_Income"] / cli["_versement"].replace(0, np.nan) ).fillna(0).round(3) log.append("FL_Debt_Service_Coverage : Revenu_Net / Versement_mensuel — ideal > 2.0") # ── 0.3 COMPORTEMENTALES ───────────────────────────────────────────────── if not rem.empty and "ID_Client" in rem.columns: # FL_Repayment_Punctuality_Rate good_mask = rem["Statut_Paiement"].str.upper().isin(["PONCTUEL","ANTICIPE","PAYE"]) punct_rate = rem.groupby("ID_Client").apply( lambda g: good_mask.reindex(g.index).fillna(False).sum() / len(g) ).round(3) cli["FL_Repayment_Punctuality_Rate"] = _safe(cli,"ID_Client").map(punct_rate) n_imp = cli["FL_Repayment_Punctuality_Rate"].isna().sum() med_p = punct_rate.median() if not punct_rate.empty else 0.8 cli["FL_Repayment_Punctuality_Rate"] = cli["FL_Repayment_Punctuality_Rate"].fillna(med_p).round(3) log.append(f"FL_Repayment_Punctuality_Rate : % paiements PONCTUEL/ANTICIPE — {n_imp} client(s) sans historique impute(s) par mediane ({med_p:.2f})") # FL_Late_Gravity_Score : somme ponderee par severite (retard^1.5) def _late_gravity(g): jr = g["Jours_Retard"].clip(lower=0) return float((jr ** 1.5).sum()) gravity = rem.groupby("ID_Client").apply(_late_gravity).round(2) cli["FL_Late_Gravity_Score"] = _safe(cli,"ID_Client").map(gravity).fillna(0).round(2) log.append("FL_Late_Gravity_Score : somme(Jours_Retard_positifs ^ 1.5) par client — penalise les gros retards") # FL_Prepayment_Index (ajout strategique) prepay = rem.groupby("ID_Client").apply( lambda g: (g["Jours_Retard"] < 0).sum() / len(g) ).round(3) cli["FL_Prepayment_Index"] = _safe(cli,"ID_Client").map(prepay).fillna(0).round(3) log.append("FL_Prepayment_Index : % paiements avec Jours_Retard < 0 (paiements anticipes)") else: for f in ["FL_Repayment_Punctuality_Rate","FL_Late_Gravity_Score","FL_Prepayment_Index"]: cli[f] = np.nan log.append("FL_Repayment_* : table Remboursements vide — toutes valeurs NaN") # FL_Renegotiation_Frequency if not upd.empty and "ID_Client" in upd.columns and not prt.empty: upd_cnt = upd.groupby("ID_Client").size().to_dict() prt_cnt = prt.groupby("ID_Client").size().to_dict() cli["FL_Renegotiation_Frequency"] = _safe(cli,"ID_Client").apply( lambda cid: round(upd_cnt.get(cid,0) / max(prt_cnt.get(cid,1), 1), 3) ) else: cli["FL_Renegotiation_Frequency"] = 0.0 log.append("FL_Renegotiation_Frequency : nb_modifications / nb_prets (0=stable, >0.5=instable)") # FL_Contract_Drift_Rate if not upd.empty and not prt.empty and "ID_Client" in upd.columns: def _drift(cid): orig = prt[prt["ID_Client"]==cid]["Montant_Total"].sum() last = (upd[upd["ID_Client"]==cid] .sort_values("Version",ascending=False) .drop_duplicates("ID_Pret")["Montant_Total"].sum()) return round((last - orig) / orig, 4) if orig > 0 else 0.0 cli["FL_Contract_Drift_Rate"] = _safe(cli,"ID_Client").apply(_drift) else: cli["FL_Contract_Drift_Rate"] = 0.0 log.append("FL_Contract_Drift_Rate : (Montant_Total_final - initial) / initial — negatif = reduction, positif = surencherissement") # FL_Adjustment_Dependency — inoperant jusqu'a alimentation Ajustements_Echeances if not adj.empty and "ID_Pret" in adj.columns and not prt.empty: adj_map = adj.groupby("ID_Pret").size().to_dict() prt["_adj"] = _safe(prt,"ID_Pret").map(adj_map).fillna(0) adj_cli = prt.groupby("ID_Client")["_adj"].sum().to_dict() cli["FL_Adjustment_Dependency"] = _safe(cli,"ID_Client").map(adj_cli).fillna(0) else: cli["FL_Adjustment_Dependency"] = 0.0 log.append("FL_Adjustment_Dependency : nb ajustements par client — 0.0 (table Ajustements vide) activable en production") # FL_Multi_Loan_Exposure (ajout strategique) if not prt.empty: active_statuts = ["ACTIF","EN_RETARD"] active_loans = prt[prt["Statut"].str.upper().isin(active_statuts)].groupby("ID_Client").size().to_dict() cli["FL_Multi_Loan_Exposure"] = _safe(cli,"ID_Client").map(active_loans).fillna(0).astype(int) else: cli["FL_Multi_Loan_Exposure"] = 0 log.append("FL_Multi_Loan_Exposure : nb prets actifs par client (statut ACTIF ou EN_RETARD)") # ── 0.4 RENTABILITE ────────────────────────────────────────────────────── # FL_Effective_APR if not prt.empty and "Taux_Hebdo" in prt.columns: apr_map = prt.groupby("ID_Client")["Taux_Hebdo"].mean().apply( lambda t: round(((1 + t/100)**52 - 1) * 100, 2) ).to_dict() cli["FL_Effective_APR"] = _safe(cli,"ID_Client").map(apr_map).fillna(0) else: cli["FL_Effective_APR"] = 0.0 log.append("FL_Effective_APR : ((1 + Taux_Hebdo/100)^52 - 1) * 100 — taux annuel effectif global moyen par client") # FL_Yield_per_Client if not rem.empty and "ID_Client" in rem.columns: yield_map = rem.groupby("ID_Client")["Montant_Interets"].sum().to_dict() cli["FL_Yield_per_Client"] = _safe(cli,"ID_Client").map(yield_map).fillna(0) else: cli["FL_Yield_per_Client"] = 0.0 log.append("FL_Yield_per_Client : somme des Montant_Interets realises par client") # FL_Operational_Cost_Index : nb modifications + nb transactions (proxy de complexite de gestion) upd_ops = upd.groupby("ID_Client").size().to_dict() if not upd.empty and "ID_Client" in upd.columns else {} rem_ops = rem.groupby("ID_Client").size().to_dict() if not rem.empty and "ID_Client" in rem.columns else {} cli["FL_Operational_Cost_Index"] = _safe(cli,"ID_Client").apply( lambda cid: int(upd_ops.get(cid,0) + rem_ops.get(cid,0)) ) log.append("FL_Operational_Cost_Index : nb_modifications + nb_transactions (complexite de gestion)") # ── TARGET : FL_Risk_Target ─────────────────────────────────────────────── # Score composite [0.0 - 1.0] — variable cible pour ML supervise # Composantes : Financial_Fragility (40%) + Late_Gravity norm (30%) + Renegotiation (30%) ffr = cli["FL_Financial_Fragility_Index"].clip(0,1) late = cli["FL_Late_Gravity_Score"] late_n = (late / late.replace(0,np.nan).max()).fillna(0) reneg = cli["FL_Renegotiation_Frequency"].clip(0,1) cli["FL_Risk_Target"] = (ffr*0.40 + late_n*0.30 + reneg*0.30).round(4) log.append("FL_Risk_Target : score de risque composite [0-1] = FFI(40%) + LateGravity_norm(30%) + RenegFreq(30%) — VARIABLE CIBLE ML SUPERVISE") # ── Nettoyage des colonnes temporaires ─────────────────────────────────── cli.drop(columns=[c for c in ["_age","_cap","_versement"] if c in cli.columns], inplace=True, errors="ignore") s["Clients_KYC"] = cli # Stocker le log dans le dict pour U2 s["_fl_log"] = log return s # ══════════════════════════════════════════════════════════════════════════════ # RENDU FRONT-END # ══════════════════════════════════════════════════════════════════════════════ _FL_COLS = [ "FL_Age_Risk_Index", "FL_Job_Seniority_vs_Age", "FL_Social_Network_Density", "FL_Geographic_Stability_Score", "FL_Life_Stage_Proxy", "FL_Net_Disposable_Income", "FL_Theoretical_Debt_Capacity", "FL_Income_MultiSource_Index", "FL_Wealth_to_Commitment_Ratio", "FL_Financial_Fragility_Index", "FL_Loan_to_Income_Ratio", "FL_Debt_Service_Coverage", "FL_Repayment_Punctuality_Rate", "FL_Late_Gravity_Score", "FL_Prepayment_Index", "FL_Renegotiation_Frequency", "FL_Contract_Drift_Rate", "FL_Adjustment_Dependency", "FL_Multi_Loan_Exposure", "FL_Effective_APR", "FL_Yield_per_Client", "FL_Operational_Cost_Index", "FL_Risk_Target", ] _FL_GROUPS = { "DEMOGRAPHIQUES": [ "FL_Age_Risk_Index","FL_Job_Seniority_vs_Age", "FL_Social_Network_Density","FL_Geographic_Stability_Score","FL_Life_Stage_Proxy", ], "PUISSANCE FINANCIERE": [ "FL_Net_Disposable_Income","FL_Theoretical_Debt_Capacity", "FL_Income_MultiSource_Index","FL_Wealth_to_Commitment_Ratio", "FL_Financial_Fragility_Index","FL_Loan_to_Income_Ratio","FL_Debt_Service_Coverage", ], "COMPORTEMENTALES": [ "FL_Repayment_Punctuality_Rate","FL_Late_Gravity_Score","FL_Prepayment_Index", "FL_Renegotiation_Frequency","FL_Contract_Drift_Rate", "FL_Adjustment_Dependency","FL_Multi_Loan_Exposure", ], "RENTABILITE": [ "FL_Effective_APR","FL_Yield_per_Client","FL_Operational_Cost_Index", ], "TARGET ML": ["FL_Risk_Target"], } def _get_fl_matrix(cli: pd.DataFrame) -> pd.DataFrame: """Extrait la matrice features x clients, impute, retourne un DataFrame propre.""" available = [c for c in _FL_COLS if c in cli.columns] mat = cli[["ID_Client"] + available].copy() if "ID_Client" in cli.columns else cli[available].copy() for col in available: if mat[col].isna().any(): mat[col] = mat[col].fillna(mat[col].median()) return mat def _render_u0_protocol(cli: pd.DataFrame, log: list): """U0 : protocole d'ingenierie — stats par feature + log des transformations.""" _title_bar("U0 · PROTOCOLE D'INGENIERIE FEATURES", _C["ml"]) available = [c for c in _FL_COLS if c in cli.columns] n_feat = len(available) n_null = sum(cli[c].isna().sum() for c in available) n_cli = len(cli) # KPI row c1,c2,c3,c4 = st.columns(4) for col,label,val,color in [ (c1,"FEATURES FL_*", n_feat, _C["ml"]), (c2,"CLIENTS", n_cli, _C["text"]), (c3,"NULLS IMPUTES", n_null, _C["warning"] if n_null>0 else _C["success"]), (c4,"TARGET DISPONIBLE",1 if "FL_Risk_Target" in cli.columns else 0, _C["success"]), ]: with col: st.markdown( f'
' f'
{label}
' f'
{val}
' f'
', unsafe_allow_html=True) st.markdown("
", unsafe_allow_html=True) # Tableau des features par groupe avec stats _title_bar("INVENTAIRE DES FEATURES PAR GROUPE", _C["ml"]) for group, feats in _FL_GROUPS.items(): avail = [f for f in feats if f in cli.columns] if not avail: continue st.markdown( f'
' f'{group}
', unsafe_allow_html=True) rows = [] for f in avail: s = cli[f] rows.append({ "Feature": f, "Type": "CIBLE" if f == "FL_Risk_Target" else "NUMERIQUE", "Min": round(float(s.min()),3) if s.notna().any() else "N/A", "Max": round(float(s.max()),3) if s.notna().any() else "N/A", "Moy": round(float(s.mean()),3) if s.notna().any() else "N/A", "Nulls": int(s.isna().sum()), }) st.dataframe(pd.DataFrame(rows), use_container_width=True, hide_index=True) st.divider() # Log de nettoyage _title_bar("LOG DES TRANSFORMATIONS", _C["neutral"]) for i, line in enumerate(log): icon = "[ CIBLE ]" if "CIBLE" in line.upper() or "TARGET" in line.upper() else \ "[ ALERTE ]" if "NOTE" in line.upper() or "INOPERANT" in line.upper() or "IMPUTE" in line.upper() else \ f"[ F{i+1:02d} ]" color = _C["warning"] if "ALERTE" in icon else \ _C["ml"] if "CIBLE" in icon else _C["subtext"] st.markdown( f'
' f'{icon}{line}
', unsafe_allow_html=True) def _render_u1_ml(cli: pd.DataFrame, sheets: dict): """U1 : diagnostics ML — heatmap Spearman, feature importance, radar, PCA.""" _title_bar("U1 · DIAGNOSTICS ML", _C["ml"]) mat = _get_fl_matrix(cli) feat_cols = [c for c in _FL_COLS if c in mat.columns and c != "FL_Risk_Target"] id_col = "ID_Client" if "ID_Client" in mat.columns else None # Alerte volume n = len(mat) st.markdown( f'
' f'VOLUME : {n} clients — Resultats indicatifs. ' f'Les correlations et le PCA gagnent en robustesse a partir de n=50+.
', unsafe_allow_html=True) # 1. Heatmap Spearman _title_bar("1 · MATRICE DE CORRELATION SPEARMAN", _C["ml"]) num_mat = mat[feat_cols + (["FL_Risk_Target"] if "FL_Risk_Target" in mat.columns else [])].copy() corr_matrix = num_mat.corr(method="spearman") fig = go.Figure(go.Heatmap( z=corr_matrix.values, x=corr_matrix.columns.tolist(), y=corr_matrix.index.tolist(), colorscale=[ [0.0, "#c0392b"], [0.35, "#2d1010"], [0.5, _C["card"]], [0.65, "#0d2240"], [1.0, "#58a6ff"], ], zmid=0, zmin=-1, zmax=1, text=corr_matrix.round(2).values, texttemplate="%{text}", textfont=dict(size=8, family=_FONT), hovertemplate="%{y} x %{x}
Spearman : %{z:.3f}", colorbar=dict(title="rho", tickvals=[-1,-0.5,0,0.5,1], tickfont=dict(size=9,family=_FONT)), xgap=1, ygap=1, )) fig.update_layout(**_L( height=max(480, len(feat_cols)*22), xaxis=dict(tickangle=-45, side="bottom", tickfont=dict(size=8,family=_FONT)), yaxis=dict(automargin=True, tickfont=dict(size=8,family=_FONT)), margin=dict(t=44, b=120, l=200, r=24), )) st.plotly_chart(fig, use_container_width=True) # 2. Feature Importance (Gradient Boosting vs FL_Risk_Target) _title_bar("2 · FEATURE IMPORTANCE (Gradient Boosting vs FL_Risk_Target)", _C["ml"]) if "FL_Risk_Target" in mat.columns and len(mat) >= 5: X = mat[feat_cols].fillna(0).values y = mat["FL_Risk_Target"].fillna(0).values scaler = StandardScaler() X_sc = scaler.fit_transform(X) gb = GradientBoostingRegressor(n_estimators=100, max_depth=3, random_state=42) gb.fit(X_sc, y) imp = gb.feature_importances_ order = np.argsort(imp) cols_ord = [feat_cols[i] for i in order] imp_ord = imp[order] bar_colors = [_C["ml"] if v > imp.mean() else _C["neutral"] for v in imp_ord] fig = go.Figure(go.Bar( x=imp_ord, y=cols_ord, orientation="h", marker_color=bar_colors, text=[f"{v:.3f}" for v in imp_ord], textposition="outside", hovertemplate="%{y}
Importance : %{x:.4f}", )) fig.add_vline(x=imp.mean(), line_dash="dot", line_color=_C["warning"], annotation_text=f"Moy : {imp.mean():.3f}", annotation_font=dict(size=9, color=_C["warning"])) fig.update_layout(**_L( title="IMPACT SUR LE SCORE DE RISQUE (violet = au-dessus de la moyenne)", height=max(400, len(feat_cols)*22), showlegend=False, xaxis_title="Importance relative", margin=dict(t=44,b=36,l=220,r=40), )) st.plotly_chart(fig, use_container_width=True) else: st.plotly_chart(_ef("FL_Risk_Target absent ou volume insuffisant (< 5).", 200), use_container_width=True) c1, c2 = st.columns(2) # 3. Radar : Bons payeurs vs Profils a risque with c1: _title_bar("3 · RADAR SEGMENT Bons payeurs vs Risque", _C["ml"]) radar_feats = [ "FL_Job_Seniority_vs_Age", "FL_Financial_Fragility_Index", "FL_Repayment_Punctuality_Rate", "FL_Social_Network_Density", "FL_Loan_to_Income_Ratio", ] rf_avail = [f for f in radar_feats if f in mat.columns] if "FL_Risk_Target" in mat.columns and len(rf_avail) >= 3: median_risk = mat["FL_Risk_Target"].median() good = mat[mat["FL_Risk_Target"] <= median_risk] risky = mat[mat["FL_Risk_Target"] > median_risk] def _normalize_group(g, feats): vals = [] for f in feats: col_min = mat[f].min() col_max = mat[f].max() rng = col_max - col_min mean_val = g[f].mean() if len(g) > 0 else 0 vals.append((mean_val - col_min) / rng if rng > 0 else 0.5) return vals good_vals = _normalize_group(good, rf_avail) risky_vals = _normalize_group(risky, rf_avail) short_labels = [f.replace("FL_","").replace("_"," ")[:14] for f in rf_avail] cats = short_labels + [short_labels[0]] fig = go.Figure() for vals, name, color in [ (good_vals, f"Bon payeur (n={len(good)})", _C["success"]), (risky_vals, f"Profil risque (n={len(risky)})", _C["critical"]), ]: v = vals + [vals[0]] fig.add_trace(go.Scatterpolar( r=v, theta=cats, fill="toself", name=name, line=dict(color=color, width=2), fillcolor=color.replace("#","rgba(").replace(")",",0.10)") if "#" in color else color, )) fig.update_layout(**_L( polar=dict( bgcolor=_C["card"], radialaxis=dict(visible=True, range=[0,1], tickfont=dict(size=8,family=_FONT), gridcolor=_C["grid"]), angularaxis=dict(tickfont=dict(size=9,family=_FONT), gridcolor=_C["grid"]), ), title="PROFIL MOYEN (normalise 0-1)", height=380, )) st.plotly_chart(fig, use_container_width=True) # 4. PCA 2D cluster with c2: _title_bar("4 · PCA 2D CLUSTERS CLIENTS", _C["ml"]) if len(mat) >= 4 and len(feat_cols) >= 3: X_pca = mat[feat_cols].fillna(0).values X_sc = StandardScaler().fit_transform(X_pca) pca = PCA(n_components=2, random_state=42) comp = pca.fit_transform(X_sc) var1 = pca.explained_variance_ratio_[0] * 100 var2 = pca.explained_variance_ratio_[1] * 100 df_pca = pd.DataFrame({ "PC1": comp[:,0], "PC2": comp[:,1], "ID_Client": mat.get("ID_Client", mat.index.astype(str)).values if "ID_Client" in mat.columns else mat.index.astype(str), "Risk": mat["FL_Risk_Target"].fillna(0).values if "FL_Risk_Target" in mat.columns else np.zeros(len(mat)), }) df_pca["Nom"] = cli.get("Nom_Complet", df_pca["ID_Client"]).fillna(df_pca["ID_Client"]).values if "Nom_Complet" in cli.columns else df_pca["ID_Client"] fig = go.Figure(go.Scatter( x=df_pca["PC1"], y=df_pca["PC2"], mode="markers+text", text=df_pca["Nom"].str[:10], textposition="top center", textfont=dict(size=8, family=_FONT), marker=dict( size=12, color=df_pca["Risk"], colorscale=[[0,"#2ecc71"],[0.5,"#f39c12"],[1,"#c0392b"]], showscale=True, colorbar=dict(title="Risque", tickfont=dict(size=9,family=_FONT)), line=dict(width=1.5, color=_C["bg"]), ), hovertemplate="%{text}
PC1 : %{x:.2f}
PC2 : %{y:.2f}
Risque : %{marker.color:.3f}", )) fig.update_layout(**_L( title=f"PCA 2D ({var1:.0f}% + {var2:.0f}% variance expliquee)", height=380, showlegend=False, xaxis_title=f"PC1 ({var1:.1f}%)", yaxis_title=f"PC2 ({var2:.1f}%)", )) st.plotly_chart(fig, use_container_width=True) else: st.plotly_chart(_ef("Volume insuffisant pour PCA (< 4 clients).", 380), use_container_width=True) # Scatter bonus : Risk_Target vs Fragility _title_bar("5 · SCATTER FL_Risk_Target vs features cles", _C["ml"]) s_cols = st.columns(3) scatter_pairs = [ ("FL_Financial_Fragility_Index","FL_Risk_Target"), ("FL_Repayment_Punctuality_Rate","FL_Risk_Target"), ("FL_Loan_to_Income_Ratio","FL_Risk_Target"), ] for i,(xf,yf) in enumerate(scatter_pairs): if xf in mat.columns and yf in mat.columns: with s_cols[i]: fig = go.Figure(go.Scatter( x=mat[xf], y=mat[yf], mode="markers+text", text=cli.get("Nom_Complet", mat.get("ID_Client", pd.Series(range(len(mat))))).str[:8] if "Nom_Complet" in cli.columns else mat.get("ID_Client",pd.Series(range(len(mat)))).astype(str), textposition="top center", textfont=dict(size=8,family=_FONT), marker=dict(size=10, color=mat[yf], colorscale=[[0,"#2ecc71"],[1,"#c0392b"]], line=dict(width=1,color=_C["bg"])), hovertemplate=f"%{{text}}
{xf.replace('FL_','')}: %{{x:.3f}}
Risk: %{{y:.3f}}", )) fig.update_layout(**_L( title=xf.replace("FL_","").replace("_"," ")[:22], height=260, showlegend=False, xaxis_title=xf.replace("FL_","")[:16], yaxis_title="Risk Target", margin=dict(t=36,b=30,l=50,r=10), )) st.plotly_chart(fig, use_container_width=True) st.divider() # 6. Cluster maps : Clients / Garants / Prets / Updates _render_cluster_section(sheets) def _pca_cluster(df: pd.DataFrame, feat_cols: list, label_col: str, color_col: str, color_title: str, title: str, colorscale=None) -> go.Figure: """ PCA 2D generique pour n'importe quelle table. feat_cols : colonnes numeriques a utiliser label_col : colonne de nom/id affichee sur chaque point color_col : colonne numerique qui pilote la couleur """ if colorscale is None: colorscale = [[0,"#2ecc71"],[0.5,"#f39c12"],[1,"#c0392b"]] avail = [c for c in feat_cols if c in df.columns] if len(df) < 3 or len(avail) < 2: return _ef(f"Volume ou features insuffisants ({len(df)} lignes, {len(avail)} features).", 360) X = df[avail].fillna(df[avail].median()).values try: X_sc = StandardScaler().fit_transform(X) n_comp = min(2, X_sc.shape[1], X_sc.shape[0]) pca = PCA(n_components=n_comp, random_state=42) comp = pca.fit_transform(X_sc) var1 = pca.explained_variance_ratio_[0] * 100 var2 = pca.explained_variance_ratio_[1] * 100 if n_comp > 1 else 0.0 except Exception as e: return _ef(f"PCA impossible : {e}", 360) labels = df[label_col].astype(str).str[:12] if label_col in df.columns else df.index.astype(str) colors = df[color_col].fillna(df[color_col].median()) if color_col in df.columns else pd.Series(0, index=df.index) pc2 = comp[:,1] if n_comp > 1 else pd.Series(0, index=range(len(comp))) fig = go.Figure(go.Scatter( x=comp[:,0], y=pc2, mode="markers+text", text=labels, textposition="top center", textfont=dict(size=8, family=_FONT, color=_C["subtext"]), marker=dict( size=13, color=colors, colorscale=colorscale, showscale=True, colorbar=dict(title=color_title, tickfont=dict(size=9, family=_FONT), len=0.7), line=dict(width=1.5, color=_C["bg"]), ), hovertemplate=( "%{text}
" "PC1 : %{x:.2f}
PC2 : %{y:.2f}
" f"{color_title} : %{{marker.color:.3f}}" ), )) fig.update_layout(**_L( title=f"{title} ({var1:.0f}% + {var2:.0f}% var.)", height=370, showlegend=False, xaxis_title=f"PC1 ({var1:.1f}%)", yaxis_title=f"PC2 ({var2:.1f}%)", )) return fig def _render_cluster_section(sheets: dict): """ PCA 2D par entite : Clients, Garants, Prets_Master, Prets_Update. """ HDR_STYLE = ( 'font-family:JetBrains Mono,monospace;font-size:0.58rem;' 'letter-spacing:2px;text-transform:uppercase;color:#58a6ff;margin:12px 0 4px 0;' ) def _hdiv(label): st.markdown(f'
{label}
', unsafe_allow_html=True) _title_bar("CLUSTER MAPS PCA 2D PAR ENTITE", _C["ml"]) st.markdown( '
' 'Chaque point = une entite. Couleur = variable de risque principale. ' 'La proximite = similarite multi-dimensionnelle (PCA).
', unsafe_allow_html=True, ) cli = sheets.get("Clients_KYC", pd.DataFrame()) gar = sheets.get("Garants_KYC", pd.DataFrame()) prt = sheets.get("Prets_Master", pd.DataFrame()) upd = sheets.get("Prets_Update", pd.DataFrame()) # ── CLIENTS ─────────────────────────────────────────────────────────────── _hdiv("CLIENTS") cli_feats = [ "FL_Age_Risk_Index","FL_Job_Seniority_vs_Age","FL_Social_Network_Density", "FL_Net_Disposable_Income","FL_Financial_Fragility_Index", "FL_Loan_to_Income_Ratio","FL_Repayment_Punctuality_Rate", "FL_Late_Gravity_Score","FL_Renegotiation_Frequency", "FL_Effective_APR","FL_Yield_per_Client", ] c1, c2 = st.columns(2) with c1: fig = _pca_cluster(cli, cli_feats, "Nom_Complet", "FL_Risk_Target", "Risque", "CLIENTS (couleur = FL_Risk_Target)") st.plotly_chart(fig, use_container_width=True) with c2: fig = _pca_cluster(cli, cli_feats, "Nom_Complet", "FL_Financial_Fragility_Index", "Fragilite", "CLIENTS (couleur = Fragilite Financiere)", colorscale=[[0,"#58a6ff"],[0.5,"#f39c12"],[1,"#c0392b"]]) st.plotly_chart(fig, use_container_width=True) st.divider() # ── GARANTS ─────────────────────────────────────────────────────────────── _hdiv("GARANTS") gar_feats = [ "Revenus_Mensuels","Autres_Revenus","Charges_Estimees", "Patrimoine_Declare","Pers_Charge","Anciennete_Emploi","Checking_frequency", "DX_Guarantee_Leverage_Score","DX_Family_Pressure_Index", "DX_Garant_Availability_Score","DX_Net_Worth_Credibility_Check", ] if not gar.empty: gar = gar.copy() rev_g = gar["Revenus_Mensuels"].fillna(0) + gar["Autres_Revenus"].fillna(0) gar["_gar_pression"] = (gar["Charges_Estimees"].fillna(0) / rev_g.replace(0, np.nan)).fillna(0).round(3) gar["_gar_leverage"] = gar.get("DX_Guarantee_Leverage_Score", pd.Series(0, index=gar.index)).fillna(0) c3, c4 = st.columns(2) with c3: fig = _pca_cluster(gar, gar_feats, "Nom_Complet", "_gar_pression", "Pression fin.", "GARANTS (couleur = Charges/Revenus)") st.plotly_chart(fig, use_container_width=True) with c4: fig = _pca_cluster(gar, gar_feats, "Nom_Complet", "_gar_leverage", "Leverage", "GARANTS (couleur = Leverage Patrimoine)", colorscale=[[0,"#c0392b"],[0.5,"#f39c12"],[1,"#2ecc71"]]) st.plotly_chart(fig, use_container_width=True) st.divider() # ── PRETS MASTER ────────────────────────────────────────────────────────── _hdiv("PRETS MASTER") prt_feats = [ "Montant_Capital","Taux_Hebdo","Taux_Endettement","Duree_Semaines", "Montant_Versement","Montant_Total","Cout_Credit","Nb_Versements", "DX_Interest_Yield_Factor","DX_Debt_to_Income_Pressure", "DX_Motif_Risk_Weight","DX_Collateral_Coverage_Ratio", "EX_Endettement_Critical_Threshold","EX_Duration_Anomaly_Detection", ] if not prt.empty: prt = prt.copy() prt["_prt_risk"] = ( prt.get("DX_Motif_Risk_Weight", pd.Series(0.5, index=prt.index)).fillna(0.5) * 0.5 + (prt["Taux_Endettement"].fillna(0) / 100).clip(0, 1) * 0.5 ).round(3) c5, c6 = st.columns(2) with c5: fig = _pca_cluster(prt, prt_feats, "ID_Pret", "_prt_risk", "Risque contrat", "PRETS (couleur = Motif Risk + Endettement)") st.plotly_chart(fig, use_container_width=True) with c6: fig = _pca_cluster(prt, prt_feats, "Nom_Complet", "Taux_Endettement", "Endettement %", "PRETS (couleur = Taux_Endettement)", colorscale=[[0,"#2ecc71"],[0.4,"#f39c12"],[1,"#c0392b"]]) st.plotly_chart(fig, use_container_width=True) st.divider() # ── PRETS UPDATE ────────────────────────────────────────────────────────── _hdiv("PRETS UPDATE (historique des modifications)") upd_feats = [ "Version","Montant_Capital","Taux_Hebdo","Taux_Endettement", "Duree_Semaines","Montant_Total","Cout_Credit", "DX_Contract_Volatility_Delta","DX_Update_Velocity", "DX_Renegotiation_Gravity_Score","DX_Structural_Fragility_Index", "EX_Modification_Pattern_Clustering","EX_Version_Drift_Analysis", ] if not upd.empty: upd = upd.copy() upd["_upd_grav"] = upd.get("DX_Renegotiation_Gravity_Score", pd.Series(0, index=upd.index)).fillna(0).round(3) raw_drift = upd.get("EX_Version_Drift_Analysis", pd.Series(0, index=upd.index)).fillna(0) mx = raw_drift.abs().max() upd["_upd_drift"] = (raw_drift / mx).round(3) if mx > 0 else pd.Series(0.0, index=upd.index) c7, c8 = st.columns(2) with c7: if upd.empty: st.plotly_chart(_ef("Table Prets_Update vide.", 360), use_container_width=True) else: fig = _pca_cluster(upd, upd_feats, "ID_Pret", "_upd_grav", "Gravite reneg.", "UPDATES (couleur = Gravite Renegociation)") st.plotly_chart(fig, use_container_width=True) with c8: if upd.empty: st.plotly_chart(_ef("Table Prets_Update vide.", 360), use_container_width=True) else: fig = _pca_cluster(upd, upd_feats, "ID_Pret", "_upd_drift", "Derive montant", "UPDATES (couleur = Derive Montant Total)", colorscale=[[0,"#58a6ff"],[0.5,"#f39c12"],[1,"#c0392b"]]) st.plotly_chart(fig, use_container_width=True) def _render_u2_export(cli: pd.DataFrame, log: list, sheets: dict): """U2 : console d'exportation OAG.""" _title_bar("U2 · CONSOLE D'EXPORTATION OAG", _C["success"]) # Construire le dataset enrichi complet prt = sheets.get("Prets_Master", pd.DataFrame()) # Colonnes metier cli (sans DX_, EX_, FL_ intermediaires ni temporaires) base_cols = [c for c in cli.columns if not c.startswith(("DX_","EX_","_")) and c not in _FL_COLS] fl_cols_present = [c for c in _FL_COLS if c in cli.columns] # Jointure avec agregats Prets pour enrichir le contexte if not prt.empty and "ID_Client" in prt.columns: prt_agg = prt.groupby("ID_Client").agg( nb_prets=("ID_Pret","count"), capital_total=("Montant_Capital","sum"), taux_moyen=("Taux_Hebdo","mean"), duree_moy=("Duree_Semaines","mean"), statut_dominant=("Statut", lambda x: x.value_counts().index[0]), ).reset_index() export_df = cli[base_cols + fl_cols_present].merge( prt_agg, on="ID_Client", how="left") else: export_df = cli[base_cols + fl_cols_present].copy() # Preview _title_bar("MATRICE OAG — APERCU (10 premieres lignes)", _C["success"]) preview_cols = ["ID_Client"] + [c for c in fl_cols_present if c != "FL_Risk_Target"] + ["FL_Risk_Target"] preview_avail = [c for c in preview_cols if c in export_df.columns] st.dataframe( export_df[preview_avail].head(10).round(4), use_container_width=True, ) st.markdown("
", unsafe_allow_html=True) # Stats de nettoyage n_total = export_df.shape[0] * export_df.shape[1] n_null_init = export_df.isnull().sum().sum() export_clean = export_df.fillna(export_df.median(numeric_only=True)) n_imputed = int(n_null_init) _title_bar("LOG DE NETTOYAGE PRE-EXPORT", _C["neutral"]) clean_log = [ f"Lignes x colonnes : {export_df.shape[0]} x {export_df.shape[1]}", f"Valeurs manquantes imputees par la mediane : {n_imputed}", f"Features FL_* incluses : {len(fl_cols_present)}", f"Variables categoriques NON encodees (necessite LabelEncoder pour modeles sklearn)", f"FL_Risk_Target : variable cible continue [0.0 - 1.0] — compatible regression et classification (seuil 0.4)", f"Taille dataset export : {export_clean.memory_usage(deep=True).sum() / 1024:.1f} KB", ] for line in clean_log: icon = "[ INFO ]" color = _C["subtext"] if "manquantes" in line or "encodees" in line: icon = "[ ALERTE ]" color = _C["warning"] elif "FL_Risk_Target" in line: icon = "[ CIBLE ]" color = _C["ml"] st.markdown( f'
' f'{icon}{line}
', unsafe_allow_html=True) st.markdown("
", unsafe_allow_html=True) # Bouton de telechargement csv_bytes = export_clean.to_csv(index=False).encode("utf-8") st.download_button( label="TELECHARGER LE DATASET ENRICHI POUR IA (.CSV)", data=csv_bytes, file_name=f"VortexFlux_OAG_Dataset_{datetime.today().strftime('%Y%m%d')}.csv", mime="text/csv", use_container_width=True, type="primary", ) # Log complet en expander with st.expander("LOG COMPLET DES TRANSFORMATIONS FL_*", expanded=False): for i,line in enumerate(log): st.markdown( f'
' f'[{i+1:02d}]' f'{line}
', unsafe_allow_html=True) # ── Point d'entree public ───────────────────────────────────────────────────── def render_feature_store(sheets: dict): """ Appele depuis Usis.py quand mode == "U3 ML FEATURE STORE". sheets doit avoir ete enrichi par enrich_dx(), enrich_ex(), enrich_fl(). """ cli = sheets.get("Clients_KYC", pd.DataFrame()) log = sheets.get("_fl_log", []) if cli.empty: st.error("[ ML ] Table Clients_KYC vide — impossible de calculer les features.") return fl_present = [c for c in _FL_COLS if c in cli.columns] if not fl_present: st.warning("[ ML ] Aucune feature FL_* detectee. Verifiez que enrich_fl() a ete appele.") return tab_u0, tab_u1, tab_u2 = st.tabs([ "U0 PROTOCOLE INGENIERIE", "U1 DIAGNOSTICS ML", "U2 EXPORT OAG", ]) with tab_u0: _render_u0_protocol(cli, log) with tab_u1: _render_u1_ml(cli, sheets) with tab_u2: _render_u2_export(cli, log, sheets)