Update src/module/AIP.py

src/module/AIP.py

@@ -1,11 +1,7 @@
 """
-MODULE JASMINE AGENT - VERSION OMNISCIENTE (ETL + DASHBOARD + DATA LAB)
-=======================================================================
+MODULE JASMINE AGENT - VERSION OMNISCIENTE V2 (ETL + DASHBOARD + DATA LAB + DATA QUALITY)
+=========================================================================================
 Responsabilité : Agent IA centralisé.
-Capacités : 
-1. Pilotage de l'ETL (Mise à jour données)
-2. Lecture du Dashboard (KPIs temps réel)
-3. Exploration du Data Lab (Stats patients, pathologies, démographie)
 """
 
 import google.generativeai as genai
@@ -22,16 +18,16 @@ import traceback
 try:
     from module.Validation_Contract import ETL_Engine
     from module.Dashboard import HospitalDashboard
-    from module.Data_Exploration import HospitalAdvancedAnalyzer # <--- NOUVELLE CAPACITÉ
-except ImportError:
-    st.error("❌ Erreur critique : Impossible d'importer les modules métiers (ETL, Dashboard ou Data Lab).")
+    from module.Data_Exploration import HospitalAdvancedAnalyzer
+    from module.Indicateurs_DQ import HospitalDataQualityAnalyzer, load_from_google_sheets
+except ImportError as e:
+    st.error(f"❌ Erreur d'importation des modules : {str(e)}")
 
 # --- STYLE GOTHAM / AIP TERMINAL ---
 def apply_custom_styles():
     st.markdown("""
     <style>
     @import url('https://fonts.googleapis.com/css2?family=Inter:wght@400;600&family=JetBrains+Mono:wght@400&display=swap');
-
     .user-msg { 
         background-color: rgba(20, 30, 45, 0.9); border: 1px solid #2b3b55;
         border-left: 3px solid #64B5F6; padding: 15px; margin: 10px 0; 
@@ -56,11 +52,10 @@ def apply_custom_styles():
 class JasmineAgent:
     def __init__(self, gspread_client):
         self.gspread_client = gspread_client
-        
-        # --- SECRETS & CLIENTS ---
         self.google_key = os.environ.get("GOOGLE_API_KEY")
         self.groq_key = os.environ.get("GROQ_API_KEY")
-
+        
+        # Fallbacks secrets
         if not self.google_key:
             try: self.google_key = st.secrets.get("GOOGLE_API_KEY")
             except: pass
@@ -71,7 +66,6 @@ class JasmineAgent:
         self.groq_client = Groq(api_key=self.groq_key) if self.groq_key else None
         if self.google_key: genai.configure(api_key=self.google_key)
 
-        # --- CASCADE DE MODÈLES ---
         self.MODEL_CASCADE = [
             "gemini-2.0-flash-exp",
             "llama-3.3-70b-versatile",
@@ -79,24 +73,26 @@ class JasmineAgent:
             "llama-3.1-8b-instant"
         ]
         
-        # --- OUTILS DISPONIBLES ---
         self.tools_map = {
             "run_etl_pipeline": {
-                "description": "Lance l'ETL pour mettre à jour les données.",
+                "description": "Met à jour et nettoie les données.",
                 "func": self._run_etl_wrapper
             },
             "get_hospital_kpis": {
-                "description": "Récupère les KPIs temps réel (Occupation, Tension RH, Stocks, Admissions du jour).",
+                "description": "Récupère les chiffres clés (Lits, RH, Stocks).",
                 "func": self._get_kpis_wrapper
             },
-            # --- NOUVEL OUTIL : DATA LAB ---
             "analyze_patient_stats": {
-                "description": "Analyse approfondie des patients : âge moyen, ratio H/F, pathologie la plus fréquente, durée moyenne de séjour.",
+                "description": "Stats patients et pathologies (Data Lab).",
                 "func": self._get_data_exploration_wrapper
+            },
+            "get_data_quality_report": {
+                "description": "Audit complet de la qualité et des erreurs dans les données.",
+                "func": self._get_dq_stats_wrapper
             }
         }
 
-    # --- WRAPPERS TECHNIQUES ---
+    # --- WRAPPERS ---
 
     def _run_etl_wrapper(self, args=None):
         try:
@@ -104,72 +100,64 @@ class JasmineAgent:
             success, result = engine.run_etl()
             if success:
                 st.session_state.data_explorer = result
-                summary = result.get_summary()
-                return {"status": "succès", "details": summary}
-            return {"status": "erreur", "message": f"Echec ETL: {result}"}
+                return {"status": "succès", "details": result.get_summary()}
+            return {"status": "erreur", "message": "Échec de l'exécution ETL."}
         except Exception as e: return {"status": "crash", "message": str(e)}
 
     def _get_kpis_wrapper(self, args=None):
         try:
             dash = HospitalDashboard(self.gspread_client, "bdd_mvp_faker")
-            if not dash.load_data():
-                return {"status": "erreur", "message": "Données non chargées."}
+            if not dash.load_data(): return {"status": "erreur", "message": "Données Dashboard indisponibles."}
             return {"status": "succès", "details": dash._get_kpis()}
         except Exception as e: return {"status": "crash", "message": str(e)}
 
     def _get_data_exploration_wrapper(self, args=None):
-        """Connecteur vers le module Data Exploration pour les stats avancées"""
         try:
             analyzer = HospitalAdvancedAnalyzer(self.gspread_client, "bdd_mvp_faker")
-            if not analyzer.load_data():
-                return {"status": "erreur", "message": "Impossible de charger le Data Lab."}
-            
-            # Extraction des stats clés depuis les dataframes du module
+            if not analyzer.load_data(): return {"status": "erreur", "message": "Data Lab inaccessible."}
             df_pat = analyzer.get_df('patients')
             df_sej = analyzer.get_df('sejours')
-            
             stats = {}
-            
-            # 1. Démographie
             if df_pat is not None and not df_pat.empty:
                 df_pat['age'] = pd.to_numeric(df_pat['age'], errors='coerce')
                 stats['age_moyen'] = round(df_pat['age'].mean(), 1)
                 stats['total_patients'] = len(df_pat)
-                counts = df_pat['sexe'].value_counts(normalize=True)
-                stats['ratio_hommes'] = round(counts.get('M', 0) * 100, 1)
-                stats['ratio_femmes'] = round(counts.get('F', 0) * 100, 1)
-                stats['top_ville'] = df_pat['ville'].mode()[0] if 'ville' in df_pat.columns else "N/A"
-
-            # 2. Séjours & Pathos
             if df_sej is not None and not df_sej.empty:
-                df_sej['duree'] = pd.to_numeric(df_sej['duree_sejour_jours'], errors='coerce')
-                stats['duree_moyenne_sejour'] = round(df_sej['duree'].mean(), 1)
                 stats['pathologie_top'] = df_sej['pathologie_code'].mode()[0]
-                stats['gravite_moyenne'] = round(pd.to_numeric(df_sej['gravite_1_5'], errors='coerce').mean(), 2)
-
             return {"status": "succès", "details": stats}
-            
-        except Exception as e:
-            return {"status": "crash", "message": f"Erreur Data Lab: {str(e)}"}
+        except Exception as e: return {"status": "crash", "message": str(e)}
+
+    def _get_dq_stats_wrapper(self, args=None):
+        try:
+            raw_data = load_from_google_sheets(self.gspread_client, "bdd_mvp_faker")
+            if not raw_data: return {"status": "erreur", "message": "Impossible de lire Google Sheets."}
+            dq_analyzer = HospitalDataQualityAnalyzer(raw_data)
+            scores = dq_analyzer.compute_global_score()
+            metrics = dq_analyzer.metrics
+            return {
+                "status": "succès", 
+                "details": {
+                    "score_global": scores['global'],
+                    "completude": scores['completeness'],
+                    "anomalies_pat": metrics['patients']['anomalies'],
+                    "doublons": metrics['patients']['doublons']
+                }
+            }
+        except Exception as e: return {"status": "crash", "message": str(e)}
 
-    # --- INTELLIGENCE CENTRALE ---
+    # --- CORE LOGIC ---
 
     def build_system_prompt(self):
         tools_desc = "\n".join([f"- {k}: {v['description']}" for k, v in self.tools_map.items()])
         return f"""
-        ROLE: Tu es Jasmine, l'IA de supervision de l'hôpital.
-        OBJECTIF: Analyser les données, piloter les outils et conseiller la direction.
-        
-        OUTILS :
+        ROLE: Tu es Jasmine, l'IA de supervision.
+        OUTILS DISPONIBLES :
         {tools_desc}
-        
         RÈGLES :
-        1. Utilise 'get_hospital_kpis' pour l'état actuel (Lits, Stocks, RH).
-        2. Utilise 'analyze_patient_stats' pour les questions démographiques ou médicales (Âge, Pathos).
-        3. Si l'utilisateur demande ton avis ou une analyse SANS outil, réponds dans le champ "answer".
-        4. Réponds TOUJOURS au format JSON strict :
+        1. 'get_data_quality_report' pour toute question sur la fiabilité ou les erreurs.
+        2. Réponds TOUJOURS au format JSON :
         {{
-          "thought": "Raisonnement...",
+          "thought": "Ton analyse...",
           "tool": "nom_outil" (ou "none"),
           "answer": "Ta réponse directe si tool=none",
           "args": {{}} 
@@ -178,11 +166,9 @@ class JasmineAgent:
 
     def ask(self, user_message, chat_history=[]):
         system_prompt = self.build_system_prompt()
-        last_error = None
         response_payload = None
         used_model = None
 
-        # Fallback Loop
         for model_name in self.MODEL_CASCADE:
             try:
                 if "gemini" in model_name:
@@ -195,112 +181,56 @@ class JasmineAgent:
                     if not self.groq_client: continue
                     completion = self.groq_client.chat.completions.create(
                         model=model_name,
-                        messages=[{"role": "system", "content": system_prompt},
-                                  {"role": "user", "content": user_message}],
+                        messages=[{"role": "system", "content": system_prompt}, {"role": "user", "content": user_message}],
                         response_format={"type": "json_object"})
                     response_payload = completion.choices[0].message.content
                 used_model = model_name
                 break 
-            except Exception as e:
-                last_error = str(e)
-                continue
+            except: continue
 
-        if not response_payload:
-            return f"❌ Erreur système : {last_error}", None, None
+        if not response_payload: return "❌ Erreur de connexion API.", None, None
 
         try:
             clean_json = re.sub(r'```json\s*|\s*```', '', response_payload).strip()
             decision = json.loads(clean_json)
-            
             tool_name = decision.get("tool")
             thought = f"Modèle: {used_model}\n\n{decision.get('thought', '')}"
             
-            # Exécution de l'outil
             if tool_name in self.tools_map:
-                tool_result = self.tools_map[tool_name]["func"]({})
-                
-                if tool_result["status"] == "succès":
-                    d = tool_result["details"]
-                    
-                    # --- FORMATAGE DE LA RÉPONSE SELON L'OUTIL ---
-                    if tool_name == "run_etl_pipeline":
-                        final_response = (f"✅ **ETL Terminé.**\n"
-                                          f"- Tables : {d['tables_loaded']}\n"
-                                          f"- Lignes : {d['total_rows_clean']:,}")
-                                          
+                res = self.tools_map[tool_name]["func"]({})
+                if res["status"] == "succès":
+                    d = res["details"]
+                    if tool_name == "get_data_quality_report":
+                        score = d['score_global']
+                        icon = "🟢" if score > 90 else "🟠" if score > 75 else "🔴"
+                        final_response = f"🛡️ **Qualité :** {icon} {score}/100\n🔹 Complétude : {d['completude']}%\n🔹 Anomalies : {d['anomalies_pat']}"
                     elif tool_name == "get_hospital_kpis":
-                        taux = float(d.get('taux_global', 0)) # Sécurité Float
-                        final_response = (f"📊 **État Hôpital :**\n\n"
-                                          f"🔹 **Occupation :** {taux:.1f}%\n"
-                                          f"🔹 **Admissions J :** {d.get('nb_adm')}\n"
-                                          f"🔹 **Alertes Stocks :** {d.get('nb_ruptures')}\n"
-                                          f"🔹 **Services tendus :** {d.get('nb_tension')}")
-                                          
-                    elif tool_name == "analyze_patient_stats":
-                        final_response = (f"🔬 **Analyse Patients :**\n\n"
-                                          f"🔹 **Population :** {d.get('total_patients')} patients\n"
-                                          f"🔹 **Âge moyen :** {d.get('age_moyen')} ans\n"
-                                          f"🔹 **Répartition :** 👨 {d.get('ratio_hommes')}% | 👩 {d.get('ratio_femmes')}%\n"
-                                          f"🔹 **Pathologie #1 :** {d.get('pathologie_top')}\n"
-                                          f"🔹 **Durée moy. séjour :** {d.get('duree_moyenne_sejour')} jours")
-                    else:
-                        final_response = "Action effectuée."
-                else:
-                    final_response = f"⚠️ Erreur technique : {tool_result['message']}"
-                
-                return final_response, thought, tool_result
-            
-            else:
-                # Mode Conversation / Conseil direct (via JSON 'answer')
-                final_response = decision.get("answer", "Je suis prête à analyser les données.")
-                return final_response, thought, None
-
-        except Exception as e:
-            return f"Erreur décodage IA ({used_model}): {str(e)}", None, None
+                        final_response = f"📊 **KPIs :** Lits à {d['taux_global']:.1f}% | Tension RH: {d['tension_rh']:.1f}%"
+                    else: final_response = f"✅ Action {tool_name} terminée."
+                else: final_response = f"⚠️ {res.get('message')}"
+                return final_response, thought, res
+            return decision.get("answer", "Analyse terminée."), thought, None
+        except Exception as e: return f"Erreur IA : {str(e)}", None, None
 
-
-# --- INTERFACE ---
 def show_terminal(client, sheet_name="bdd_mvp_faker"):
     apply_custom_styles()
     st.markdown("## 🤖 AIP COMMAND TERMINAL")
-    st.caption("Agent intelligent connecté : ETL • Dashboard • Data Lab")
-
-    if 'agent_instance' not in st.session_state:
-        st.session_state.agent_instance = JasmineAgent(client)
+    if 'agent_instance' not in st.session_state: st.session_state.agent_instance = JasmineAgent(client)
+    if "chat_history" not in st.session_state: st.session_state.chat_history = []
 
-    if "chat_history" not in st.session_state:
-        st.session_state.chat_history = []
-
-    # Affichage Historique
     for msg in st.session_state.chat_history:
         role_class = "user-msg" if msg["role"] == "user" else "bot-msg"
         st.markdown(f'<div class="{role_class}">{msg["content"]}</div>', unsafe_allow_html=True)
-        
         if msg.get("thought"):
-            with st.expander("👁️ AIP LOGIC TRACE"):
-                st.markdown(f'<div class="debug-block">{msg["thought"]}</div>', unsafe_allow_html=True)
-        
-        if msg.get("data"):
-            with st.expander("💾 SYSTEM PAYLOAD"):
-                st.json(msg["data"])
-
-    # Input
-    user_input = st.chat_input("Ex: 'Quelle est la pathologie la plus fréquente ?' ou 'Mets à jour les données'...")
-
+            with st.expander("👁️ AIP LOGIC TRACE"): st.markdown(f'<div class="debug-block">{msg["thought"]}</div>', unsafe_allow_html=True)
+    
+    user_input = st.chat_input("Dites 'Quel est le score de qualité ?'...")
     if user_input:
         st.session_state.chat_history.append({"role": "user", "content": user_input})
         st.rerun()
 
     if st.session_state.chat_history and st.session_state.chat_history[-1]["role"] == "user":
-        last_prompt = st.session_state.chat_history[-1]["content"]
-        
         with st.spinner("Jasmine analyse..."):
-            response, thought, data = st.session_state.agent_instance.ask(last_prompt)
-            
-            st.session_state.chat_history.append({
-                "role": "assistant", 
-                "content": response,
-                "thought": thought,
-                "data": data
-            })
+            response, thought, data = st.session_state.agent_instance.ask(st.session_state.chat_history[-1]["content"])
+            st.session_state.chat_history.append({"role": "assistant", "content": response, "thought": thought, "data": data})
             st.rerun()