Update src/module/Validation_Contract.py

src/module/Validation_Contract.py

@@ -0,0 +1,281 @@
+"""
+MODULE: VALIDATION CONTRACT (ETL ENGINE)
+========================================
+Responsabilité : Centraliser le schéma, extraire, nettoyer et VISUALISER l'impact du nettoyage.
+Pipeline: Extract -> Transform -> Validate -> Load -> Report
+"""
+import pandas as pd
+import numpy as np
+import streamlit as st
+import plotly.express as px
+import plotly.graph_objects as go
+from datetime import datetime
+
+class DataExplorer:
+    """Conteneur intelligent pour stocker les données et les métriques ETL"""
+    
+    def __init__(self):
+        self.dataframes = {}
+        self.metadata = {}
+        self.load_timestamp = None
+        
+    def add_dataframe(self, name, df, metrics):
+        """
+        Ajoute un dataframe et ses métriques de performance ETL
+        metrics = {
+            'rows_before': int, 'rows_after': int,
+            'mem_before': float, 'mem_after': float
+        }
+        """
+        self.dataframes[name] = df
+        
+        # Calcul des métadonnées détaillées
+        numeric_cols = df.select_dtypes(include=[np.number]).columns.tolist()
+        categorical_cols = df.select_dtypes(include=['object']).columns.tolist()
+        date_cols = df.select_dtypes(include=['datetime64']).columns.tolist()
+        
+        # Détection des anomalies si la colonne existe (spécifique à ton projet)
+        anomalies_count = 0
+        if 'dq_is_anomaly' in df.columns:
+            anomalies_count = int(df['dq_is_anomaly'].sum())
+
+        self.metadata[name] = {
+            # Métriques ETL (Avant/Après)
+            'rows_before': metrics.get('rows_before', len(df)),
+            'rows_after': len(df),
+            'mem_before_mb': metrics.get('mem_before', 0),
+            'mem_after_mb': df.memory_usage(deep=True).sum() / 1024**2,
+            
+            # Structure
+            'columns': len(df.columns),
+            'dtypes': df.dtypes.astype(str).to_dict(),
+            'null_cells': int(df.isnull().sum().sum()),
+            'anomalies_detected': anomalies_count,
+            
+            # Typologie
+            'numeric_columns': numeric_cols,
+            'categorical_columns': categorical_cols,
+            'datetime_columns': date_cols
+        }
+    
+    def get_summary(self):
+        """Retourne un résumé global"""
+        total_rows = sum(m['rows_after'] for m in self.metadata.values())
+        total_mem = sum(m['mem_after_mb'] for m in self.metadata.values())
+        return {
+            'tables_loaded': len(self.dataframes),
+            'total_rows': total_rows,
+            'total_memory_mb': round(total_mem, 2),
+            'timestamp': self.load_timestamp
+        }
+
+    def show_etl_report(self):
+        """Génère un tableau de bord visuel de la performance ETL"""
+        st.markdown("---")
+        st.markdown("### 🏗️ Rapport de Performance ETL")
+        
+        if not self.metadata:
+            st.warning("Aucune métadonnée disponible.")
+            return
+
+        # Préparation des données pour les graphiques
+        names = []
+        rows_delta = []
+        mem_delta = []
+        anomalies = []
+        
+        for name, meta in self.metadata.items():
+            names.append(name)
+            # Calcul du % de lignes conservées
+            retention = (meta['rows_after'] / meta['rows_before'] * 100) if meta['rows_before'] > 0 else 0
+            rows_delta.append(retention)
+            # Gain mémoire
+            gain = meta['mem_before_mb'] - meta['mem_after_mb']
+            mem_delta.append(gain)
+            anomalies.append(meta['anomalies_detected'])
+
+        col1, col2 = st.columns(2)
+        
+        with col1:
+            # Graphique 1 : Optimisation Mémoire
+            fig_mem = go.Figure()
+            fig_mem.add_trace(go.Bar(
+                x=names, 
+                y=[m['mem_before_mb'] for m in self.metadata.values()],
+                name='Avant Clean', marker_color='gray'
+            ))
+            fig_mem.add_trace(go.Bar(
+                x=names, 
+                y=[m['mem_after_mb'] for m in self.metadata.values()],
+                name='Après Clean', marker_color='#00CC96'
+            ))
+            fig_mem.update_layout(title="Optimisation Mémoire (MB)", barmode='group')
+            st.plotly_chart(fig_mem, use_container_width=True)
+
+        with col2:
+            # Graphique 2 : Qualité Intrinsèque (Anomalies flagguées)
+            fig_qual = px.bar(
+                x=names, y=anomalies, 
+                title="Lignes marquées 'Anomaly' par l'ETL",
+                labels={'x': 'Table', 'y': 'Nb Anomalies'},
+                color=anomalies, color_continuous_scale='Reds'
+            )
+            st.plotly_chart(fig_qual, use_container_width=True)
+            
+        # Graphique 3 : Rétention des données
+        with st.expander("📊 Détails volumétrie et types"):
+            df_meta = pd.DataFrame.from_dict(self.metadata, orient='index')
+            df_display = df_meta[['rows_before', 'rows_after', 'mem_after_mb', 'null_cells', 'anomalies_detected']]
+            df_display.columns = ['Lignes (Brut)', 'Lignes (Clean)', 'Mémoire (MB)', 'Cellules Vides', 'Anomalies DQ']
+            st.dataframe(df_display, use_container_width=True)
+
+
+class ETL_Engine:
+    """Moteur ETL garantissant le Contrat de Données"""
+
+    def __init__(self, gspread_client, sheet_name="bdd_mvp_faker"):
+        self.client = gspread_client
+        self.sheet_name = sheet_name
+        
+        # --- SCHEMA CONTRACTUEL ---
+        self.SCHEMA = {
+            'patients': {
+                'numeric': ['age', 'code_postal'],
+                'date': ['date_naissance'],
+                'categorical': ['patient_id', 'nom', 'prenom', 'sexe', 'ville', 
+                               'num_carte_vitale_fictif', 'num_carte_vitale_masque', 
+                               'pathologie_frequente_code'],
+                'preserve_as_is': ['dq_is_anomaly', 'dq_anomaly_type', 'dq_anomaly_detail']
+            },
+            'sejours': {
+                'numeric': ['gravite_1_5', 'duree_sejour_jours', 'consommables_estimes_score'],
+                'date': ['date_entree', 'date_sortie'],
+                'categorical': ['stay_id', 'patient_id', 'service_code', 'mode_entree', 
+                               'type_lit', 'pathologie_code', 'scenario_jour', 
+                               'equipement_principal_code'],
+                'preserve_as_is': ['dq_is_anomaly', 'dq_anomaly_type', 'dq_anomaly_detail']
+            },
+            'ressources_jour': {
+                'numeric': ['intensity', 'lits_total', 'lits_occupes', 'lits_disponibles', 
+                           'taux_occupation', 'personnel_total', 'taux_absence_conges', 
+                           'taux_absence_scenario', 'est_periode_conges', 'personnel_disponible', 
+                           'personnel_requis', 'tension_personnel'],
+                'date': ['date'],
+                'categorical': ['service_code', 'scenario', 'periode_conges_label'],
+                'preserve_as_is': ['dq_is_anomaly', 'dq_anomaly_type', 'dq_anomaly_detail']
+            },
+            'admissions_jour': {
+                'numeric': ['nb_admissions', 'intensity'],
+                'date': ['date'],
+                'categorical': ['service_code', 'scenario']
+            },
+            'stock_materiel_jour': {
+                'numeric': ['stock_total', 'stock_disponible', 'consommation_estimee_jour', 
+                           'tension_stock'],
+                'date': ['date'],
+                'categorical': ['service_code', 'materiel_code', 'alerte_stock'],
+                'preserve_as_is': ['dq_is_anomaly', 'dq_anomaly_type', 'dq_anomaly_detail']
+            },
+            'scenarios_calendrier': {
+                'numeric': ['intensity'],
+                'date': ['date'],
+                'categorical': ['scenario']
+            }
+        }
+        
+        self.REFERENCE_TABLES = [
+            'service_ref', 'pathologie_ref', 'materiel_ref', 
+            'mode_entree_ref', 'type_lit_ref', 'calendrier_conges', 'hypotheses'
+        ]
+
+    def _clean_dataframe(self, name, df_raw):
+        """Transform: Applique le schéma et nettoie les types"""
+        df = df_raw.copy()
+        name_normalized = name.lower().strip().replace(' ', '_').replace('-', '_')
+        
+        if name_normalized in self.REFERENCE_TABLES:
+            return df
+        
+        config = self.SCHEMA.get(name_normalized)
+        if not config:
+            return self._generic_cleaning(df)
+        
+        # 1. DATES
+        for col in config.get('date', []):
+            if col in df.columns:
+                df[col] = pd.to_datetime(df[col], errors='coerce')
+        
+        # 2. NUMÉRIQUES
+        for col in config.get('numeric', []):
+            if col in df.columns:
+                if df[col].dtype == 'object':
+                    df[col] = df[col].astype(str).str.replace(',', '.').str.replace(' ', '')
+                    df[col] = df[col].replace(['Full', 'Erreur', 'full', 'erreur', ''], np.nan)
+                df[col] = pd.to_numeric(df[col], errors='coerce')
+        
+        # 3. CATÉGORIELLES
+        for col in config.get('categorical', []):
+            if col in df.columns and df[col].dtype == 'object':
+                df[col] = df[col].astype(str).str.strip()
+                df[col] = df[col].replace(['nan', 'NaN', 'None', ''], np.nan)
+        
+        return df
+
+    def _generic_cleaning(self, df):
+        """Nettoyage fallback"""
+        df_clean = df.copy()
+        for col in df_clean.columns:
+            if 'date' in col.lower():
+                df_clean[col] = pd.to_datetime(df_clean[col], errors='coerce')
+        return df_clean
+
+    def run_etl(self):
+        """Pipeline complet avec visualisation"""
+        data_explorer = DataExplorer()
+        
+        try:
+            spreadsheet = self.client.open(self.sheet_name)
+            worksheets = spreadsheet.worksheets()
+            relevant_sheets = [ws for ws in worksheets if ws.title.lower() not in ['', 'sheet1']]
+            
+            progress_bar = st.progress(0, text="🔄 ETL Engine: Extraction...")
+            
+            for i, ws in enumerate(relevant_sheets):
+                # EXTRACT
+                raw_data = ws.get_all_values()
+                if len(raw_data) > 1:
+                    headers = raw_data[0]
+                    rows = raw_data[1:]
+                    df_raw = pd.DataFrame(rows, columns=headers)
+                    
+                    # Capture métriques AVANT
+                    metrics = {
+                        'rows_before': len(df_raw),
+                        'mem_before': df_raw.memory_usage(deep=True).sum() / 1024**2
+                    }
+                    
+                    # TRANSFORM
+                    std_name = ws.title.lower().strip().replace(' ', '_').replace('-', '_').split('.')[0]
+                    df_clean = self._clean_dataframe(std_name, df_raw)
+                    
+                    # LOAD
+                    data_explorer.add_dataframe(std_name, df_clean, metrics)
+                
+                progress_bar.progress((i + 1) / len(relevant_sheets))
+            
+            data_explorer.load_timestamp = datetime.now()
+            progress_bar.empty()
+            st.session_state.data_explorer = data_explorer
+            
+            # --- AFFICHAGE DU RAPPORT VISUEL ---
+            summary = data_explorer.get_summary()
+            st.success(f"✅ ETL Terminé : {summary['total_rows']:,} lignes traitées.")
+            
+            # APPEL DU DASHBOARD VISUEL
+            data_explorer.show_etl_report()
+            
+            return True, data_explorer
+            
+        except Exception as e:
+            st.error(f"❌ Erreur ETL : {str(e)}")
+            return False, str(e)