Spaces:
Sleeping
Sleeping
Update src/module/Data_Exploration.py
Browse files- src/module/Data_Exploration.py +274 -171
src/module/Data_Exploration.py
CHANGED
|
@@ -1,278 +1,381 @@
|
|
| 1 |
"""
|
| 2 |
-
MODULE: DATA EXPLORATION (ADVANCED EDA) -
|
| 3 |
-
=================================================
|
| 4 |
-
|
|
|
|
| 5 |
"""
|
|
|
|
| 6 |
import streamlit as st
|
| 7 |
import pandas as pd
|
| 8 |
import numpy as np
|
| 9 |
import plotly.express as px
|
| 10 |
import plotly.graph_objects as go
|
| 11 |
from plotly.subplots import make_subplots
|
|
|
|
| 12 |
|
| 13 |
-
#
|
| 14 |
try:
|
| 15 |
import statsmodels.api as sm
|
| 16 |
from statsmodels.tsa.seasonal import seasonal_decompose
|
| 17 |
from statsmodels.graphics.tsaplots import acf, pacf
|
| 18 |
STATS_AVAILABLE = True
|
| 19 |
-
except
|
| 20 |
STATS_AVAILABLE = False
|
| 21 |
|
| 22 |
-
# Import du Contrat
|
| 23 |
-
# AVANT : from Function.Validation_Contract import ETL_Engine
|
| 24 |
-
# APRÈS :
|
| 25 |
from module.Validation_Contract import ETL_Engine
|
| 26 |
|
| 27 |
-
# Remplacez la définition de la classe et la méthode load_data par ceci :
|
| 28 |
|
| 29 |
-
class AdvancedAnalyzer:
|
|
|
|
|
|
|
| 30 |
def __init__(self, gspread_client, sheet_name):
|
| 31 |
self.etl = ETL_Engine(gspread_client, sheet_name)
|
| 32 |
self.dataframes = {}
|
| 33 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 34 |
def load_data(self):
|
| 35 |
-
"""Récupère
|
| 36 |
-
#
|
| 37 |
if 'data_explorer' in st.session_state and hasattr(st.session_state.data_explorer, 'dataframes'):
|
| 38 |
self.dataframes = st.session_state.data_explorer.dataframes
|
| 39 |
return True
|
| 40 |
|
| 41 |
-
#
|
| 42 |
-
success, result = self.
|
| 43 |
if success:
|
| 44 |
-
self.dataframes = result.dataframes
|
| 45 |
return True
|
| 46 |
return False
|
| 47 |
|
| 48 |
-
# ---
|
| 49 |
def get_df(self, keyword):
|
| 50 |
-
"""
|
| 51 |
-
Cherche une table qui contient le mot-clé.
|
| 52 |
-
Ex: 'stock' trouvera 'stock_materiel_jour'
|
| 53 |
-
"""
|
| 54 |
if not self.dataframes:
|
| 55 |
return None
|
| 56 |
-
|
| 57 |
-
# 1. Essai exact
|
| 58 |
if keyword in self.dataframes:
|
| 59 |
return self.dataframes[keyword]
|
| 60 |
-
|
| 61 |
-
|
| 62 |
-
|
| 63 |
-
|
|
|
|
|
|
|
|
|
|
| 64 |
return df
|
| 65 |
-
|
| 66 |
return None
|
| 67 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 68 |
|
| 69 |
-
# ---
|
| 70 |
-
|
| 71 |
-
|
| 72 |
-
|
| 73 |
-
|
| 74 |
-
|
| 75 |
-
|
| 76 |
-
|
| 77 |
-
st.warning("Table introuvable pour missing matrix.")
|
| 78 |
-
|
| 79 |
return
|
| 80 |
-
|
| 81 |
-
|
| 82 |
-
|
| 83 |
-
|
| 84 |
-
|
| 85 |
-
|
| 86 |
-
|
| 87 |
-
st.warning("Beaucoup de colonnes (>40) — afficher un résumé des top colonnes manquantes.")
|
| 88 |
-
|
| 89 |
-
col_missing = (miss.sum().sort_values(ascending=False) / len(df) * 100).head(40)
|
| 90 |
-
df_plot = pd.DataFrame({'col': col_missing.index, 'pct_missing': col_missing.values})
|
| 91 |
-
fig = px.bar(df_plot, x='col', y='pct_missing', title=f"Colonnes les plus manquantes ({table_keyword or 'table'})", text='pct_missing')
|
| 92 |
-
fig.update_traces(texttemplate='%{text:.1f}%', textposition='outside')
|
| 93 |
fig.update_layout(xaxis_tickangle=-45)
|
| 94 |
st.plotly_chart(fig, use_container_width=True)
|
| 95 |
|
| 96 |
-
|
| 97 |
-
|
| 98 |
-
|
| 99 |
-
def plot_numeric_summary(self, table_keyword, top_n=8):
|
| 100 |
-
df = self.get_df(table_keyword)
|
| 101 |
-
if df is None: return
|
| 102 |
num_cols = df.select_dtypes(include='number').columns.tolist()
|
| 103 |
if not num_cols:
|
| 104 |
-
st.info(
|
| 105 |
return
|
| 106 |
num_cols = num_cols[:top_n]
|
| 107 |
for col in num_cols:
|
| 108 |
-
c1, c2 = st.columns([
|
| 109 |
with c1:
|
| 110 |
-
fig = px.histogram(df, x=col, nbins=60,
|
| 111 |
st.plotly_chart(fig, use_container_width=True)
|
| 112 |
with c2:
|
| 113 |
st.write(df[col].describe().to_frame().T)
|
| 114 |
|
| 115 |
-
|
| 116 |
-
|
| 117 |
-
|
| 118 |
-
def plot_correlation(self, table_keyword):
|
| 119 |
-
df = self.get_df(table_keyword)
|
| 120 |
-
if df is None: return
|
| 121 |
-
num = df.select_dtypes(include='number').dropna(axis=1, how='all')
|
| 122 |
if num.shape[1] < 2:
|
| 123 |
-
st.info(
|
| 124 |
return
|
| 125 |
corr = num.corr()
|
| 126 |
-
fig = px.imshow(corr, text_auto=
|
| 127 |
st.plotly_chart(fig, use_container_width=True)
|
| 128 |
|
| 129 |
-
|
| 130 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 131 |
def referential_check(self, left_table, right_table, key='patient_id'):
|
| 132 |
-
left = self.get_df(left_table)
|
|
|
|
| 133 |
if left is None or right is None:
|
| 134 |
-
st.warning(
|
|
|
|
|
|
|
|
|
|
| 135 |
return
|
| 136 |
left_keys = set(left[key].dropna().unique())
|
| 137 |
right_keys = set(right[key].dropna().unique())
|
| 138 |
in_left_not_right = len(left_keys - right_keys)
|
| 139 |
in_right_not_left = len(right_keys - left_keys)
|
| 140 |
-
st.write(f
|
| 141 |
-
st.write(f
|
| 142 |
-
st.metric(
|
| 143 |
-
st.metric(
|
| 144 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 145 |
|
| 146 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 147 |
|
|
|
|
| 148 |
def tab_time_series_advanced(self):
|
| 149 |
-
st.markdown(
|
| 150 |
-
|
| 151 |
-
# UTILISATION DU SMART GETTER
|
| 152 |
df_adm = self.get_df('admissions')
|
| 153 |
-
|
| 154 |
-
|
| 155 |
-
st.warning("⚠️ Table 'admissions' introuvable.")
|
| 156 |
return
|
| 157 |
-
|
| 158 |
if not STATS_AVAILABLE:
|
| 159 |
-
st.error("Module 'statsmodels' manquant. Installez-le pour
|
| 160 |
return
|
| 161 |
-
|
| 162 |
-
# Préparation TS : Aggrégation par jour
|
| 163 |
try:
|
| 164 |
-
df_ts = df_adm.
|
| 165 |
-
|
| 166 |
-
if
|
| 167 |
-
st.warning(
|
| 168 |
return
|
| 169 |
-
|
| 170 |
-
|
| 171 |
-
|
| 172 |
-
|
|
|
|
| 173 |
fig = make_subplots(rows=4, cols=1, shared_xaxes=True,
|
| 174 |
subplot_titles=("Observé", "Tendance", "Saisonnalité", "Résidus"))
|
| 175 |
-
|
| 176 |
-
fig.add_trace(go.Scatter(x=
|
| 177 |
-
fig.add_trace(go.Scatter(x=
|
| 178 |
-
fig.add_trace(go.Scatter(x=
|
| 179 |
-
fig.
|
| 180 |
-
|
| 181 |
-
fig.update_layout(height=800, title_text="Décomposition STL des Admissions")
|
| 182 |
st.plotly_chart(fig, use_container_width=True)
|
| 183 |
-
|
| 184 |
except Exception as e:
|
| 185 |
-
st.error(f
|
| 186 |
|
| 187 |
def tab_stationarity(self):
|
| 188 |
-
st.markdown(
|
| 189 |
-
|
| 190 |
df_adm = self.get_df('admissions')
|
| 191 |
-
if df_adm is None:
|
| 192 |
-
|
| 193 |
if not STATS_AVAILABLE:
|
| 194 |
-
st.warning(
|
| 195 |
return
|
| 196 |
-
|
| 197 |
-
df_ts = df_adm.groupby('date')['nb_admissions'].sum().fillna(0)
|
| 198 |
-
|
| 199 |
try:
|
| 200 |
-
|
| 201 |
-
|
| 202 |
-
|
| 203 |
-
|
| 204 |
col1, col2 = st.columns(2)
|
| 205 |
with col1:
|
| 206 |
fig_acf = go.Figure()
|
| 207 |
-
fig_acf.add_trace(go.Bar(x=list(range(len(acf_vals))), y=acf_vals, name='ACF'
|
| 208 |
-
fig_acf.update_layout(title=
|
| 209 |
st.plotly_chart(fig_acf, use_container_width=True)
|
| 210 |
-
|
| 211 |
with col2:
|
| 212 |
fig_pacf = go.Figure()
|
| 213 |
-
fig_pacf.add_trace(go.Bar(x=list(range(len(pacf_vals))), y=pacf_vals, name='PACF'
|
| 214 |
-
fig_pacf.update_layout(title=
|
| 215 |
st.plotly_chart(fig_pacf, use_container_width=True)
|
| 216 |
-
|
| 217 |
except Exception as e:
|
| 218 |
-
st.error(f
|
| 219 |
|
| 220 |
def tab_heatmap_flux(self):
|
| 221 |
-
st.markdown(
|
| 222 |
-
|
| 223 |
df_adm = self.get_df('admissions')
|
| 224 |
-
if df_adm is None:
|
| 225 |
-
|
| 226 |
try:
|
| 227 |
-
|
| 228 |
-
|
| 229 |
-
|
| 230 |
-
|
| 231 |
-
|
| 232 |
-
|
| 233 |
-
|
| 234 |
-
|
| 235 |
-
fig = px.imshow(heatmap_data,
|
| 236 |
-
labels=dict(x="Mois", y="Jour", color="Moyenne Adm."),
|
| 237 |
-
color_continuous_scale="Viridis",
|
| 238 |
-
title="Intensité Hebdo vs Mensuelle")
|
| 239 |
st.plotly_chart(fig, use_container_width=True)
|
| 240 |
except Exception as e:
|
| 241 |
-
st.error(f
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 242 |
|
| 243 |
-
#
|
| 244 |
|
| 245 |
-
def show_terminal(client, sheet_name):
|
| 246 |
-
"""
|
| 247 |
-
|
| 248 |
-
|
| 249 |
if 'advanced_analyzer' not in st.session_state:
|
| 250 |
st.session_state.advanced_analyzer = AdvancedAnalyzer(client, sheet_name)
|
| 251 |
-
|
| 252 |
-
|
| 253 |
-
|
| 254 |
-
|
| 255 |
-
|
| 256 |
-
|
| 257 |
-
|
| 258 |
-
|
| 259 |
-
|
| 260 |
-
if success:
|
| 261 |
-
st.rerun()
|
| 262 |
-
else:
|
| 263 |
-
st.error("Échec du chargement. Lancez d'abord 'ETL Control'.")
|
| 264 |
return
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 265 |
|
| 266 |
-
|
| 267 |
-
tabs = st.tabs(["📈 STL Décomposition", "📊 Stationnarité (ACF)", "🔥 Heatmap Flux", "📋 Données Propres"])
|
| 268 |
-
|
| 269 |
-
with tabs[0]: explorer.tab_time_series_advanced()
|
| 270 |
-
with tabs[1]: explorer.tab_stationarity()
|
| 271 |
-
with tabs[2]: explorer.tab_heatmap_flux()
|
| 272 |
-
|
| 273 |
-
# Onglet Debug
|
| 274 |
-
with tabs[3]:
|
| 275 |
-
st.info(f"Tables disponibles : {list(explorer.dataframes.keys())}")
|
| 276 |
-
if explorer.dataframes:
|
| 277 |
-
table_choice = st.selectbox("Choisir une table", list(explorer.dataframes.keys()))
|
| 278 |
-
st.dataframe(explorer.dataframes[table_choice], use_container_width=True)
|
|
|
|
| 1 |
"""
|
| 2 |
+
MODULE: DATA EXPLORATION (ADVANCED EDA) - FUSION COMPLÈTE
|
| 3 |
+
=========================================================
|
| 4 |
+
Version complète : fusionne ton EDA existant + Quality Gates + Caching + Reporting
|
| 5 |
+
Compatible avec : Validation_Contract.py, Indicateurs_DQ.py, st.session_state
|
| 6 |
"""
|
| 7 |
+
|
| 8 |
import streamlit as st
|
| 9 |
import pandas as pd
|
| 10 |
import numpy as np
|
| 11 |
import plotly.express as px
|
| 12 |
import plotly.graph_objects as go
|
| 13 |
from plotly.subplots import make_subplots
|
| 14 |
+
from io import StringIO
|
| 15 |
|
| 16 |
+
# Stats avancées (optionnelles)
|
| 17 |
try:
|
| 18 |
import statsmodels.api as sm
|
| 19 |
from statsmodels.tsa.seasonal import seasonal_decompose
|
| 20 |
from statsmodels.graphics.tsaplots import acf, pacf
|
| 21 |
STATS_AVAILABLE = True
|
| 22 |
+
except Exception:
|
| 23 |
STATS_AVAILABLE = False
|
| 24 |
|
|
|
|
|
|
|
|
|
|
| 25 |
from module.Validation_Contract import ETL_Engine
|
| 26 |
|
|
|
|
| 27 |
|
| 28 |
+
class AdvancedAnalyzer:
|
| 29 |
+
"""EDA avancée : Smart Getter, Quality Gates, Visuals, TS, Export"""
|
| 30 |
+
|
| 31 |
def __init__(self, gspread_client, sheet_name):
|
| 32 |
self.etl = ETL_Engine(gspread_client, sheet_name)
|
| 33 |
self.dataframes = {}
|
| 34 |
|
| 35 |
+
# ------------------ A. CACHING / LOAD ------------------
|
| 36 |
+
@st.cache_data(show_spinner=False)
|
| 37 |
+
def _cached_run_etl(self):
|
| 38 |
+
return self.etl.run_etl()
|
| 39 |
+
|
| 40 |
def load_data(self):
|
| 41 |
+
"""Récupère DataExplorer depuis session ou lance l'ETL (cached)"""
|
| 42 |
+
# Priorité : session global
|
| 43 |
if 'data_explorer' in st.session_state and hasattr(st.session_state.data_explorer, 'dataframes'):
|
| 44 |
self.dataframes = st.session_state.data_explorer.dataframes
|
| 45 |
return True
|
| 46 |
|
| 47 |
+
# Sinon, lancer ETL (cached)
|
| 48 |
+
success, result = self._cached_run_etl()
|
| 49 |
if success:
|
| 50 |
+
self.dataframes = result.dataframes
|
| 51 |
return True
|
| 52 |
return False
|
| 53 |
|
| 54 |
+
# ------------------ SMART GETTER ------------------
|
| 55 |
def get_df(self, keyword):
|
| 56 |
+
"""Trouve une table par mot-clé (exact / partiel)."""
|
|
|
|
|
|
|
|
|
|
| 57 |
if not self.dataframes:
|
| 58 |
return None
|
| 59 |
+
# exact
|
|
|
|
| 60 |
if keyword in self.dataframes:
|
| 61 |
return self.dataframes[keyword]
|
| 62 |
+
# smart partial match
|
| 63 |
+
for k, df in self.dataframes.items():
|
| 64 |
+
if keyword in k:
|
| 65 |
+
return df
|
| 66 |
+
# case-insensitive partial
|
| 67 |
+
for k, df in self.dataframes.items():
|
| 68 |
+
if keyword.lower() in k.lower():
|
| 69 |
return df
|
|
|
|
| 70 |
return None
|
| 71 |
|
| 72 |
+
# ------------------ B. QUALITY GATES & REPORT ------------------
|
| 73 |
+
def quality_report(self, df, keys_to_check=None):
|
| 74 |
+
"""Génère un rapport qualité sommaire et un score (0-100)."""
|
| 75 |
+
report = {}
|
| 76 |
+
report['rows'] = len(df)
|
| 77 |
+
# moyenne du pourcentage manquant par colonne
|
| 78 |
+
report['missing_pct_mean'] = float(df.isnull().mean().mean() * 100)
|
| 79 |
+
# top missing columns
|
| 80 |
+
top_missing = (df.isnull().mean() * 100).sort_values(ascending=False).head(10)
|
| 81 |
+
report['top_missing'] = top_missing.to_dict()
|
| 82 |
+
report['duplicates'] = int(df.duplicated().sum())
|
| 83 |
+
report['unique_rows'] = int(df.drop_duplicates().shape[0])
|
| 84 |
+
report['numeric_cols'] = len(df.select_dtypes(include='number').columns)
|
| 85 |
+
# referential nulls check
|
| 86 |
+
if keys_to_check:
|
| 87 |
+
for k in keys_to_check:
|
| 88 |
+
if k in df.columns:
|
| 89 |
+
report[f'{k}_nulls'] = int(df[k].isnull().sum())
|
| 90 |
+
# Score simple
|
| 91 |
+
score = 100 - report['missing_pct_mean'] - min(report['duplicates'] * 0.02, 20)
|
| 92 |
+
report['quality_score'] = round(max(0, score), 2)
|
| 93 |
+
return report
|
| 94 |
|
| 95 |
+
# ------------------ C. EDA VISUALS ------------------
|
| 96 |
+
def plot_missing_matrix(self, df, name):
|
| 97 |
+
"""Affiche bar chart des colonnes les plus manquantes.
|
| 98 |
+
Si >40 colonnes : affiche message et top 40 seulement."""
|
| 99 |
+
miss = (df.isnull().mean() * 100).sort_values(ascending=False)
|
| 100 |
+
if miss.empty or miss.max() == 0:
|
| 101 |
+
st.success('✅ Aucune valeur manquante détectée')
|
|
|
|
|
|
|
|
|
|
| 102 |
return
|
| 103 |
+
if len(miss) > 40:
|
| 104 |
+
st.warning('Beaucoup de colonnes détectées — affichage des top 40 colonnes manquantes')
|
| 105 |
+
miss = miss.head(40)
|
| 106 |
+
fig = px.bar(x=miss.index, y=miss.values, labels={'x': 'Colonne', 'y': '% manquante'},
|
| 107 |
+
title=f'Colonnes manquantes (%) — {name}')
|
| 108 |
+
fig.update_traces(texttemplate='%{y:.1f}%', textposition='outside')
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 109 |
fig.update_layout(xaxis_tickangle=-45)
|
| 110 |
st.plotly_chart(fig, use_container_width=True)
|
| 111 |
|
| 112 |
+
def plot_numeric_summary(self, df, top_n=8):
|
| 113 |
+
"""Histogram + box + descriptive stats pour les colonnes numériques (top_n)."""
|
|
|
|
|
|
|
|
|
|
|
|
|
| 114 |
num_cols = df.select_dtypes(include='number').columns.tolist()
|
| 115 |
if not num_cols:
|
| 116 |
+
st.info('Aucune colonne numérique pour cette table.')
|
| 117 |
return
|
| 118 |
num_cols = num_cols[:top_n]
|
| 119 |
for col in num_cols:
|
| 120 |
+
c1, c2 = st.columns([3, 1])
|
| 121 |
with c1:
|
| 122 |
+
fig = px.histogram(df, x=col, nbins=60, title=f'Distribution — {col}', marginal='box')
|
| 123 |
st.plotly_chart(fig, use_container_width=True)
|
| 124 |
with c2:
|
| 125 |
st.write(df[col].describe().to_frame().T)
|
| 126 |
|
| 127 |
+
def plot_correlation(self, df, name):
|
| 128 |
+
num = df.select_dtypes(include='number')
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 129 |
if num.shape[1] < 2:
|
| 130 |
+
st.info('Pas assez de colonnes numériques pour une heatmap de corrélation')
|
| 131 |
return
|
| 132 |
corr = num.corr()
|
| 133 |
+
fig = px.imshow(corr, text_auto='.2f', title=f'Corrélations — {name}')
|
| 134 |
st.plotly_chart(fig, use_container_width=True)
|
| 135 |
|
| 136 |
+
def plot_outliers_iqr(self, df, name, max_plots=6):
|
| 137 |
+
"""Détecte outliers via IQR et affiche boxplots des top numeric cols."""
|
| 138 |
+
num = df.select_dtypes(include='number')
|
| 139 |
+
if num.shape[1] == 0:
|
| 140 |
+
st.info('Pas de colonnes numériques pour détection outliers.')
|
| 141 |
+
return
|
| 142 |
+
cols = num.columns.tolist()[:max_plots]
|
| 143 |
+
for col in cols:
|
| 144 |
+
q1 = df[col].quantile(0.25)
|
| 145 |
+
q3 = df[col].quantile(0.75)
|
| 146 |
+
iqr = q3 - q1
|
| 147 |
+
low = q1 - 1.5 * iqr
|
| 148 |
+
high = q3 + 1.5 * iqr
|
| 149 |
+
num_out = ((df[col] < low) | (df[col] > high)).sum()
|
| 150 |
+
st.markdown(f'**{col}** — outliers IQR: {int(num_out):,}')
|
| 151 |
+
fig = px.box(df, y=col, title=f'Outliers (IQR) — {col}')
|
| 152 |
+
st.plotly_chart(fig, use_container_width=True)
|
| 153 |
+
|
| 154 |
+
# ------------------ Referential / Duplicates / Top values ------------------
|
| 155 |
def referential_check(self, left_table, right_table, key='patient_id'):
|
| 156 |
+
left = self.get_df(left_table)
|
| 157 |
+
right = self.get_df(right_table)
|
| 158 |
if left is None or right is None:
|
| 159 |
+
st.warning('Tables introuvables pour vérification référentielle')
|
| 160 |
+
return
|
| 161 |
+
if key not in left.columns or key not in right.columns:
|
| 162 |
+
st.warning(f'Clé {key} absente dans l une des tables')
|
| 163 |
return
|
| 164 |
left_keys = set(left[key].dropna().unique())
|
| 165 |
right_keys = set(right[key].dropna().unique())
|
| 166 |
in_left_not_right = len(left_keys - right_keys)
|
| 167 |
in_right_not_left = len(right_keys - left_keys)
|
| 168 |
+
st.write(f'- Clés {key} dans {left_table}: {len(left_keys):,}')
|
| 169 |
+
st.write(f'- Clés {key} dans {right_table}: {len(right_keys):,}')
|
| 170 |
+
st.metric('Dans left mais pas right', f'{in_left_not_right:,}')
|
| 171 |
+
st.metric('Dans right mais pas left', f'{in_right_not_left:,}')
|
| 172 |
|
| 173 |
+
def duplicates_and_top_values(self, table_keyword, cat_top=5):
|
| 174 |
+
df = self.get_df(table_keyword)
|
| 175 |
+
if df is None:
|
| 176 |
+
st.warning('Table introuvable')
|
| 177 |
+
return
|
| 178 |
+
st.markdown('### 🔎 Doublons et top valeurs')
|
| 179 |
+
full_dups = df.duplicated().sum()
|
| 180 |
+
st.write(f'Doublons lignes exactes: {int(full_dups):,}')
|
| 181 |
+
if 'patient_id' in df.columns:
|
| 182 |
+
pid_dups = df['patient_id'].duplicated().sum()
|
| 183 |
+
st.write(f'Doublons patient_id: {int(pid_dups):,}')
|
| 184 |
|
| 185 |
+
# Top valeurs pour catégories
|
| 186 |
+
cats = df.select_dtypes(include=['object', 'category']).columns.tolist()[:cat_top]
|
| 187 |
+
for col in cats:
|
| 188 |
+
st.write(f'**{col}**')
|
| 189 |
+
st.dataframe(df[col].value_counts().head(10).to_frame(name='count'))
|
| 190 |
|
| 191 |
+
# ------------------ TIME SERIES ADVANCED ------------------
|
| 192 |
def tab_time_series_advanced(self):
|
| 193 |
+
st.markdown('### 📈 Analyse Avancée des Séries Temporelles (STL)')
|
|
|
|
|
|
|
| 194 |
df_adm = self.get_df('admissions')
|
| 195 |
+
if df_adm is None:
|
| 196 |
+
st.warning("Table 'admissions' introuvable.")
|
|
|
|
| 197 |
return
|
|
|
|
| 198 |
if not STATS_AVAILABLE:
|
| 199 |
+
st.error("Module 'statsmodels' manquant. Installez-le pour la décomposition STL.")
|
| 200 |
return
|
|
|
|
|
|
|
| 201 |
try:
|
| 202 |
+
df_ts = df_adm.set_index('date').sort_index()
|
| 203 |
+
# ensure numeric target exists
|
| 204 |
+
if 'nb_admissions' not in df_ts.columns:
|
| 205 |
+
st.warning('Colonne nb_admissions manquante')
|
| 206 |
return
|
| 207 |
+
series = df_ts['nb_admissions'].resample('D').sum().fillna(0)
|
| 208 |
+
if len(series) < 30:
|
| 209 |
+
st.warning('Pas assez de points pour STL (min 30)')
|
| 210 |
+
return
|
| 211 |
+
decomposition = seasonal_decompose(series, model='additive', period=7)
|
| 212 |
fig = make_subplots(rows=4, cols=1, shared_xaxes=True,
|
| 213 |
subplot_titles=("Observé", "Tendance", "Saisonnalité", "Résidus"))
|
| 214 |
+
fig.add_trace(go.Scatter(x=series.index, y=decomposition.observed, name='Observé'), row=1, col=1)
|
| 215 |
+
fig.add_trace(go.Scatter(x=series.index, y=decomposition.trend, name='Tendance'), row=2, col=1)
|
| 216 |
+
fig.add_trace(go.Scatter(x=series.index, y=decomposition.seasonal, name='Saisonnalité'), row=3, col=1)
|
| 217 |
+
fig.add_trace(go.Scatter(x=series.index, y=decomposition.resid, name='Résidus', mode='markers'), row=4, col=1)
|
| 218 |
+
fig.update_layout(height=800, title_text='Décomposition STL des Admissions')
|
|
|
|
|
|
|
| 219 |
st.plotly_chart(fig, use_container_width=True)
|
|
|
|
| 220 |
except Exception as e:
|
| 221 |
+
st.error(f'Erreur calcul STL : {e}')
|
| 222 |
|
| 223 |
def tab_stationarity(self):
|
| 224 |
+
st.markdown('### 📊 Stationnarité (ACF / PACF)')
|
|
|
|
| 225 |
df_adm = self.get_df('admissions')
|
| 226 |
+
if df_adm is None:
|
| 227 |
+
return
|
| 228 |
if not STATS_AVAILABLE:
|
| 229 |
+
st.warning('Statsmodels requis pour ACF/PACF')
|
| 230 |
return
|
|
|
|
|
|
|
|
|
|
| 231 |
try:
|
| 232 |
+
series = df_adm.set_index('date')['nb_admissions'].resample('D').sum().fillna(0)
|
| 233 |
+
lag_count = min(60, max(20, int(len(series) / 2)))
|
| 234 |
+
acf_vals = acf(series, nlags=lag_count)
|
| 235 |
+
pacf_vals = pacf(series, nlags=lag_count)
|
| 236 |
col1, col2 = st.columns(2)
|
| 237 |
with col1:
|
| 238 |
fig_acf = go.Figure()
|
| 239 |
+
fig_acf.add_trace(go.Bar(x=list(range(len(acf_vals))), y=acf_vals, name='ACF'))
|
| 240 |
+
fig_acf.update_layout(title='Autocorrélation (ACF)', xaxis_title='Lag')
|
| 241 |
st.plotly_chart(fig_acf, use_container_width=True)
|
|
|
|
| 242 |
with col2:
|
| 243 |
fig_pacf = go.Figure()
|
| 244 |
+
fig_pacf.add_trace(go.Bar(x=list(range(len(pacf_vals))), y=pacf_vals, name='PACF'))
|
| 245 |
+
fig_pacf.update_layout(title='Autocorrélation partielle (PACF)', xaxis_title='Lag')
|
| 246 |
st.plotly_chart(fig_pacf, use_container_width=True)
|
|
|
|
| 247 |
except Exception as e:
|
| 248 |
+
st.error(f'Erreur ACF/PACF : {e}')
|
| 249 |
|
| 250 |
def tab_heatmap_flux(self):
|
| 251 |
+
st.markdown('### 🔥 Heatmap des Flux d\'Entrée')
|
|
|
|
| 252 |
df_adm = self.get_df('admissions')
|
| 253 |
+
if df_adm is None:
|
| 254 |
+
return
|
| 255 |
try:
|
| 256 |
+
tmp = df_adm.copy()
|
| 257 |
+
tmp['day_name'] = tmp['date'].dt.day_name()
|
| 258 |
+
tmp['month'] = tmp['date'].dt.month_name()
|
| 259 |
+
heat = tmp.pivot_table(index='day_name', columns='month', values='nb_admissions', aggfunc='mean')
|
| 260 |
+
days = ['Monday','Tuesday','Wednesday','Thursday','Friday','Saturday','Sunday']
|
| 261 |
+
# try both cases for month ordering
|
| 262 |
+
heat = heat.reindex(days)
|
| 263 |
+
fig = px.imshow(heat, labels=dict(x='Mois', y='Jour', color='Moyenne Adm.'), title='Intensité Hebdo vs Mensuelle')
|
|
|
|
|
|
|
|
|
|
|
|
|
| 264 |
st.plotly_chart(fig, use_container_width=True)
|
| 265 |
except Exception as e:
|
| 266 |
+
st.error(f'Erreur Heatmap : {e}')
|
| 267 |
+
|
| 268 |
+
# ------------------ D. EXPORTS & REPORTS ------------------
|
| 269 |
+
def export_csv(self, df):
|
| 270 |
+
return df.to_csv(index=False)
|
| 271 |
+
|
| 272 |
+
def export_quality_report(self, report):
|
| 273 |
+
# report est dict ; on transforme en csv simple
|
| 274 |
+
rows = []
|
| 275 |
+
for k, v in report.items():
|
| 276 |
+
if isinstance(v, dict):
|
| 277 |
+
rows.append({ 'metric': k, 'value': str(v) })
|
| 278 |
+
else:
|
| 279 |
+
rows.append({ 'metric': k, 'value': v })
|
| 280 |
+
df = pd.DataFrame(rows)
|
| 281 |
+
return df.to_csv(index=False)
|
| 282 |
+
|
| 283 |
|
| 284 |
+
# ------------------ UI ENTRYPOINT ------------------
|
| 285 |
|
| 286 |
+
def show_terminal(client, sheet_name, gates_blocking=False):
|
| 287 |
+
"""Affiche l'UI EDA complète en onglets.
|
| 288 |
+
gates_blocking: si True, bloque certaines vues si quality_score < threshold
|
| 289 |
+
"""
|
| 290 |
if 'advanced_analyzer' not in st.session_state:
|
| 291 |
st.session_state.advanced_analyzer = AdvancedAnalyzer(client, sheet_name)
|
| 292 |
+
|
| 293 |
+
analyzer = st.session_state.advanced_analyzer
|
| 294 |
+
st.markdown('## 🔬 DATA LAB : ANALYSE AVANCÉE')
|
| 295 |
+
|
| 296 |
+
# LOAD
|
| 297 |
+
if not analyzer.dataframes:
|
| 298 |
+
with st.spinner('🔄 Liaison avec l ETL...'):
|
| 299 |
+
if not analyzer.load_data():
|
| 300 |
+
st.error('❌ Échec ETL — vérifiez Validation_Contract')
|
|
|
|
|
|
|
|
|
|
|
|
|
| 301 |
return
|
| 302 |
+
st.rerun()
|
| 303 |
+
|
| 304 |
+
# Tabs
|
| 305 |
+
tabs = st.tabs(["📈 STL Décomposition", "📊 Stationnarité (ACF)", "🔥 Heatmap Flux", "📋 Table & DQ", "🧾 Export & Rapport"])
|
| 306 |
+
|
| 307 |
+
# Tab 0: STL
|
| 308 |
+
with tabs[0]:
|
| 309 |
+
analyzer.tab_time_series_advanced()
|
| 310 |
+
|
| 311 |
+
# Tab 1: Stationnarité
|
| 312 |
+
with tabs[1]:
|
| 313 |
+
analyzer.tab_stationarity()
|
| 314 |
+
|
| 315 |
+
# Tab 2: Heatmap Flux
|
| 316 |
+
with tabs[2]:
|
| 317 |
+
analyzer.tab_heatmap_flux()
|
| 318 |
+
|
| 319 |
+
# Tab 3: Table & DQ
|
| 320 |
+
with tabs[3]:
|
| 321 |
+
st.markdown('### 📋 Exploration par table')
|
| 322 |
+
table_choice = st.selectbox('Choisir une table', list(analyzer.dataframes.keys()))
|
| 323 |
+
df = analyzer.dataframes[table_choice]
|
| 324 |
+
|
| 325 |
+
# Quality
|
| 326 |
+
q = analyzer.quality_report(df, keys_to_check=['patient_id'] if 'patient_id' in df.columns else None)
|
| 327 |
+
st.metric('Quality Score', f"{q['quality_score']:.1f} / 100")
|
| 328 |
+
st.json({k: v for k, v in q.items() if k not in ['top_missing']})
|
| 329 |
+
if q.get('top_missing'):
|
| 330 |
+
st.markdown('**Top colonnes manquantes :**')
|
| 331 |
+
st.write(pd.Series(q['top_missing']).to_frame('pct_missing'))
|
| 332 |
+
|
| 333 |
+
# Gates informative
|
| 334 |
+
if gates_blocking and q['quality_score'] < 50:
|
| 335 |
+
st.error('⚠️ Quality gates critiques — affichage limité')
|
| 336 |
+
# Visuals
|
| 337 |
+
analyzer.plot_missing_matrix(df, table_choice)
|
| 338 |
+
analyzer.plot_numeric_summary(df)
|
| 339 |
+
analyzer.plot_correlation(df, table_choice)
|
| 340 |
+
analyzer.plot_outliers_iqr(df, table_choice)
|
| 341 |
+
analyzer.duplicates_and_top_values(table_choice)
|
| 342 |
+
|
| 343 |
+
# Referential checks quick buttons
|
| 344 |
+
cols = list(analyzer.dataframes.keys())
|
| 345 |
+
if 'patients' in cols and 'sejours' in cols:
|
| 346 |
+
if st.button('Vérifier référentiel patients <-> sejours'):
|
| 347 |
+
analyzer.referential_check('patients', 'sejours', key='patient_id')
|
| 348 |
+
|
| 349 |
+
# Tab 4: Export & Report
|
| 350 |
+
with tabs[4]:
|
| 351 |
+
st.markdown('### 🧾 Export & Rapports')
|
| 352 |
+
table_choice = st.selectbox('Table à exporter', list(analyzer.dataframes.keys()), key='export_table')
|
| 353 |
+
df = analyzer.dataframes[table_choice]
|
| 354 |
+
csv = analyzer.export_csv(df)
|
| 355 |
+
st.download_button('📥 Télécharger CSV (table)', csv, f'{table_choice}.csv', 'text/csv')
|
| 356 |
+
|
| 357 |
+
# quality report
|
| 358 |
+
qrep = analyzer.quality_report(df)
|
| 359 |
+
qcsv = analyzer.export_quality_report(qrep)
|
| 360 |
+
st.download_button('📥 Télécharger rapport qualité', qcsv, f'{table_choice}_quality_report.csv', 'text/csv')
|
| 361 |
+
|
| 362 |
+
# small EDA snapshot
|
| 363 |
+
if st.button('Générer snapshot EDA (top columns)'):
|
| 364 |
+
# build a small summary
|
| 365 |
+
summary = {
|
| 366 |
+
'table': table_choice,
|
| 367 |
+
'rows': len(df),
|
| 368 |
+
'cols': df.shape[1],
|
| 369 |
+
'quality_score': qrep['quality_score']
|
| 370 |
+
}
|
| 371 |
+
st.json(summary)
|
| 372 |
+
|
| 373 |
+
# Debugging panel (always accessible)
|
| 374 |
+
st.markdown('---')
|
| 375 |
+
with st.expander('Debug : tables disponibles'):
|
| 376 |
+
st.write(list(analyzer.dataframes.keys()))
|
| 377 |
+
for k, v in analyzer.dataframes.items():
|
| 378 |
+
st.write(f"- {k}: {v.shape[0]:,} lignes, {v.shape[1]} colonnes")
|
| 379 |
+
|
| 380 |
|
| 381 |
+
# EOF
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|