Download scripts/train_nb10.py from xbizot/saearch: direct link, hf CLI and curl.
- Browser
- Download file 29.5 kB
-
https://huggingface.co/spaces/xbizot/saearch/resolve/main/scripts/train_nb10.py
- Command line
-
hf download hf://spaces/xbizot/saearch/scripts/train_nb10.py
-
curl -L -o train_nb10.py https://huggingface.co/spaces/xbizot/saearch/resolve/main/scripts/train_nb10.py
29.5 kB
| """ | |
| Script d'entrainement reproductible du modele ML NB10. | |
| Charge le dataset EM-DAT multi-type, construit les features (lags par pays x type | |
| + warming NASA + cross-type), entraine le meilleur modele de regression et | |
| classification, puis sauvegarde les artefacts dans outputs/. | |
| Appele par le CI/CD avant le build Docker pour generer les artefacts sans | |
| dependre du notebook. Equivalent au NB10 executable mais en format .py. | |
| Usage : | |
| python scripts/train_nb10.py | |
| """ | |
| import logging | |
| import os | |
| import platform | |
| import subprocess | |
| import sys | |
| from datetime import datetime | |
| from pathlib import Path | |
| import joblib | |
| import numpy as np | |
| import pandas as pd | |
| import sklearn | |
| from sklearn.ensemble import ( | |
| GradientBoostingClassifier, | |
| GradientBoostingRegressor, | |
| HistGradientBoostingClassifier, | |
| HistGradientBoostingRegressor, | |
| RandomForestClassifier, | |
| RandomForestRegressor, | |
| ) | |
| from sklearn.impute import SimpleImputer | |
| from sklearn.linear_model import LogisticRegression, Ridge | |
| from sklearn.metrics import ( | |
| accuracy_score, | |
| f1_score, | |
| mean_absolute_error, | |
| mean_squared_error, | |
| r2_score, | |
| ) | |
| from sklearn.pipeline import Pipeline | |
| from sklearn.preprocessing import StandardScaler | |
| from sklearn.tree import DecisionTreeClassifier, DecisionTreeRegressor | |
| from sklearn.utils.class_weight import compute_sample_weight | |
| try: | |
| import mlflow | |
| from mlflow.models.signature import infer_signature | |
| HAS_MLFLOW = True | |
| except ImportError: | |
| HAS_MLFLOW = False | |
| def _git_commit() -> str: | |
| """Retourne le SHA court du commit courant, ou 'unknown'.""" | |
| try: | |
| return subprocess.check_output( | |
| ["git", "rev-parse", "--short", "HEAD"], | |
| stderr=subprocess.DEVNULL, | |
| text=True, | |
| ).strip() | |
| except (subprocess.CalledProcessError, FileNotFoundError): | |
| return os.getenv("GITHUB_SHA", "unknown")[:7] | |
| def _git_branch() -> str: | |
| """Retourne la branche courante.""" | |
| try: | |
| return subprocess.check_output( | |
| ["git", "rev-parse", "--abbrev-ref", "HEAD"], | |
| stderr=subprocess.DEVNULL, | |
| text=True, | |
| ).strip() | |
| except (subprocess.CalledProcessError, FileNotFoundError): | |
| return os.getenv("GITHUB_REF_NAME", "unknown") | |
| logging.basicConfig( | |
| level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s" | |
| ) | |
| logger = logging.getLogger(__name__) | |
| SEED = 42 | |
| np.random.seed(SEED) | |
| BASE = Path(__file__).resolve().parent.parent | |
| DATA_PATH = BASE / "data" / "decadal-deaths-disasters-by-type.csv" | |
| OUTPUT_DIR = BASE / "outputs" | |
| OUTPUT_DIR.mkdir(exist_ok=True) | |
| CLIMATE_TYPES = [ | |
| "drought", | |
| "flood", | |
| "extreme_weather", | |
| "extreme_temperature", | |
| "wildfire", | |
| ] | |
| AGGREGATES = [ | |
| "World", | |
| "Asia", | |
| "Europe", | |
| "Africa", | |
| "North America", | |
| "South America", | |
| "Oceania", | |
| "USSR", | |
| "European Union (27)", | |
| "High-income countries", | |
| "Low-income countries", | |
| "Lower-middle-income countries", | |
| "Upper-middle-income countries", | |
| ] | |
| # Source : NASA GISS GISTEMP v4, baseline 1951-1980. Valeur 2030 = IPCC SSP2-4.5. | |
| WARMING_INDEX = { | |
| 1900: -0.17, | |
| 1910: -0.30, | |
| 1920: -0.20, | |
| 1930: -0.10, | |
| 1940: 0.04, | |
| 1950: -0.03, | |
| 1960: -0.02, | |
| 1970: 0.00, | |
| 1980: 0.20, | |
| 1990: 0.35, | |
| 2000: 0.59, | |
| 2010: 0.81, | |
| 2020: 1.00, | |
| 2030: 1.40, | |
| } | |
| RISK_NAMES = ["Aucun", "Faible", "Modere", "Eleve", "Critique"] | |
| RISK_MAP = {name: i for i, name in enumerate(RISK_NAMES)} | |
| FEATURES_REG = [ | |
| "decade_index", | |
| "continent_enc", | |
| "type_enc", | |
| "log_lag_1", | |
| "log_lag_2", | |
| "trend", | |
| "log_cumul_mean", | |
| "cumul_std", | |
| "cumul_max", | |
| "log_impact_other", | |
| "warming", | |
| "warming_sq", | |
| ] | |
| # Mapping continent (meme que NB10) | |
| CONTINENT_MAP = { | |
| "Afghanistan": "Asie", | |
| "Albania": "Europe", | |
| "Algeria": "Afrique", | |
| "Angola": "Afrique", | |
| "Argentina": "Amerique du Sud", | |
| "Armenia": "Asie", | |
| "Australia": "Oceanie", | |
| "Austria": "Europe", | |
| "Azerbaijan": "Asie", | |
| "Bangladesh": "Asie", | |
| "Belarus": "Europe", | |
| "Belgium": "Europe", | |
| "Benin": "Afrique", | |
| "Bhutan": "Asie", | |
| "Bolivia": "Amerique du Sud", | |
| "Bosnia and Herzegovina": "Europe", | |
| "Botswana": "Afrique", | |
| "Brazil": "Amerique du Sud", | |
| "Bulgaria": "Europe", | |
| "Burkina Faso": "Afrique", | |
| "Burundi": "Afrique", | |
| "Cambodia": "Asie", | |
| "Cameroon": "Afrique", | |
| "Canada": "Amerique du Nord", | |
| "Central African Republic": "Afrique", | |
| "Chad": "Afrique", | |
| "Chile": "Amerique du Sud", | |
| "China": "Asie", | |
| "Colombia": "Amerique du Sud", | |
| "Comoros": "Afrique", | |
| "Congo": "Afrique", | |
| "Costa Rica": "Amerique centrale", | |
| "Croatia": "Europe", | |
| "Cuba": "Caraibes", | |
| "Cyprus": "Europe", | |
| "Czechia": "Europe", | |
| "Denmark": "Europe", | |
| "Democratic Republic of Congo": "Afrique", | |
| "Djibouti": "Afrique", | |
| "Dominican Republic": "Caraibes", | |
| "East Timor": "Asie", | |
| "Ecuador": "Amerique du Sud", | |
| "Egypt": "Afrique", | |
| "El Salvador": "Amerique centrale", | |
| "Eritrea": "Afrique", | |
| "Estonia": "Europe", | |
| "Eswatini": "Afrique", | |
| "Ethiopia": "Afrique", | |
| "Fiji": "Oceanie", | |
| "Finland": "Europe", | |
| "France": "Europe", | |
| "Gabon": "Afrique", | |
| "Gambia": "Afrique", | |
| "Georgia": "Asie", | |
| "Germany": "Europe", | |
| "Ghana": "Afrique", | |
| "Greece": "Europe", | |
| "Guatemala": "Amerique centrale", | |
| "Guinea": "Afrique", | |
| "Guinea-Bissau": "Afrique", | |
| "Guyana": "Amerique du Sud", | |
| "Haiti": "Caraibes", | |
| "Honduras": "Amerique centrale", | |
| "Hungary": "Europe", | |
| "Iceland": "Europe", | |
| "India": "Asie", | |
| "Indonesia": "Asie", | |
| "Iran": "Asie", | |
| "Iraq": "Asie", | |
| "Ireland": "Europe", | |
| "Israel": "Asie", | |
| "Italy": "Europe", | |
| "Jamaica": "Caraibes", | |
| "Japan": "Asie", | |
| "Jordan": "Asie", | |
| "Kazakhstan": "Asie", | |
| "Kenya": "Afrique", | |
| "Kyrgyzstan": "Asie", | |
| "Laos": "Asie", | |
| "Latvia": "Europe", | |
| "Lebanon": "Asie", | |
| "Lesotho": "Afrique", | |
| "Liberia": "Afrique", | |
| "Libya": "Afrique", | |
| "Lithuania": "Europe", | |
| "Luxembourg": "Europe", | |
| "Madagascar": "Afrique", | |
| "Malawi": "Afrique", | |
| "Malaysia": "Asie", | |
| "Mali": "Afrique", | |
| "Mauritania": "Afrique", | |
| "Mauritius": "Afrique", | |
| "Mexico": "Amerique du Nord", | |
| "Moldova": "Europe", | |
| "Mongolia": "Asie", | |
| "Montenegro": "Europe", | |
| "Morocco": "Afrique", | |
| "Mozambique": "Afrique", | |
| "Myanmar": "Asie", | |
| "Namibia": "Afrique", | |
| "Nepal": "Asie", | |
| "Netherlands": "Europe", | |
| "New Zealand": "Oceanie", | |
| "Nicaragua": "Amerique centrale", | |
| "Niger": "Afrique", | |
| "Nigeria": "Afrique", | |
| "North Korea": "Asie", | |
| "North Macedonia": "Europe", | |
| "Norway": "Europe", | |
| "Oman": "Asie", | |
| "Pakistan": "Asie", | |
| "Palestine": "Asie", | |
| "Panama": "Amerique centrale", | |
| "Papua New Guinea": "Oceanie", | |
| "Paraguay": "Amerique du Sud", | |
| "Peru": "Amerique du Sud", | |
| "Philippines": "Asie", | |
| "Poland": "Europe", | |
| "Portugal": "Europe", | |
| "Puerto Rico": "Caraibes", | |
| "Romania": "Europe", | |
| "Russia": "Europe", | |
| "Rwanda": "Afrique", | |
| "Saudi Arabia": "Asie", | |
| "Senegal": "Afrique", | |
| "Serbia": "Europe", | |
| "Sierra Leone": "Afrique", | |
| "Slovakia": "Europe", | |
| "Slovenia": "Europe", | |
| "Somalia": "Afrique", | |
| "South Africa": "Afrique", | |
| "South Korea": "Asie", | |
| "South Sudan": "Afrique", | |
| "Spain": "Europe", | |
| "Sri Lanka": "Asie", | |
| "Sudan": "Afrique", | |
| "Suriname": "Amerique du Sud", | |
| "Sweden": "Europe", | |
| "Switzerland": "Europe", | |
| "Syria": "Asie", | |
| "Taiwan": "Asie", | |
| "Tajikistan": "Asie", | |
| "Tanzania": "Afrique", | |
| "Thailand": "Asie", | |
| "Togo": "Afrique", | |
| "Trinidad and Tobago": "Caraibes", | |
| "Tunisia": "Afrique", | |
| "Turkey": "Asie", | |
| "Turkmenistan": "Asie", | |
| "Uganda": "Afrique", | |
| "Ukraine": "Europe", | |
| "United Arab Emirates": "Asie", | |
| "United Kingdom": "Europe", | |
| "United States": "Amerique du Nord", | |
| "Uruguay": "Amerique du Sud", | |
| "Uzbekistan": "Asie", | |
| "Venezuela": "Amerique du Sud", | |
| "Vietnam": "Asie", | |
| "Yemen": "Asie", | |
| "Zambia": "Afrique", | |
| "Zimbabwe": "Afrique", | |
| } | |
| CONTINENT_NAMES = sorted(set(CONTINENT_MAP.values())) + ["Autre"] | |
| CONTINENT_ENC = {name: i for i, name in enumerate(CONTINENT_NAMES)} | |
| TYPE_ENC = {t: i for i, t in enumerate(CLIMATE_TYPES)} | |
| def categoriser_risque(val: float) -> str: | |
| """Categorise l'impact en niveau de risque ordinal.""" | |
| if val == 0: | |
| return "Aucun" | |
| if val < 100: | |
| return "Faible" | |
| if val < 1000: | |
| return "Modere" | |
| if val < 10000: | |
| return "Eleve" | |
| return "Critique" | |
| def charger_dataset() -> pd.DataFrame: | |
| """Charge le CSV EM-DAT et le pivote en format long (pays x type x annee).""" | |
| logger.info("Chargement du dataset : %s", DATA_PATH) | |
| if not DATA_PATH.exists(): | |
| raise FileNotFoundError(f"Dataset introuvable : {DATA_PATH}") | |
| df_raw = pd.read_csv(DATA_PATH) | |
| df_wide = df_raw[~df_raw["entity"].isin(AGGREGATES)].copy() | |
| value_cols = [f"total_dead_{t}_decadal" for t in CLIMATE_TYPES] | |
| df = df_wide.melt( | |
| id_vars=["entity", "code", "year"], | |
| value_vars=value_cols, | |
| var_name="disaster_type", | |
| value_name="impact", | |
| ) | |
| df["disaster_type"] = ( | |
| df["disaster_type"].str.replace("total_dead_", "").str.replace("_decadal", "") | |
| ) | |
| df = df.rename(columns={"entity": "country"}) | |
| df["impact"] = df["impact"].fillna(0) | |
| logger.info( | |
| "Dataset long : %d lignes, %d pays, %d types", | |
| len(df), | |
| df["country"].nunique(), | |
| df["disaster_type"].nunique(), | |
| ) | |
| return df | |
| def construire_features(df: pd.DataFrame) -> pd.DataFrame: | |
| """Construit les features (continent, lags par pays x type, warming, cross-type).""" | |
| df["continent"] = df["country"].map(CONTINENT_MAP).fillna("Autre") | |
| df["continent_enc"] = df["continent"].map(CONTINENT_ENC) | |
| df["type_enc"] = df["disaster_type"].map(TYPE_ENC) | |
| df["warming"] = df["year"].map(WARMING_INDEX) | |
| df["warming_sq"] = df["warming"] ** 2 | |
| df = df.sort_values(["country", "disaster_type", "year"]).reset_index(drop=True) | |
| df["log_impact"] = np.log1p(df["impact"]) | |
| df["decade_index"] = (df["year"] - 1900) // 10 | |
| grp = ["country", "disaster_type"] | |
| df["lag_1"] = df.groupby(grp)["impact"].shift(1) | |
| df["lag_2"] = df.groupby(grp)["impact"].shift(2) | |
| df["log_lag_1"] = np.log1p(df["lag_1"]) | |
| df["log_lag_2"] = np.log1p(df["lag_2"]) | |
| df["trend"] = (df["impact"] - df["lag_1"]) / (df["lag_1"] + 1) | |
| df["cumul_mean"] = df.groupby(grp)["impact"].transform( | |
| lambda x: x.expanding().mean().shift(1) | |
| ) | |
| df["cumul_std"] = df.groupby(grp)["impact"].transform( | |
| lambda x: x.expanding().std().shift(1) | |
| ) | |
| df["cumul_max"] = df.groupby(grp)["impact"].transform( | |
| lambda x: x.expanding().max().shift(1) | |
| ) | |
| df["log_cumul_mean"] = np.log1p(df["cumul_mean"]) | |
| df["impact_other_types"] = ( | |
| df.groupby(["country", "year"])["impact"].transform("sum") - df["impact"] | |
| ) | |
| df["log_impact_other"] = np.log1p(df["impact_other_types"]) | |
| df["risk_level"] = df["impact"].apply(categoriser_risque) | |
| return df | |
| REG_MODELS = { | |
| # Quantile loss (median) = robuste aux outliers (ex: canicule 2003 France) | |
| "GBM_quantile_median": { | |
| "class": GradientBoostingRegressor, | |
| "kwargs": { | |
| "n_estimators": 200, | |
| "max_depth": 5, | |
| "learning_rate": 0.1, | |
| "loss": "quantile", | |
| "alpha": 0.5, | |
| }, | |
| }, | |
| "GBM_quantile_deep": { | |
| "class": GradientBoostingRegressor, | |
| "kwargs": { | |
| "n_estimators": 400, | |
| "max_depth": 7, | |
| "learning_rate": 0.05, | |
| "loss": "quantile", | |
| "alpha": 0.5, | |
| }, | |
| }, | |
| "GBM_quantile_shallow": { | |
| "class": GradientBoostingRegressor, | |
| "kwargs": { | |
| "n_estimators": 100, | |
| "max_depth": 3, | |
| "learning_rate": 0.1, | |
| "loss": "quantile", | |
| "alpha": 0.5, | |
| }, | |
| }, | |
| # Reference : MSE (mean) - sensible aux outliers, garde pour comparaison | |
| "GBM_mean": { | |
| "class": GradientBoostingRegressor, | |
| "kwargs": {"n_estimators": 200, "max_depth": 5, "learning_rate": 0.1}, | |
| }, | |
| "RF_200": { | |
| "class": RandomForestRegressor, | |
| "kwargs": {"n_estimators": 200, "max_depth": 10, "n_jobs": -1}, | |
| }, | |
| "HistGBM": { | |
| "class": HistGradientBoostingRegressor, | |
| "kwargs": {"max_iter": 300, "max_depth": 7, "learning_rate": 0.1}, | |
| }, | |
| "Ridge": {"class": Ridge, "kwargs": {"alpha": 1.0}}, | |
| "DecisionTree": {"class": DecisionTreeRegressor, "kwargs": {"max_depth": 10}}, | |
| } | |
| CLS_MODELS = { | |
| "GBM_base": { | |
| "class": GradientBoostingClassifier, | |
| "kwargs": {"n_estimators": 200, "max_depth": 5, "learning_rate": 0.1}, | |
| }, | |
| "GBM_deep": { | |
| "class": GradientBoostingClassifier, | |
| "kwargs": {"n_estimators": 400, "max_depth": 7, "learning_rate": 0.05}, | |
| }, | |
| "GBM_shallow": { | |
| "class": GradientBoostingClassifier, | |
| "kwargs": {"n_estimators": 100, "max_depth": 3, "learning_rate": 0.1}, | |
| }, | |
| "RF_200": { | |
| "class": RandomForestClassifier, | |
| "kwargs": { | |
| "n_estimators": 200, | |
| "max_depth": 10, | |
| "n_jobs": -1, | |
| "class_weight": "balanced", | |
| }, | |
| }, | |
| "RF_500": { | |
| "class": RandomForestClassifier, | |
| "kwargs": { | |
| "n_estimators": 500, | |
| "max_depth": 15, | |
| "n_jobs": -1, | |
| "class_weight": "balanced", | |
| }, | |
| }, | |
| "HistGBM": { | |
| "class": HistGradientBoostingClassifier, | |
| "kwargs": { | |
| "max_iter": 300, | |
| "max_depth": 7, | |
| "learning_rate": 0.1, | |
| "class_weight": "balanced", | |
| }, | |
| }, | |
| "LogReg": { | |
| "class": LogisticRegression, | |
| "kwargs": {"max_iter": 1000, "class_weight": "balanced"}, | |
| }, | |
| "DecisionTree": { | |
| "class": DecisionTreeClassifier, | |
| "kwargs": {"max_depth": 10, "class_weight": "balanced"}, | |
| }, | |
| } | |
| class _noop: | |
| """Context manager inerte quand MLflow est absent.""" | |
| def __enter__(self): | |
| return None | |
| def __exit__(self, *a): | |
| return False | |
| def _build_pipeline(estimator, step_name: str) -> Pipeline: | |
| """Construit un pipeline SimpleImputer + StandardScaler + estimator.""" | |
| return Pipeline( | |
| [ | |
| ("imputer", SimpleImputer(strategy="median")), | |
| ("scaler", StandardScaler()), | |
| (step_name, estimator), | |
| ] | |
| ) | |
| def entrainer_et_exporter(df: pd.DataFrame) -> None: | |
| """Grid de modeles + export des meilleurs artefacts.""" | |
| df_model = df[df["year"] >= 1920].copy() | |
| mask_train = df_model["year"] < 2010 | |
| mask_test = df_model["year"] >= 2010 | |
| X_train = df_model.loc[mask_train, FEATURES_REG].values | |
| X_test = df_model.loc[mask_test, FEATURES_REG].values | |
| y_train_reg = df_model.loc[mask_train, "log_impact"].values | |
| y_test_reg = df_model.loc[mask_test, "log_impact"].values | |
| y_train_cls = df_model.loc[mask_train, "risk_level"].map(RISK_MAP).values | |
| y_test_cls = df_model.loc[mask_test, "risk_level"].map(RISK_MAP).values | |
| sw_train = compute_sample_weight("balanced", y_train_cls) | |
| logger.info( | |
| "Train : %d lignes | Test : %d lignes | %d features", | |
| len(X_train), | |
| len(X_test), | |
| X_train.shape[1], | |
| ) | |
| # Prepare MLflow context (if available) | |
| tracking_uri = None | |
| env_label = "local" | |
| if HAS_MLFLOW: | |
| tracking_uri = ( | |
| os.getenv("MLFLOW_TRACKING_URI") | |
| or f"sqlite:///{(BASE / 'mlflow.db').resolve()}" | |
| ) | |
| env_label = os.getenv("ENV_LABEL") or ("ci" if os.getenv("CI") else "local") | |
| mlflow.set_tracking_uri(tracking_uri) | |
| exp_name = os.getenv("MLFLOW_EXPERIMENT") or "NB10_train_script" | |
| mlflow.set_experiment(exp_name) | |
| parent_cm = ( | |
| mlflow.start_run( | |
| run_name=f"train_nb10_{env_label}_{datetime.now():%Y%m%d_%H%M%S}" | |
| ) | |
| if HAS_MLFLOW | |
| else _noop() | |
| ) | |
| with parent_cm as parent_run: | |
| if HAS_MLFLOW: | |
| mlflow.set_tags( | |
| { | |
| "git_commit": _git_commit(), | |
| "git_branch": _git_branch(), | |
| "env": env_label, | |
| "python": sys.version.split()[0], | |
| "sklearn": sklearn.__version__, | |
| "platform": platform.platform(), | |
| "trigger": "github_actions" if os.getenv("CI") else "manual", | |
| "experiment_type": "quantile_pivot", # bascule median vs mean | |
| "motivation": "robustesse aux outliers (canicule France 2003)", | |
| } | |
| ) | |
| mlflow.log_params( | |
| { | |
| "n_features": len(FEATURES_REG), | |
| "features": str(FEATURES_REG), | |
| "split": "temporal_2010", | |
| "n_train": len(X_train), | |
| "n_test": len(X_test), | |
| "seed": SEED, | |
| "n_reg_models": len(REG_MODELS), | |
| "n_cls_models": len(CLS_MODELS), | |
| "warming_2030": WARMING_INDEX[2030], | |
| } | |
| ) | |
| # ββ GRID REGRESSION ββββββββββββββββββββββββββββββββββββββββββββ | |
| results_reg = [] | |
| # Selection par MAE_test (minimize) : plus robuste que R2 aux outliers | |
| # Mieux aligne avec Quantile Regression qui cible la mediane | |
| best_reg, best_reg_name, best_reg_mae = None, None, np.inf | |
| best_reg_r2 = -np.inf | |
| for name, spec in REG_MODELS.items(): | |
| t0 = datetime.now() | |
| kwargs = {**spec["kwargs"]} | |
| if "random_state" in spec["class"]().get_params(): | |
| kwargs["random_state"] = SEED | |
| estimator = spec["class"](**kwargs) | |
| pipe = _build_pipeline(estimator, "reg") | |
| pipe.fit(X_train, y_train_reg) | |
| dur = (datetime.now() - t0).total_seconds() | |
| y_pred_tr = pipe.predict(X_train) | |
| y_pred_te = pipe.predict(X_test) | |
| r2_tr = float(r2_score(y_train_reg, y_pred_tr)) | |
| r2_te = float(r2_score(y_test_reg, y_pred_te)) | |
| mae_te = float(mean_absolute_error(y_test_reg, y_pred_te)) | |
| rmse_te = float(np.sqrt(mean_squared_error(y_test_reg, y_pred_te))) | |
| results_reg.append( | |
| { | |
| "name": name, | |
| "r2_test": r2_te, | |
| "mae_test": mae_te, | |
| "rmse_test": rmse_te, | |
| "dur_s": dur, | |
| } | |
| ) | |
| logger.info( | |
| "REG %s : R2_test=%.4f MAE=%.4f (%.1fs)", name, r2_te, mae_te, dur | |
| ) | |
| if HAS_MLFLOW: | |
| # Tag loss family pour tracabilite (mean vs quantile/median vs other) | |
| loss_kind = spec["kwargs"].get("loss", "mse") | |
| if loss_kind == "quantile": | |
| loss_family = f"quantile_q{spec['kwargs'].get('alpha', 0.5)}" | |
| elif loss_kind in ("mse", "squared_error"): | |
| loss_family = "mean_mse" | |
| else: | |
| loss_family = loss_kind | |
| with mlflow.start_run(run_name=f"reg_{name}", nested=True): | |
| mlflow.set_tags( | |
| { | |
| "loss_family": loss_family, | |
| "selection_metric": "mae_test", | |
| "pivot_date": "2026-04-15", # bascule quantile | |
| } | |
| ) | |
| mlflow.log_params( | |
| { | |
| "model_type": spec["class"].__name__, | |
| "task": "regression", | |
| **spec["kwargs"], | |
| } | |
| ) | |
| mlflow.log_metric("r2_train", r2_tr) | |
| mlflow.log_metric("r2_test", r2_te) | |
| mlflow.log_metric("mae_test", mae_te) | |
| mlflow.log_metric("rmse_test", rmse_te) | |
| mlflow.log_metric("train_time_s", dur) | |
| if mae_te < best_reg_mae: | |
| best_reg_mae = mae_te | |
| best_reg_r2 = r2_te | |
| best_reg = pipe | |
| best_reg_name = name | |
| # ββ GRID CLASSIFICATION ββββββββββββββββββββββββββββββββββββββββ | |
| results_cls = [] | |
| best_cls, best_cls_name, best_cls_f1 = None, None, -np.inf | |
| for name, spec in CLS_MODELS.items(): | |
| t0 = datetime.now() | |
| kwargs = {**spec["kwargs"]} | |
| if "random_state" in spec["class"]().get_params(): | |
| kwargs["random_state"] = SEED | |
| estimator = spec["class"](**kwargs) | |
| pipe = _build_pipeline(estimator, "clf") | |
| try: | |
| pipe.fit(X_train, y_train_cls, clf__sample_weight=sw_train) | |
| except (TypeError, ValueError): | |
| pipe.fit(X_train, y_train_cls) | |
| dur = (datetime.now() - t0).total_seconds() | |
| y_pred_te = pipe.predict(X_test) | |
| acc_te = float(accuracy_score(y_test_cls, y_pred_te)) | |
| f1_te = float(f1_score(y_test_cls, y_pred_te, average="macro")) | |
| f1_w_te = float(f1_score(y_test_cls, y_pred_te, average="weighted")) | |
| results_cls.append( | |
| { | |
| "name": name, | |
| "accuracy_test": acc_te, | |
| "f1_macro_test": f1_te, | |
| "dur_s": dur, | |
| } | |
| ) | |
| logger.info( | |
| "CLS %s : acc_test=%.4f f1_macro=%.4f (%.1fs)", name, acc_te, f1_te, dur | |
| ) | |
| if HAS_MLFLOW: | |
| with mlflow.start_run(run_name=f"cls_{name}", nested=True): | |
| mlflow.log_params( | |
| { | |
| "model_type": spec["class"].__name__, | |
| "task": "classification", | |
| **spec["kwargs"], | |
| } | |
| ) | |
| mlflow.log_metric("accuracy_test", acc_te) | |
| mlflow.log_metric("f1_macro_test", f1_te) | |
| mlflow.log_metric("f1_weighted_test", f1_w_te) | |
| mlflow.log_metric("train_time_s", dur) | |
| if f1_te > best_cls_f1: | |
| best_cls_f1 = f1_te | |
| best_cls = pipe | |
| best_cls_name = name | |
| logger.info( | |
| "MEILLEUR REG : %s (MAE_test=%.4f, R2_test=%.4f)", | |
| best_reg_name, | |
| best_reg_mae, | |
| best_reg_r2, | |
| ) | |
| logger.info("MEILLEUR CLS : %s (f1_macro=%.4f)", best_cls_name, best_cls_f1) | |
| # Metriques finales + enregistrement modele registry | |
| y_pred_reg = best_reg.predict(X_train) | |
| y_pred_cls = best_cls.predict(X_train) | |
| metrics = { | |
| "best_reg_r2_test": best_reg_r2, | |
| "best_cls_f1_test": best_cls_f1, | |
| "train_r2": float(r2_score(y_train_reg, y_pred_reg)), | |
| "train_mae": float(mean_absolute_error(y_train_reg, y_pred_reg)), | |
| "train_accuracy": float(accuracy_score(y_train_cls, y_pred_cls)), | |
| "train_f1_macro": float(f1_score(y_train_cls, y_pred_cls, average="macro")), | |
| } | |
| pipe_reg = best_reg | |
| pipe_cls = best_cls | |
| if HAS_MLFLOW: | |
| # Metriques globales sur le parent | |
| for k, v in metrics.items(): | |
| mlflow.log_metric(k, v) | |
| mlflow.set_tag("best_reg_model", best_reg_name) | |
| mlflow.set_tag("best_cls_model", best_cls_name) | |
| # Log & registry des meilleurs modeles | |
| sig_reg = infer_signature(X_train, y_pred_reg) | |
| mlflow.sklearn.log_model( | |
| pipe_reg, | |
| name="best_regression", | |
| signature=sig_reg, | |
| input_example=X_train[:3], | |
| registered_model_name="NB10_regression" if env_label != "ci" else None, | |
| ) | |
| sig_cls = infer_signature(X_train, y_pred_cls) | |
| mlflow.sklearn.log_model( | |
| pipe_cls, | |
| name="best_classification", | |
| signature=sig_cls, | |
| input_example=X_train[:3], | |
| registered_model_name=( | |
| "NB10_classification" if env_label != "ci" else None | |
| ), | |
| ) | |
| # Artifacts : plots | |
| for png in [ | |
| "NB10_eda.png", | |
| "NB10_regression.png", | |
| "NB10_classification.png", | |
| "NB10_predictions_2030.png", | |
| ]: | |
| p = OUTPUT_DIR / png | |
| if p.exists(): | |
| mlflow.log_artifact(str(p), artifact_path="plots") | |
| # Dataset logging | |
| try: | |
| dataset = mlflow.data.from_pandas( | |
| df_model.head(1000), | |
| source=str(DATA_PATH), | |
| name="emdat_decadal", | |
| ) | |
| mlflow.log_input(dataset, context="training") | |
| except Exception as e: | |
| logger.debug("mlflow.log_input skip : %s", e) | |
| # Tableau comparatif | |
| pd.DataFrame(results_reg).to_csv( | |
| OUTPUT_DIR / "NB10_mlflow_reg_grid.csv", index=False | |
| ) | |
| pd.DataFrame(results_cls).to_csv( | |
| OUTPUT_DIR / "NB10_mlflow_cls_grid.csv", index=False | |
| ) | |
| mlflow.log_artifact(str(OUTPUT_DIR / "NB10_mlflow_reg_grid.csv")) | |
| mlflow.log_artifact(str(OUTPUT_DIR / "NB10_mlflow_cls_grid.csv")) | |
| logger.info( | |
| "MLflow : %d runs reg + %d runs cls + parent sur %s", | |
| len(results_reg), | |
| len(results_cls), | |
| tracking_uri, | |
| ) | |
| # Predictions 2030 pour chaque (pays, type) | |
| df_2020 = df[df["year"] == 2020].copy() | |
| df_2010 = df[df["year"] == 2010].copy() | |
| pred_rows = [] | |
| for _, row in df_2020.iterrows(): | |
| pays = row["country"] | |
| typ = row["disaster_type"] | |
| hist_2010 = df_2010[ | |
| (df_2010["country"] == pays) & (df_2010["disaster_type"] == typ) | |
| ] | |
| lag_2_val = hist_2010.iloc[0]["impact"] if len(hist_2010) > 0 else 0 | |
| all_hist = df[(df["country"] == pays) & (df["disaster_type"] == typ)] | |
| other_2020 = df_2020[ | |
| (df_2020["country"] == pays) & (df_2020["disaster_type"] != typ) | |
| ] | |
| features = { | |
| "decade_index": 13, | |
| "continent_enc": row["continent_enc"], | |
| "type_enc": row["type_enc"], | |
| "log_lag_1": np.log1p(row["impact"]), | |
| "log_lag_2": np.log1p(lag_2_val), | |
| "trend": row["trend"] if pd.notnull(row["trend"]) else 0, | |
| "log_cumul_mean": np.log1p(all_hist["impact"].mean()), | |
| "cumul_std": all_hist["impact"].std() if len(all_hist) > 1 else 0, | |
| "cumul_max": all_hist["impact"].max(), | |
| "log_impact_other": np.log1p(other_2020["impact"].sum()), | |
| "warming": WARMING_INDEX[2030], | |
| "warming_sq": WARMING_INDEX[2030] ** 2, | |
| } | |
| pred_rows.append( | |
| { | |
| "country": pays, | |
| "continent": row["continent"], | |
| "disaster_type": typ, | |
| **features, | |
| } | |
| ) | |
| df_pred = pd.DataFrame(pred_rows) | |
| X_2030 = df_pred[FEATURES_REG].values | |
| df_pred["log_impact_pred"] = pipe_reg.predict(X_2030) | |
| df_pred["impact_pred"] = np.expm1(df_pred["log_impact_pred"]).clip(lower=0) | |
| df_pred["risk_pred_enc"] = pipe_cls.predict(X_2030) | |
| df_pred["risk_pred"] = df_pred["risk_pred_enc"].map( | |
| {i: n for i, n in enumerate(RISK_NAMES)} | |
| ) | |
| df_country = ( | |
| df_pred.groupby(["country", "continent"])["impact_pred"].sum().reset_index() | |
| ) | |
| # Sauvegarde : 7 joblib + 2 CSV | |
| joblib.dump(pipe_reg, OUTPUT_DIR / "NB10_best_reg.joblib") | |
| joblib.dump(pipe_cls, OUTPUT_DIR / "NB10_best_cls.joblib") | |
| joblib.dump(CONTINENT_ENC, OUTPUT_DIR / "NB10_continent_enc.joblib") | |
| joblib.dump(TYPE_ENC, OUTPUT_DIR / "NB10_type_enc.joblib") | |
| joblib.dump(RISK_NAMES, OUTPUT_DIR / "NB10_risk_names.joblib") | |
| joblib.dump(WARMING_INDEX, OUTPUT_DIR / "NB10_warming_index.joblib") | |
| joblib.dump(FEATURES_REG, OUTPUT_DIR / "NB10_features.joblib") | |
| df_pred[ | |
| ["country", "continent", "disaster_type", "impact_pred", "risk_pred"] | |
| ].to_csv(OUTPUT_DIR / "NB10_predictions_2030_detail.csv", index=False) | |
| df_country.sort_values("impact_pred", ascending=False).to_csv( | |
| OUTPUT_DIR / "NB10_predictions_2030_country.csv", index=False | |
| ) | |
| logger.info( | |
| "Artefacts sauvegardes dans %s : 7 joblib + 2 CSV (%d pays x type, %d pays)", | |
| OUTPUT_DIR, | |
| len(df_pred), | |
| len(df_country), | |
| ) | |
| def main() -> None: | |
| """Pipeline complete : chargement, features, entrainement, export.""" | |
| df = charger_dataset() | |
| df = construire_features(df) | |
| entrainer_et_exporter(df) | |
| logger.info("Entrainement NB10 termine avec succes") | |
| if __name__ == "__main__": | |
| main() | |