import os import json import time import joblib import pandas as pd import numpy as np from sklearn.model_selection import train_test_split from sklearn.preprocessing import OneHotEncoder, StandardScaler from sklearn.compose import ColumnTransformer from sklearn.pipeline import Pipeline from sklearn.ensemble import ExtraTreesRegressor, RandomForestRegressor from sklearn.tree import DecisionTreeRegressor from sklearn.metrics import r2_score, mean_absolute_error, mean_squared_error import lightgbm as lgb import xgboost as xgb def load_and_clean_data(): excel_path = os.path.join("information", "Untitled spreadsheet.xlsx") csv_path = "yield_df.csv" if os.path.exists(excel_path): print(f"Loading data from {excel_path}...") df = pd.read_excel(excel_path) elif os.path.exists(csv_path): print(f"Loading data from {csv_path}...") df = pd.read_csv(csv_path) else: raise FileNotFoundError("Neither Excel nor CSV dataset found!") # Drop unwanted index column if present if 'Unnamed: 0' in df.columns: df = df.drop(columns=['Unnamed: 0']) # Ensure proper data types df['Area'] = df['Area'].astype(str).str.strip() df['Item'] = df['Item'].astype(str).str.strip() df['Year'] = df['Year'].astype(int) df['hg/ha_yield'] = df['hg/ha_yield'].astype(float) df['average_rain_fall_mm_per_year'] = df['average_rain_fall_mm_per_year'].astype(float) df['pesticides_tonnes'] = df['pesticides_tonnes'].astype(float) df['avg_temp'] = df['avg_temp'].astype(float) # Save optimized CSV df.to_csv(csv_path, index=False) print(f"Dataset saved to {csv_path} with shape {df.shape}") return df def train_and_evaluate(): df = load_and_clean_data() features = ['Area', 'Item', 'Year', 'average_rain_fall_mm_per_year', 'pesticides_tonnes', 'avg_temp'] target = 'hg/ha_yield' X = df[features] y = df[target] categorical_cols = ['Area', 'Item'] numerical_cols = ['Year', 'average_rain_fall_mm_per_year', 'pesticides_tonnes', 'avg_temp'] # Preprocessing preprocessor = ColumnTransformer( transformers=[ ('cat', OneHotEncoder(drop='first', handle_unknown='ignore', sparse_output=False), categorical_cols), ('num', StandardScaler(), numerical_cols) ] ) X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42) print(f"Train samples: {len(X_train)}, Test samples: {len(X_test)}") # Candidate models models = { "Extra Trees": ExtraTreesRegressor(n_estimators=100, random_state=42, n_jobs=-1), "Random Forest": RandomForestRegressor(n_estimators=100, random_state=42, n_jobs=-1), "Decision Tree": DecisionTreeRegressor(random_state=42), "LightGBM": lgb.LGBMRegressor(random_state=42, verbose=-1, n_estimators=200), "XGBoost": xgb.XGBRegressor(random_state=42, n_estimators=200, learning_rate=0.1) } benchmark_results = [] trained_pipelines = {} for name, model in models.items(): print(f"Training {name}...") pipe = Pipeline([ ('preprocessor', preprocessor), ('model', model) ]) start_time = time.time() pipe.fit(X_train, y_train) fit_time = time.time() - start_time y_pred = pipe.predict(X_test) r2 = r2_score(y_test, y_pred) mae = mean_absolute_error(y_test, y_pred) rmse = float(np.sqrt(mean_squared_error(y_test, y_pred))) benchmark_results.append({ "model_name": name, "r2_score": round(float(r2), 4), "mae_hg_ha": round(float(mae), 2), "rmse_hg_ha": round(rmse, 2), "training_time_sec": round(float(fit_time), 2) }) trained_pipelines[name] = pipe print(f" -> R2: {r2:.4f}, MAE: {mae:.2f}, RMSE: {rmse:.2f} ({fit_time:.2f}s)") # Select Best Model (Extra Trees) best_model_name = "Extra Trees" best_pipeline = trained_pipelines[best_model_name] # Extract feature names & importances ohe = best_pipeline.named_steps['preprocessor'].named_transformers_['cat'] cat_feature_names = ohe.get_feature_names_out(categorical_cols).tolist() all_feature_names = cat_feature_names + numerical_cols feature_importances = best_pipeline.named_steps['model'].feature_importances_ # Group feature importances into broad categories for intuitive charts grouped_importance = { "Crop Type (Item)": 0.0, "Country (Area)": 0.0, "Year / Trend": 0.0, "Average Rainfall": 0.0, "Pesticides Application": 0.0, "Average Temperature": 0.0 } for fname, imp in zip(all_feature_names, feature_importances): if fname.startswith("Item_"): grouped_importance["Crop Type (Item)"] += float(imp) elif fname.startswith("Area_"): grouped_importance["Country (Area)"] += float(imp) elif fname == "Year": grouped_importance["Year / Trend"] += float(imp) elif fname == "average_rain_fall_mm_per_year": grouped_importance["Average Rainfall"] += float(imp) elif fname == "pesticides_tonnes": grouped_importance["Pesticides Application"] += float(imp) elif fname == "avg_temp": grouped_importance["Average Temperature"] += float(imp) # Normalize grouped importance tot_imp = sum(grouped_importance.values()) if tot_imp > 0: grouped_importance = {k: round(v / tot_imp * 100, 2) for k, v in grouped_importance.items()} # Top 15 specific one-hot feature importances top_specific_importances = sorted( [{"feature": fname, "importance_pct": round(float(imp) * 100, 3)} for fname, imp in zip(all_feature_names, feature_importances)], key=lambda x: x["importance_pct"], reverse=True )[:15] # Calculate Country-level climate defaults & historical statistics countries = sorted(df['Area'].unique().tolist()) crops = sorted(df['Item'].unique().tolist()) country_stats = {} for country in countries: c_df = df[df['Area'] == country] country_stats[country] = { "avg_rainfall": round(float(c_df['average_rain_fall_mm_per_year'].mean()), 1), "avg_temp": round(float(c_df['avg_temp'].mean()), 1), "avg_pesticides": round(float(c_df['pesticides_tonnes'].mean()), 1), "min_rainfall": float(c_df['average_rain_fall_mm_per_year'].min()), "max_rainfall": float(c_df['average_rain_fall_mm_per_year'].max()), "min_temp": float(c_df['avg_temp'].min()), "max_temp": float(c_df['avg_temp'].max()), "min_pesticides": float(c_df['pesticides_tonnes'].min()), "max_pesticides": float(c_df['pesticides_tonnes'].max()), "total_records": int(len(c_df)) } crop_stats = {} for crop in crops: cr_df = df[df['Item'] == crop] crop_stats[crop] = { "mean_yield_hg_ha": round(float(cr_df['hg/ha_yield'].mean()), 1), "median_yield_hg_ha": round(float(cr_df['hg/ha_yield'].median()), 1), "min_yield_hg_ha": float(cr_df['hg/ha_yield'].min()), "max_yield_hg_ha": float(cr_df['hg/ha_yield'].max()), "total_records": int(len(cr_df)) } # Global limits for sliders global_stats = { "min_year": int(df['Year'].min()), "max_year": int(df['Year'].max()), "min_rainfall": float(df['average_rain_fall_mm_per_year'].min()), "max_rainfall": float(df['average_rain_fall_mm_per_year'].max()), "min_temp": float(df['avg_temp'].min()), "max_temp": float(df['avg_temp'].max()), "min_pesticides": float(df['pesticides_tonnes'].min()), "max_pesticides": float(df['pesticides_tonnes'].max()), "total_samples": int(len(df)) } metadata = { "champion_model": best_model_name, "metrics": next(item for item in benchmark_results if item["model_name"] == best_model_name), "benchmark_summary": benchmark_results, "grouped_feature_importance": grouped_importance, "top_feature_importances": top_specific_importances, "countries": countries, "crops": crops, "country_stats": country_stats, "crop_stats": crop_stats, "global_stats": global_stats, "trained_at": time.strftime("%Y-%m-%d %H:%M:%S") } # Save Model and Metadata model_output_path = "crop_yield_model.joblib" metadata_output_path = "model_metadata.json" joblib.dump(best_pipeline, model_output_path, compress=3) print(f"Production model saved to {model_output_path}") with open(metadata_output_path, "w") as f: json.dump(metadata, f, indent=2) print(f"Model metadata saved to {metadata_output_path}") print("\n=== Model Training Complete ===") print(f"Selected Champion Model: {best_model_name} (R² = {metadata['metrics']['r2_score']})") if __name__ == "__main__": train_and_evaluate()