Upload folder using huggingface_hub

ed77cf2 verified 2 months ago

9.26 kB

import os
import json
import time
import joblib
import pandas as pd
import numpy as np
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import OneHotEncoder, StandardScaler
from sklearn.compose import ColumnTransformer
from sklearn.pipeline import Pipeline
from sklearn.ensemble import ExtraTreesRegressor, RandomForestRegressor
from sklearn.tree import DecisionTreeRegressor
from sklearn.metrics import r2_score, mean_absolute_error, mean_squared_error
import lightgbm as lgb
import xgboost as xgb

def load_and_clean_data():
    excel_path = os.path.join("information", "Untitled spreadsheet.xlsx")
    csv_path = "yield_df.csv"
    
    if os.path.exists(excel_path):
        print(f"Loading data from {excel_path}...")
        df = pd.read_excel(excel_path)
    elif os.path.exists(csv_path):
        print(f"Loading data from {csv_path}...")
        df = pd.read_csv(csv_path)
    else:
        raise FileNotFoundError("Neither Excel nor CSV dataset found!")
    
    # Drop unwanted index column if present
    if 'Unnamed: 0' in df.columns:
        df = df.drop(columns=['Unnamed: 0'])
    
    # Ensure proper data types
    df['Area'] = df['Area'].astype(str).str.strip()
    df['Item'] = df['Item'].astype(str).str.strip()
    df['Year'] = df['Year'].astype(int)
    df['hg/ha_yield'] = df['hg/ha_yield'].astype(float)
    df['average_rain_fall_mm_per_year'] = df['average_rain_fall_mm_per_year'].astype(float)
    df['pesticides_tonnes'] = df['pesticides_tonnes'].astype(float)
    df['avg_temp'] = df['avg_temp'].astype(float)
    
    # Save optimized CSV
    df.to_csv(csv_path, index=False)
    print(f"Dataset saved to {csv_path} with shape {df.shape}")
    return df

def train_and_evaluate():
    df = load_and_clean_data()
    
    features = ['Area', 'Item', 'Year', 'average_rain_fall_mm_per_year', 'pesticides_tonnes', 'avg_temp']
    target = 'hg/ha_yield'
    
    X = df[features]
    y = df[target]
    
    categorical_cols = ['Area', 'Item']
    numerical_cols = ['Year', 'average_rain_fall_mm_per_year', 'pesticides_tonnes', 'avg_temp']
    
    # Preprocessing
    preprocessor = ColumnTransformer(
        transformers=[
            ('cat', OneHotEncoder(drop='first', handle_unknown='ignore', sparse_output=False), categorical_cols),
            ('num', StandardScaler(), numerical_cols)
        ]
    )
    
    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)
    print(f"Train samples: {len(X_train)}, Test samples: {len(X_test)}")
    
    # Candidate models
    models = {
        "Extra Trees": ExtraTreesRegressor(n_estimators=100, random_state=42, n_jobs=-1),
        "Random Forest": RandomForestRegressor(n_estimators=100, random_state=42, n_jobs=-1),
        "Decision Tree": DecisionTreeRegressor(random_state=42),
        "LightGBM": lgb.LGBMRegressor(random_state=42, verbose=-1, n_estimators=200),
        "XGBoost": xgb.XGBRegressor(random_state=42, n_estimators=200, learning_rate=0.1)
    }
    
    benchmark_results = []
    trained_pipelines = {}
    
    for name, model in models.items():
        print(f"Training {name}...")
        pipe = Pipeline([
            ('preprocessor', preprocessor),
            ('model', model)
        ])
        
        start_time = time.time()
        pipe.fit(X_train, y_train)
        fit_time = time.time() - start_time
        
        y_pred = pipe.predict(X_test)
        r2 = r2_score(y_test, y_pred)
        mae = mean_absolute_error(y_test, y_pred)
        rmse = float(np.sqrt(mean_squared_error(y_test, y_pred)))
        
        benchmark_results.append({
            "model_name": name,
            "r2_score": round(float(r2), 4),
            "mae_hg_ha": round(float(mae), 2),
            "rmse_hg_ha": round(rmse, 2),
            "training_time_sec": round(float(fit_time), 2)
        })
        trained_pipelines[name] = pipe
        print(f"  -> R2: {r2:.4f}, MAE: {mae:.2f}, RMSE: {rmse:.2f} ({fit_time:.2f}s)")
    
    # Select Best Model (Extra Trees)
    best_model_name = "Extra Trees"
    best_pipeline = trained_pipelines[best_model_name]
    
    # Extract feature names & importances
    ohe = best_pipeline.named_steps['preprocessor'].named_transformers_['cat']
    cat_feature_names = ohe.get_feature_names_out(categorical_cols).tolist()
    all_feature_names = cat_feature_names + numerical_cols
    
    feature_importances = best_pipeline.named_steps['model'].feature_importances_
    
    # Group feature importances into broad categories for intuitive charts
    grouped_importance = {
        "Crop Type (Item)": 0.0,
        "Country (Area)": 0.0,
        "Year / Trend": 0.0,
        "Average Rainfall": 0.0,
        "Pesticides Application": 0.0,
        "Average Temperature": 0.0
    }
    
    for fname, imp in zip(all_feature_names, feature_importances):
        if fname.startswith("Item_"):
            grouped_importance["Crop Type (Item)"] += float(imp)
        elif fname.startswith("Area_"):
            grouped_importance["Country (Area)"] += float(imp)
        elif fname == "Year":
            grouped_importance["Year / Trend"] += float(imp)
        elif fname == "average_rain_fall_mm_per_year":
            grouped_importance["Average Rainfall"] += float(imp)
        elif fname == "pesticides_tonnes":
            grouped_importance["Pesticides Application"] += float(imp)
        elif fname == "avg_temp":
            grouped_importance["Average Temperature"] += float(imp)
            
    # Normalize grouped importance
    tot_imp = sum(grouped_importance.values())
    if tot_imp > 0:
        grouped_importance = {k: round(v / tot_imp * 100, 2) for k, v in grouped_importance.items()}
    
    # Top 15 specific one-hot feature importances
    top_specific_importances = sorted(
        [{"feature": fname, "importance_pct": round(float(imp) * 100, 3)} 
         for fname, imp in zip(all_feature_names, feature_importances)],
        key=lambda x: x["importance_pct"],
        reverse=True
    )[:15]
    
    # Calculate Country-level climate defaults & historical statistics
    countries = sorted(df['Area'].unique().tolist())
    crops = sorted(df['Item'].unique().tolist())
    
    country_stats = {}
    for country in countries:
        c_df = df[df['Area'] == country]
        country_stats[country] = {
            "avg_rainfall": round(float(c_df['average_rain_fall_mm_per_year'].mean()), 1),
            "avg_temp": round(float(c_df['avg_temp'].mean()), 1),
            "avg_pesticides": round(float(c_df['pesticides_tonnes'].mean()), 1),
            "min_rainfall": float(c_df['average_rain_fall_mm_per_year'].min()),
            "max_rainfall": float(c_df['average_rain_fall_mm_per_year'].max()),
            "min_temp": float(c_df['avg_temp'].min()),
            "max_temp": float(c_df['avg_temp'].max()),
            "min_pesticides": float(c_df['pesticides_tonnes'].min()),
            "max_pesticides": float(c_df['pesticides_tonnes'].max()),
            "total_records": int(len(c_df))
        }
        
    crop_stats = {}
    for crop in crops:
        cr_df = df[df['Item'] == crop]
        crop_stats[crop] = {
            "mean_yield_hg_ha": round(float(cr_df['hg/ha_yield'].mean()), 1),
            "median_yield_hg_ha": round(float(cr_df['hg/ha_yield'].median()), 1),
            "min_yield_hg_ha": float(cr_df['hg/ha_yield'].min()),
            "max_yield_hg_ha": float(cr_df['hg/ha_yield'].max()),
            "total_records": int(len(cr_df))
        }
        
    # Global limits for sliders
    global_stats = {
        "min_year": int(df['Year'].min()),
        "max_year": int(df['Year'].max()),
        "min_rainfall": float(df['average_rain_fall_mm_per_year'].min()),
        "max_rainfall": float(df['average_rain_fall_mm_per_year'].max()),
        "min_temp": float(df['avg_temp'].min()),
        "max_temp": float(df['avg_temp'].max()),
        "min_pesticides": float(df['pesticides_tonnes'].min()),
        "max_pesticides": float(df['pesticides_tonnes'].max()),
        "total_samples": int(len(df))
    }
    
    metadata = {
        "champion_model": best_model_name,
        "metrics": next(item for item in benchmark_results if item["model_name"] == best_model_name),
        "benchmark_summary": benchmark_results,
        "grouped_feature_importance": grouped_importance,
        "top_feature_importances": top_specific_importances,
        "countries": countries,
        "crops": crops,
        "country_stats": country_stats,
        "crop_stats": crop_stats,
        "global_stats": global_stats,
        "trained_at": time.strftime("%Y-%m-%d %H:%M:%S")
    }
    
    # Save Model and Metadata
    model_output_path = "crop_yield_model.joblib"
    metadata_output_path = "model_metadata.json"
    
    joblib.dump(best_pipeline, model_output_path, compress=3)
    print(f"Production model saved to {model_output_path}")
    
    with open(metadata_output_path, "w") as f:
        json.dump(metadata, f, indent=2)
    print(f"Model metadata saved to {metadata_output_path}")
    
    print("\n=== Model Training Complete ===")
    print(f"Selected Champion Model: {best_model_name} (R² = {metadata['metrics']['r2_score']})")

if __name__ == "__main__":
    train_and_evaluate()

	import os
	import json
	import time
	import joblib
	import pandas as pd
	import numpy as np
	from sklearn.model_selection import train_test_split
	from sklearn.preprocessing import OneHotEncoder, StandardScaler
	from sklearn.compose import ColumnTransformer
	from sklearn.pipeline import Pipeline
	from sklearn.ensemble import ExtraTreesRegressor, RandomForestRegressor
	from sklearn.tree import DecisionTreeRegressor
	from sklearn.metrics import r2_score, mean_absolute_error, mean_squared_error
	import lightgbm as lgb
	import xgboost as xgb

	def load_and_clean_data():
	excel_path = os.path.join("information", "Untitled spreadsheet.xlsx")
	csv_path = "yield_df.csv"

	if os.path.exists(excel_path):
	print(f"Loading data from {excel_path}...")
	df = pd.read_excel(excel_path)
	elif os.path.exists(csv_path):
	print(f"Loading data from {csv_path}...")
	df = pd.read_csv(csv_path)
	else:
	raise FileNotFoundError("Neither Excel nor CSV dataset found!")

	# Drop unwanted index column if present
	if 'Unnamed: 0' in df.columns:
	df = df.drop(columns=['Unnamed: 0'])

	# Ensure proper data types
	df['Area'] = df['Area'].astype(str).str.strip()
	df['Item'] = df['Item'].astype(str).str.strip()
	df['Year'] = df['Year'].astype(int)
	df['hg/ha_yield'] = df['hg/ha_yield'].astype(float)
	df['average_rain_fall_mm_per_year'] = df['average_rain_fall_mm_per_year'].astype(float)
	df['pesticides_tonnes'] = df['pesticides_tonnes'].astype(float)
	df['avg_temp'] = df['avg_temp'].astype(float)

	# Save optimized CSV
	df.to_csv(csv_path, index=False)
	print(f"Dataset saved to {csv_path} with shape {df.shape}")
	return df

	def train_and_evaluate():
	df = load_and_clean_data()

	features = ['Area', 'Item', 'Year', 'average_rain_fall_mm_per_year', 'pesticides_tonnes', 'avg_temp']
	target = 'hg/ha_yield'

	X = df[features]
	y = df[target]

	categorical_cols = ['Area', 'Item']
	numerical_cols = ['Year', 'average_rain_fall_mm_per_year', 'pesticides_tonnes', 'avg_temp']

	# Preprocessing
	preprocessor = ColumnTransformer(
	transformers=[
	('cat', OneHotEncoder(drop='first', handle_unknown='ignore', sparse_output=False), categorical_cols),
	('num', StandardScaler(), numerical_cols)
	]
	)

	X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)
	print(f"Train samples: {len(X_train)}, Test samples: {len(X_test)}")

	# Candidate models
	models = {
	"Extra Trees": ExtraTreesRegressor(n_estimators=100, random_state=42, n_jobs=-1),
	"Random Forest": RandomForestRegressor(n_estimators=100, random_state=42, n_jobs=-1),
	"Decision Tree": DecisionTreeRegressor(random_state=42),
	"LightGBM": lgb.LGBMRegressor(random_state=42, verbose=-1, n_estimators=200),
	"XGBoost": xgb.XGBRegressor(random_state=42, n_estimators=200, learning_rate=0.1)
	}

	benchmark_results = []
	trained_pipelines = {}

	for name, model in models.items():
	print(f"Training {name}...")
	pipe = Pipeline([
	('preprocessor', preprocessor),
	('model', model)
	])

	start_time = time.time()
	pipe.fit(X_train, y_train)
	fit_time = time.time() - start_time

	y_pred = pipe.predict(X_test)
	r2 = r2_score(y_test, y_pred)
	mae = mean_absolute_error(y_test, y_pred)
	rmse = float(np.sqrt(mean_squared_error(y_test, y_pred)))

	benchmark_results.append({
	"model_name": name,
	"r2_score": round(float(r2), 4),
	"mae_hg_ha": round(float(mae), 2),
	"rmse_hg_ha": round(rmse, 2),
	"training_time_sec": round(float(fit_time), 2)
	})
	trained_pipelines[name] = pipe
	print(f" -> R2: {r2:.4f}, MAE: {mae:.2f}, RMSE: {rmse:.2f} ({fit_time:.2f}s)")

	# Select Best Model (Extra Trees)
	best_model_name = "Extra Trees"
	best_pipeline = trained_pipelines[best_model_name]

	# Extract feature names & importances
	ohe = best_pipeline.named_steps['preprocessor'].named_transformers_['cat']
	cat_feature_names = ohe.get_feature_names_out(categorical_cols).tolist()
	all_feature_names = cat_feature_names + numerical_cols

	feature_importances = best_pipeline.named_steps['model'].feature_importances_

	# Group feature importances into broad categories for intuitive charts
	grouped_importance = {
	"Crop Type (Item)": 0.0,
	"Country (Area)": 0.0,
	"Year / Trend": 0.0,
	"Average Rainfall": 0.0,
	"Pesticides Application": 0.0,
	"Average Temperature": 0.0
	}

	for fname, imp in zip(all_feature_names, feature_importances):
	if fname.startswith("Item_"):
	grouped_importance["Crop Type (Item)"] += float(imp)
	elif fname.startswith("Area_"):
	grouped_importance["Country (Area)"] += float(imp)
	elif fname == "Year":
	grouped_importance["Year / Trend"] += float(imp)
	elif fname == "average_rain_fall_mm_per_year":
	grouped_importance["Average Rainfall"] += float(imp)
	elif fname == "pesticides_tonnes":
	grouped_importance["Pesticides Application"] += float(imp)
	elif fname == "avg_temp":
	grouped_importance["Average Temperature"] += float(imp)

	# Normalize grouped importance
	tot_imp = sum(grouped_importance.values())
	if tot_imp > 0:
	grouped_importance = {k: round(v / tot_imp * 100, 2) for k, v in grouped_importance.items()}

	# Top 15 specific one-hot feature importances
	top_specific_importances = sorted(
	[{"feature": fname, "importance_pct": round(float(imp) * 100, 3)}
	for fname, imp in zip(all_feature_names, feature_importances)],
	key=lambda x: x["importance_pct"],
	reverse=True
	)[:15]

	# Calculate Country-level climate defaults & historical statistics
	countries = sorted(df['Area'].unique().tolist())
	crops = sorted(df['Item'].unique().tolist())

	country_stats = {}
	for country in countries:
	c_df = df[df['Area'] == country]
	country_stats[country] = {
	"avg_rainfall": round(float(c_df['average_rain_fall_mm_per_year'].mean()), 1),
	"avg_temp": round(float(c_df['avg_temp'].mean()), 1),
	"avg_pesticides": round(float(c_df['pesticides_tonnes'].mean()), 1),
	"min_rainfall": float(c_df['average_rain_fall_mm_per_year'].min()),
	"max_rainfall": float(c_df['average_rain_fall_mm_per_year'].max()),
	"min_temp": float(c_df['avg_temp'].min()),
	"max_temp": float(c_df['avg_temp'].max()),
	"min_pesticides": float(c_df['pesticides_tonnes'].min()),
	"max_pesticides": float(c_df['pesticides_tonnes'].max()),
	"total_records": int(len(c_df))
	}

	crop_stats = {}
	for crop in crops:
	cr_df = df[df['Item'] == crop]
	crop_stats[crop] = {
	"mean_yield_hg_ha": round(float(cr_df['hg/ha_yield'].mean()), 1),
	"median_yield_hg_ha": round(float(cr_df['hg/ha_yield'].median()), 1),
	"min_yield_hg_ha": float(cr_df['hg/ha_yield'].min()),
	"max_yield_hg_ha": float(cr_df['hg/ha_yield'].max()),
	"total_records": int(len(cr_df))
	}

	# Global limits for sliders
	global_stats = {
	"min_year": int(df['Year'].min()),
	"max_year": int(df['Year'].max()),
	"min_rainfall": float(df['average_rain_fall_mm_per_year'].min()),
	"max_rainfall": float(df['average_rain_fall_mm_per_year'].max()),
	"min_temp": float(df['avg_temp'].min()),
	"max_temp": float(df['avg_temp'].max()),
	"min_pesticides": float(df['pesticides_tonnes'].min()),
	"max_pesticides": float(df['pesticides_tonnes'].max()),
	"total_samples": int(len(df))
	}

	metadata = {
	"champion_model": best_model_name,
	"metrics": next(item for item in benchmark_results if item["model_name"] == best_model_name),
	"benchmark_summary": benchmark_results,
	"grouped_feature_importance": grouped_importance,
	"top_feature_importances": top_specific_importances,
	"countries": countries,
	"crops": crops,
	"country_stats": country_stats,
	"crop_stats": crop_stats,
	"global_stats": global_stats,
	"trained_at": time.strftime("%Y-%m-%d %H:%M:%S")
	}

	# Save Model and Metadata
	model_output_path = "crop_yield_model.joblib"
	metadata_output_path = "model_metadata.json"

	joblib.dump(best_pipeline, model_output_path, compress=3)
	print(f"Production model saved to {model_output_path}")

	with open(metadata_output_path, "w") as f:
	json.dump(metadata, f, indent=2)
	print(f"Model metadata saved to {metadata_output_path}")

	print("\n=== Model Training Complete ===")
	print(f"Selected Champion Model: {best_model_name} (R² = {metadata['metrics']['r2_score']})")

	if __name__ == "__main__":
	train_and_evaluate()