updated

app.py

@@ -3,15 +3,14 @@ import numpy as np
 import matplotlib.pyplot as plt
 import seaborn as sns
 import warnings
-import io
-import base64
+import gradio as gr
 from sklearn.model_selection import train_test_split
 from sklearn.ensemble import RandomForestClassifier
 from sklearn.metrics import classification_report
 from sklearn.preprocessing import StandardScaler
 from sklearn.cluster import KMeans
 from sklearn.decomposition import PCA
-import gradio as gr
+import io
 
 # Suppress specific FutureWarnings
 warnings.filterwarnings("ignore", category=FutureWarning)
@@ -39,19 +38,6 @@ def clean_data(df):
 
     df = df.drop(columns=['nsn'], errors='ignore')
 
-    category_cols = [
-        'gender', 'ethnicity', 'year_level', 'contributing_primary_school',
-        'year_11_english_teacher', 'year_11_maths_teacher', 'year_12_english_teacher',
-        'year_12_maths_teacher', 'form_teacher', 'leaving_date', 'primary_language',
-        'first_language', 'secondary_language', 'term_1_intervention',
-        'term_2_intervention', 'term_3_intervention', 'term_4_intervention',
-        'major_life_event', 'learning_difficulty', 'pastoral_care_incident',
-        'pastoral_care_action_taken', 'pastoral_care_follow_up'
-    ]
-    for col in category_cols:
-        if col in df.columns:
-            df[col] = df[col].astype('category')
-
     if 'ncea_results' in df.columns:
         ncea_results = []
         for idx, row in df.iterrows():
@@ -68,11 +54,6 @@ def clean_data(df):
     else:
         df['Total Credits'] = 0
 
-    if 'pastoral_care_follow_up' in df.columns:
-        df['action_effective'] = df['pastoral_care_follow_up'].apply(
-            lambda x: 'Effective' if 'resolved' in str(x).lower() else 'Not Effective'
-        )
-
     df['credit_threshold'] = df['year_level'].apply(lambda x: 80 if x == 'Year 11' else 60)
     df['credit_achievement_rate'] = df['Total Credits'] / df['credit_threshold']
 
@@ -106,264 +87,87 @@ def identify_at_risk_students(df):
     model.fit(X_train, y_train)
     y_pred = model.predict(X_test)
 
-    report = classification_report(y_test, y_pred)
+    classification_rep = classification_report(y_test, y_pred)
 
     importances = model.feature_importances_
     feature_names = features.columns
     feature_importance_df = pd.DataFrame({'Feature': feature_names, 'Importance': importances})
     feature_importance_df = feature_importance_df.sort_values(by='Importance', ascending=False)
 
-    graphs = []
-    tables = {}
-
-    tables['classification_report'] = report
-    tables['feature_importance'] = feature_importance_df.head(10).to_string()
-
-    if feature_importance_df['Importance'].sum() > 0:
-        plt.figure(figsize=(12, 6))
-        sns.barplot(data=feature_importance_df.head(10), x='Importance', y='Feature', palette='viridis')
-        plt.title('Top 10 Important Features for Predicting At-Risk Students', fontsize=14)
-        plt.xlabel('Importance', fontsize=12)
-        plt.ylabel('Feature', fontsize=12)
-        plt.tight_layout()
-        graphs.append(plt_to_base64())
-
-    return graphs, tables
-
-def process_extra_curricular(df):
-    df['extra_curricular_activities'] = df['extra_curricular_activities'].apply(
-        lambda x: eval(x) if isinstance(x, str) else []
-    )
-    activities = df['extra_curricular_activities'].explode().unique()
-    activities = [activity for activity in activities if activity is not None]
-    for activity in activities:
-        df[activity] = df['extra_curricular_activities'].apply(lambda x: int(activity in x))
-    return df
-
-def analyze_extra_curricular_impact(df):
-    graphs = []
-    activity_cols = [col for col in df.columns if col in ['Cricket', 'Debating', 'Football', 'Art Club', 'Drama Club', 'Rugby']]
-    for activity in activity_cols:
-        if activity in df.columns:
-            data = df.copy()
-            group = data.groupby(activity)['credit_achievement_rate'].mean().reset_index()
-            group[activity] = group[activity].map({0: 'Not Involved', 1: 'Involved'})
-            plt.figure(figsize=(6, 4))
-            sns.barplot(data=group, x=activity, y='credit_achievement_rate', palette='Set2', edgecolor='w', errorbar=None)
-            plt.title(f'Impact of {activity} on Credit Achievement Rate', fontsize=14)
-            plt.xlabel('Participation Status', fontsize=12)
-            plt.ylabel('Average Credit Achievement Rate', fontsize=12)
-            plt.tight_layout()
-            graphs.append(plt_to_base64())
-    return graphs
+    return classification_rep, feature_importance_df
 
-def analyze_teacher_performance(df):
-    graphs = []
-    teacher_year_levels = {
-        'year_11_english_teacher': 'Year 11',
-        'year_11_maths_teacher': 'Year 11',
-        'year_12_english_teacher': 'Year 12',
-        'year_12_maths_teacher': 'Year 12'
-    }
-    for col, year_level in teacher_year_levels.items():
-        data = df[(df[col] != 'Unknown') & (df['year_level'] == year_level)]
-        if not data.empty:
-            group = data.groupby(col)['credit_achievement_rate'].mean().reset_index()
-            plt.figure(figsize=(10, 6))
-            sns.barplot(data=group, x=col, y='credit_achievement_rate', palette='Set3', edgecolor='w', errorbar=None)
-            plt.title(f'Average Credit Achievement Rate by {col.replace("_", " ").title()} ({year_level})', fontsize=14)
-            plt.xlabel('Teacher', fontsize=12)
-            plt.ylabel('Average Credit Achievement Rate', fontsize=12)
-            plt.xticks(rotation=45)
-            plt.tight_layout()
-            graphs.append(plt_to_base64())
-    return graphs
-
-def analyze_language_impact(df):
-    graphs = []
-    data = df[df['primary_language'] != 'Unknown']
-    if not data.empty:
-        group = data.groupby('primary_language')['credit_achievement_rate'].mean().reset_index()
+def plot_credit_achievement_rate(df):
+    """
+    Example analysis and plot function
+    """
+    if 'credit_achievement_rate' in df.columns:
         plt.figure(figsize=(10, 6))
-        sns.barplot(data=group, x='primary_language', y='credit_achievement_rate', palette='Pastel1', edgecolor='w', errorbar=None)
-        plt.title('Average Credit Achievement Rate by Primary Language', fontsize=14)
-        plt.xlabel('Primary Language', fontsize=12)
-        plt.ylabel('Average Credit Achievement Rate', fontsize=12)
-        plt.xticks(rotation=45)
-        plt.tight_layout()
-        graphs.append(plt_to_base64())
-    return graphs
-
-def perform_clustering(df):
-    graphs = []
-    tables = {}
-    attendance_cols = [col for col in df.columns if 'attendance' in col]
-    features = df[['credit_achievement_rate', 'age'] + attendance_cols]
-    features = features.fillna(0)
-    scaler = StandardScaler()
-    scaled_features = scaler.fit_transform(features)
-    pca = PCA(n_components=2)
-    principal_components = pca.fit_transform(scaled_features)
-    kmeans = KMeans(n_clusters=3, random_state=42)
-    clusters = kmeans.fit_predict(principal_components)
-    df['Cluster'] = clusters
-    cluster_analysis = df.groupby('Cluster')[['credit_achievement_rate', 'age'] + attendance_cols].mean()
-    tables['cluster_analysis'] = cluster_analysis.to_string()
-    plt.figure(figsize=(8, 6))
-    sns.scatterplot(x=principal_components[:,0], y=principal_components[:,1], hue=clusters, palette='Set1', s=100, alpha=0.7)
-    plt.title('Student Clusters', fontsize=14)
-    plt.xlabel('Principal Component 1', fontsize=12)
-    plt.ylabel('Principal Component 2', fontsize=12)
-    plt.legend(title='Cluster')
-    plt.tight_layout()
-    graphs.append(plt_to_base64())
-    return graphs, tables
-
-def plot_correlation_with_credit_achievement(df):
-    graphs = []
-    tables = {}
-    numeric_cols = df.select_dtypes(include=['float64', 'int64']).columns
-    corr_matrix = df[numeric_cols].corr()
-
-    if 'credit_achievement_rate' not in corr_matrix.columns:
-        return graphs, tables
-
-    corr_matrix = corr_matrix.drop(['credit_threshold', 'Total Credits'], axis=1, errors='ignore')
-    corr_matrix = corr_matrix.drop(['credit_threshold', 'Total Credits'], axis=0, errors='ignore')
-
-    correlation_with_credit = corr_matrix[['credit_achievement_rate']].sort_values(by='credit_achievement_rate', ascending=False)
-
-    plt.figure(figsize=(8, 10))
-    sns.heatmap(correlation_with_credit, annot=True, cmap='coolwarm', fmt='.2f', annot_kws={"size": 10}, cbar=True)
-    plt.title('Correlation with Credit Achievement Rate', fontsize=16)
-    plt.xticks(rotation=45, ha='right', fontsize=10)
+        sns.histplot(df['credit_achievement_rate'], kde=True, bins=30)
+        plt.title('Distribution of Credit Achievement Rate')
+        plt.xlabel('Credit Achievement Rate')
+        plt.ylabel('Frequency')
+        buf = io.BytesIO()
+        plt.savefig(buf, format='png')
+        buf.seek(0)
+        plt.close()
+        return buf
+    else:
+        return None
+
+def plot_feature_importance(feature_importance_df):
+    """
+    Plot the top 10 important features for predicting at-risk students
+    """
+    plt.figure(figsize=(12, 6))
+    sns.barplot(data=feature_importance_df.head(10), x='Importance', y='Feature', palette='viridis')
+    plt.title('Top 10 Important Features for Predicting At-Risk Students', fontsize=14)
+    plt.xlabel('Importance', fontsize=12)
+    plt.ylabel('Feature', fontsize=12)
     plt.tight_layout()
-    graphs.append(plt_to_base64())
-
-    tables['correlation_with_credit'] = correlation_with_credit.to_string()
-
-    corr_matrix_clean = corr_matrix.replace([np.inf, -np.inf], np.nan).fillna(0)
-
-    plt.figure(figsize=(12, 12))
-    sns.clustermap(corr_matrix_clean, annot=False, cmap='coolwarm', figsize=(12, 12), method='average')
-    plt.title('Cluster Map of Feature Correlations (excluding credit_threshold, Total Credits)', fontsize=16)
-    graphs.append(plt_to_base64())
-
-    return graphs, tables
-
-def plot_top_features_vs_credit(df):
-    graphs = []
-    numeric_cols = df.select_dtypes(include=['float64', 'int64']).columns
-    corr_matrix = df[numeric_cols].corr()
-
-    corr_matrix = corr_matrix.drop(['credit_threshold', 'Total Credits'], axis=0, errors='ignore')
-    corr_matrix = corr_matrix.drop(['credit_threshold', 'Total Credits'], axis=1, errors='ignore')
-
-    top_corr_features = corr_matrix['credit_achievement_rate'].abs().sort_values(ascending=False).index[1:6]
-
-    for feature in top_corr_features:
-        if pd.api.types.is_numeric_dtype(df[feature]):
-            df_sorted = df[[feature, 'credit_achievement_rate']].sort_values(by=feature)
-
-            plt.figure(figsize=(10, 6))
-            sns.lineplot(x=df_sorted[feature], y=df_sorted['credit_achievement_rate'], marker='o')
-            plt.title(f'Line Graph: {feature} vs Credit Achievement Rate', fontsize=14)
-            plt.xlabel(feature.replace('_', ' ').title(), fontsize=12)
-            plt.ylabel('Credit Achievement Rate', fontsize=12)
-            plt.tight_layout()
-            graphs.append(plt_to_base64())
-        elif pd.api.types.is_categorical_dtype(df[feature]) or pd.api.types.is_object_dtype(df[feature]):
-            group = df.groupby(feature)['credit_achievement_rate'].mean().reset_index()
-
-            plt.figure(figsize=(10, 6))
-            sns.barplot(x=group[feature], y=group['credit_achievement_rate'], palette='Set2')
-            plt.title(f'Bar Plot: {feature} vs Credit Achievement Rate', fontsize=14)
-            plt.xlabel(feature.replace('_', ' ').title(), fontsize=12)
-            plt.ylabel('Average Credit Achievement Rate', fontsize=12)
-            plt.xticks(rotation=45)
-            plt.tight_layout()
-            graphs.append(plt_to_base64())
-
-    return graphs
-
-def plt_to_file():
-    filename = f"temp_{uuid.uuid4()}.png"
-    plt.savefig(filename)
+    buf = io.BytesIO()
+    plt.savefig(buf, format='png')
+    buf.seek(0)
     plt.close()
-    return filename
-
-def perform_comprehensive_analysis(df):
-    all_graphs = []
-    all_tables = {}
-
-    # 1. Identifying At-Risk Students
-    graphs, tables = identify_at_risk_students(df)
-    all_graphs.extend(graphs)
-    all_tables.update(tables)
-
-    # 2. Analyzing Impact of Extra-Curricular Activities
-    df = process_extra_curricular(df)
-    graphs = analyze_extra_curricular_impact(df)
-    all_graphs.extend(graphs)
-
-    # 3. Analyzing Teacher Performance
-    graphs = analyze_teacher_performance(df)
-    all_graphs.extend(graphs)
-
-    # 4. Analyzing Language Proficiency Impact
-    graphs = analyze_language_impact(df)
-    all_graphs.extend(graphs)
-
-    # 5. Performing Cluster Analysis
-    graphs, tables = perform_clustering(df)
-    all_graphs.extend(graphs)
-    all_tables.update(tables)
-
-    # 6. Correlation Analysis for Credit Achievement Rate
-    graphs, tables = plot_correlation_with_credit_achievement(df)
-    all_graphs.extend(graphs)
-    all_tables.update(tables)
-
-    # 7. Plotting Top Features vs Credit Achievement Rate
-    graphs = plot_top_features_vs_credit(df)
-    all_graphs.extend(graphs)
-
-    return all_graphs, all_tables
+    return buf
+
+def analyze_data(file):
+    try:
+        df = pd.read_csv(file)
+        df = clean_data(df)
+        # Generate classification report and feature importance
+        classification_rep, feature_importance_df = identify_at_risk_students(df)
+        # Generate plots
+        plot_buf = plot_credit_achievement_rate(df)
+        feature_importance_buf = plot_feature_importance(feature_importance_df)
+        result_text = f"Data analysis complete.\n\nClassification Report:\n{classification_rep}"
+        return result_text, plot_buf, feature_importance_buf
+    except Exception as e:
+        return f"An error occurred: {str(e)}", None, None
+
+# Gradio Interface
+def analyze_uploaded_file(file):
+    text, plot, feature_importance_plot = analyze_data(file)
+    if plot and feature_importance_plot:
+        return text, plot, feature_importance_plot
+    elif plot:
+        return text, plot, None
+    else:
+        return text, None, None
 
-def gradio_wrapper(file):
-    df = pd.read_csv(file.name)
-    df = clean_data(df)
-    graphs, tables = perform_comprehensive_analysis(df)
+with gr.Blocks() as demo:
+    gr.Markdown("""
+        # Student Data Analysis Tool
+        Upload your CSV file to analyze student data and generate insights.
+    """)
     
-    # Convert tables to a list of strings for easier display
-    table_outputs = [f"### {k}\n```\n{v}\n```" for k, v in tables.items()]
+    with gr.Row():
+        file_input = gr.File(label="Upload CSV File")
+        text_output = gr.Textbox(label="Analysis Summary")
     
-    return graphs + table_outputs
-
-# Create Gradio interface
-iface = gr.Interface(
-    fn=gradio_wrapper,
-    inputs=gr.File(label="Upload CSV"),
-    outputs=[
-        gr.Gallery(label="Graphs", columns=2, rows=3, height="auto"),
-        gr.Markdown(label="Classification Report"),
-        gr.Markdown(label="Feature Importance"),
-        gr.Markdown(label="Cluster Analysis"),
-        gr.Markdown(label="Correlation with Credit Achievement Rate")
-    ],
-    title="Comprehensive Student Data Analysis",
-    description="Upload a CSV file to analyze student data. The analysis includes identifying at-risk students, impact of extra-curricular activities, teacher performance, language proficiency impact, cluster analysis, and correlation analysis."
-)
-
-# Launch the interface
-iface.launch()
-
-# Clean up temporary files
-def cleanup_temp_files():
-    for filename in os.listdir():
-        if filename.startswith("temp_") and filename.endswith(".png"):
-            os.remove(filename)
+    with gr.Row():
+        image_output = gr.Image(label="Generated Plot", type="pil")
+        feature_importance_output = gr.Image(label="Feature Importance Plot", type="pil")
+    
+    file_input.change(analyze_uploaded_file, inputs=file_input, outputs=[text_output, image_output, feature_importance_output])
 
-# Register the cleanup function to be called when the script exits
-import atexit
-atexit.register(cleanup_temp_files)
+demo.launch()