app.py

import matplotlib
matplotlib.use('Agg')
import gradio as gr
import pandas as pd
import joblib
import matplotlib.pyplot as plt
from wordcloud import WordCloud
from src.feature_extraction import FeatureExtractor
import os
import numpy as np

# Load Model
MODEL_PATH = 'best_svm_model.joblib'
try:
    model = joblib.load(MODEL_PATH)
    print(f"Model loaded from {MODEL_PATH}")
except Exception as e:
    print(f"Error loading model: {e}")
    print("Please run src/train_deployment_model.py first.")
    model = None

feature_extractor = FeatureExtractor()

def predict_summary(title, text):
    if not title or not text:
        return "Please provide both Title and Content.", pd.DataFrame()
    
    # 1. Extract Features
    features_list = feature_extractor.calculate_features(title, text)
    if not features_list:
        return "Could not extract sentences.", pd.DataFrame()
    
    df_features = pd.DataFrame(features_list)
    
    # 2. Prepare X for prediction
    feature_cols = ['f1_lead', 'f2_judul_sim', 'f3_freq_word', 'f4_sim_sent', 'f5_len_norm', 'f6_overlap']
    X_pred = df_features[feature_cols]
    
    # 3. Predict
    if model:
        predictions = model.predict(X_pred)
        # Get probabilities for fallback
        if hasattr(model, "predict_proba"):
            probs = model.predict_proba(X_pred)[:, 1] # Probability of class 1
        else:
            probs = model.decision_function(X_pred)
    else:
        return "Model not loaded.", pd.DataFrame()
    
    df_features['prediction'] = predictions
    df_features['probability'] = probs
    
    # 4. Generate Summary Text
    summary_sentences = df_features[df_features['prediction'] == 1]['teks_kalimat'].tolist()
    
    # Fallback Logic: If no sentences selected
    if not summary_sentences:
        best_idx = np.argmax(probs)
        summary_sentences = [df_features.iloc[best_idx]['teks_kalimat']]
        df_features.at[best_idx, 'prediction'] = 1
        
    summary_text = " ".join(summary_sentences)
        
    return summary_text, df_features[['teks_kalimat', 'prediction', 'probability'] + feature_cols]


def process_batch_file(file_path):
    # Initialize returns for consistent tuple size (8 items: 7 data + 1 status message)
    # output_csv, ui_df, df_results, fig1, fig2, fig3, fig4, status_msg
    empty_returns = (None, None, None, None, None, None, None, "No file uploaded.")
    
    if file_path is None:
        return empty_returns
    
    if hasattr(file_path, 'name'):
        file_path_str = file_path.name
    else:
        file_path_str = str(file_path)

    # Read File
    try:
        if file_path_str.endswith('.csv'):
            df = pd.read_csv(file_path_str)
        elif file_path_str.endswith('.xlsx'):
            df = pd.read_excel(file_path_str)
        else:
            return None, None, None, None, None, None, None, "Invalid file format. Please upload .csv or .xlsx"
    except Exception as e:
        print(f"Error reading file: {e}")
        return None, None, None, None, None, None, None, f"Error reading file: {str(e)}"

    # --- Column Detection & Logic ---
    cols = [c.lower() for c in df.columns]
    
    # Case A: Sentence-level data (raw corpus) -> Needs Aggregation
    # Check for ID columns AND Sentence columns
    is_sentence_level = False
    id_col = next((c for c in df.columns if c.lower() in ['id_berita', 'id', 'article_id']), None)
    sent_col = next((c for c in df.columns if c.lower() in ['teks_kalimat', 'sentence', 'kalimat', 'text']), None)
    
    if id_col and sent_col:
        is_sentence_level = True
    
    df_processed = pd.DataFrame(columns=['title', 'content'])
    
    if is_sentence_level:
        # Check for title column in raw data
        title_col = next((c for c in df.columns if c.lower() in ['id_judul', 'judul', 'title', 'headline']), None)
        
        # Aggregate
        # Group by ID
        grouped = df.groupby(id_col)
        processed_rows = []
        
        for name, group in grouped:
            # Extract Title (first non-null)
            title_text = "Untitled"
            if title_col:
                possible_titles = group[title_col].dropna().unique()
                if len(possible_titles) > 0:
                    title_text = possible_titles[0]
            
            # Combine sentences
            sentences = group[sent_col].dropna().astype(str).tolist()
            content_text = " ".join(sentences)
            
            processed_rows.append({'title': title_text, 'content': content_text, 'original_id': name})
            
        df_processed = pd.DataFrame(processed_rows)
        status_msg = f"Detected sentence-level data. Aggregated {len(df)} sentences into {len(df_processed)} articles."
        
    else:
        # Case B: Article-level data -> Direct Mapping
        title_col = next((c for c in df.columns if c.lower() in ['title', 'judul', 'headline']), None)
        content_col = next((c for c in df.columns if c.lower() in ['content', 'isi', 'text', 'full_text', 'artikel']), None)
        
        if not title_col or not content_col:
            found_cols = ", ".join(df.columns)
            return None, None, None, None, None, None, None, f"Could not find required columns. Found: {found_cols}. Need 'title'/'judul' and 'content'/'isi'/'teks_kalimat'."
            
        df_processed = df.rename(columns={title_col: 'title', content_col: 'content'})
        status_msg = f"Processed {len(df_processed)} articles."

    # --- Summarization Logic ---
    all_results = []
    
    for idx, row in df_processed.iterrows():
        title_raw = row['title']
        content_raw = row['content']
        
        # Validation
        if pd.isna(title_raw) or pd.isna(content_raw):
            continue
        
        title = str(title_raw).strip()
        content = str(content_raw).strip()
        
        if not title or not content or content.lower() == 'nan':
            continue
        
        features_list = feature_extractor.calculate_features(title, content)
        if not features_list:
            continue
            
        df_feat = pd.DataFrame(features_list)
        X_pred = df_feat[['f1_lead', 'f2_judul_sim', 'f3_freq_word', 'f4_sim_sent', 'f5_len_norm', 'f6_overlap']]
        
        if model:
            preds = model.predict(X_pred)
            if hasattr(model, "predict_proba"):
                probs = model.predict_proba(X_pred)[:, 1]
            else:
                probs = model.decision_function(X_pred)
                
            df_feat['prediction'] = preds
            df_feat['probability'] = probs
            
            summary_sents = df_feat[df_feat['prediction'] == 1]['teks_kalimat'].tolist()
            
            if not summary_sents:
                best_idx = np.argmax(probs)
                summary_sents = [df_feat.iloc[best_idx]['teks_kalimat']]
            
            summary_text = " ".join(summary_sents)
            
            result_row = {
                'article_id': row.get('original_id', idx),
                'title': title,
                'original_content': content,
                'generated_summary': summary_text,
                'original_length': len(content.split()),
                'summary_length': len(summary_text.split()),
                'compression_ratio': len(summary_text.split()) / len(content.split()) if len(content.split()) > 0 else 0,
                'num_sentences_original': len(features_list),
                'num_sentences_summary': len(summary_sents)
            }
            all_results.append(result_row)
            
    if not all_results:
        return None, pd.DataFrame(), pd.DataFrame(), None, None, None, None, status_msg + " But no valid rows processed."

    df_results = pd.DataFrame(all_results)
    output_csv = "batch_results.csv"
    df_results.to_csv(output_csv, index=False)
    
    # Analysis Plots
    # 1. Ratio Distribution
    fig1, ax1 = plt.subplots(figsize=(6, 4))
    ax1.hist(df_results['compression_ratio'], bins=10, color='skyblue', edgecolor='black')
    ax1.set_title('Distribution of Compression Ratios')
    ax1.set_xlabel('Ratio')
    ax1.set_ylabel('Frequency')
    plt.tight_layout()
    
    # 2. Length Scatter
    fig2, ax2 = plt.subplots(figsize=(6, 4))
    ax2.scatter(df_results['original_length'], df_results['summary_length'], alpha=0.5)
    ax2.set_title('Original vs Summary Length (Words)')
    ax2.set_xlabel('Original Length')
    ax2.set_ylabel('Summary Length')
    plt.tight_layout()
    
    # 3. Sentence Count Comparison
    fig3, ax3 = plt.subplots(figsize=(6, 4))
    indices = range(len(df_results))
    if len(df_results) > 20:
        plot_df = df_results.head(20)
        indices = range(20)
    else:
        plot_df = df_results
        
    width = 0.35
    # Use string casting for x-axis if article_ids are weird strings
    x_labels = [str(x) for x in plot_df['article_id']]
    ax3.bar(indices, plot_df['num_sentences_original'], width, label='Original')
    ax3.bar([i + width for i in indices], plot_df['num_sentences_summary'], width, label='Summary')
    ax3.set_title('Sentence Count Comparison (First 20)')
    ax3.set_xticks([i + width/2 for i in indices])
    ax3.set_xticklabels(x_labels, rotation=45, ha='right')
    ax3.set_ylabel('Count')
    ax3.legend()
    plt.tight_layout()

    # 4. Word Cloud
    all_summary_text = " ".join(df_results['generated_summary'].tolist())
    if all_summary_text.strip():
        wordcloud = WordCloud(width=800, height=400, background_color='white').generate(all_summary_text)
        fig4, ax4 = plt.subplots(figsize=(10, 5))
        ax4.imshow(wordcloud, interpolation='bilinear')
        ax4.axis('off')
        ax4.set_title('Word Cloud of Summaries')
    else:
        fig4 = None

    ui_df = df_results[['article_id', 'title', 'generated_summary', 'compression_ratio', 'num_sentences_original', 'num_sentences_summary']]
    
    return output_csv, ui_df, df_results, fig1, fig2, fig3, fig4, status_msg

def get_template():
    df = pd.DataFrame([{'title': 'Judul Berita', 'content': 'Isi berita lengkap...'}])
    output_path = 'template.xlsx'
    df.to_excel(output_path, index=False)
    return output_path

def show_details(evt: gr.SelectData, df):
    if df is None or evt.index[0] >= len(df):
        return "", "", ""
    row = df.iloc[evt.index[0]]
    return row['title'], row['original_content'], row['generated_summary']

# Gradio Interface
with gr.Blocks(title="Balinese News Summarizer") as demo:
    gr.Markdown("# 📰 Balinese News Summarizer (SVM)")
    
    with gr.Tab("📝 Manual Input"):
        with gr.Row():
            with gr.Column():
                input_title = gr.Textbox(label="News Title", placeholder="Enter title here...")
                input_content = gr.Textbox(label="News Content", placeholder="Enter full news text here...", lines=10)
                btn_predict = gr.Button("Summarize", variant="primary")
            
            with gr.Column():
                output_summary = gr.Textbox(label="Generated Summary", lines=5, interactive=False)
                output_df = gr.Dataframe(label="Sentence Analysis", headers=['teks_kalimat', 'prediction', 'probability', 'f1', 'f2', 'f3', 'f4', 'f5', 'f6'])
        
        btn_predict.click(fn=predict_summary, inputs=[input_title, input_content], outputs=[output_summary, output_df])

    with gr.Tab("📂 Batch Analysis"):
        gr.Markdown("Upload a CSV/Excel file. Supports both **Article-level** (columns: title, content) and **Sentence-level** (columns: id_berita, id_judul, teks_kalimat) formats.")
        
        batch_results_state = gr.State()
        
        with gr.Row():
            file_upload = gr.File(label="Upload File", file_types=['.csv', '.xlsx'])
            with gr.Column():
                btn_template = gr.Button("Download Excel Template")
                btn_process = gr.Button("Process Batch", variant="primary")
        
        # Status Message
        status_output = gr.Textbox(label="Status", interactive=False)
        
        # Output Section
        with gr.Row():
            download_result = gr.File(label="Download Results (CSV)")
        
        gr.Markdown("### Batch Results Preview (Select a row to view details below)")
        batch_output_df = gr.Dataframe(label="Results Table", interactive=False)
        
        # Detailed View
        with gr.Group():
            gr.Markdown("### Selected Article Details")
            with gr.Row():
                detail_title = gr.Textbox(label="Title", interactive=False)
            with gr.Row():
                detail_original = gr.Textbox(label="Original Content", lines=5, interactive=False)
                detail_summary = gr.Textbox(label="Generated Summary", lines=5, interactive=False)

        # Plots
        gr.Markdown("### Analysis Visualization")
        with gr.Row():
            plot1 = gr.Plot(label="Compression Ratio Distribution")
            plot2 = gr.Plot(label="Length Comparison")
        
        with gr.Row():
            plot3 = gr.Plot(label="Sentence Count Comparison")
        
        with gr.Row():
            plot4 = gr.Plot(label="Word Cloud")

        # Event Wiring
        btn_template.click(fn=get_template, inputs=None, outputs=file_upload)
        
        btn_process.click(
            fn=process_batch_file, 
            inputs=file_upload, 
            outputs=[download_result, batch_output_df, batch_results_state, plot1, plot2, plot3, plot4, status_output]
        )
        
        # Row Selection Event
        batch_output_df.select(
            fn=show_details,
            inputs=[batch_results_state],
            outputs=[detail_title, detail_original, detail_summary]
        )

if __name__ == "__main__":
    demo.launch(server_name="0.0.0.0", server_port=7860)