import matplotlib matplotlib.use('Agg') import gradio as gr import pandas as pd import joblib import matplotlib.pyplot as plt from wordcloud import WordCloud from src.feature_extraction import FeatureExtractor import os import numpy as np # Load Model MODEL_PATH = 'best_svm_model.joblib' try: model = joblib.load(MODEL_PATH) print(f"Model loaded from {MODEL_PATH}") except Exception as e: print(f"Error loading model: {e}") print("Please run src/train_deployment_model.py first.") model = None feature_extractor = FeatureExtractor() def predict_summary(title, text): if not title or not text: return "Please provide both Title and Content.", pd.DataFrame() # 1. Extract Features features_list = feature_extractor.calculate_features(title, text) if not features_list: return "Could not extract sentences.", pd.DataFrame() df_features = pd.DataFrame(features_list) # 2. Prepare X for prediction feature_cols = ['f1_lead', 'f2_judul_sim', 'f3_freq_word', 'f4_sim_sent', 'f5_len_norm', 'f6_overlap'] X_pred = df_features[feature_cols] # 3. Predict if model: predictions = model.predict(X_pred) # Get probabilities for fallback if hasattr(model, "predict_proba"): probs = model.predict_proba(X_pred)[:, 1] # Probability of class 1 else: probs = model.decision_function(X_pred) else: return "Model not loaded.", pd.DataFrame() df_features['prediction'] = predictions df_features['probability'] = probs # 4. Generate Summary Text summary_sentences = df_features[df_features['prediction'] == 1]['teks_kalimat'].tolist() # Fallback Logic: If no sentences selected if not summary_sentences: best_idx = np.argmax(probs) summary_sentences = [df_features.iloc[best_idx]['teks_kalimat']] df_features.at[best_idx, 'prediction'] = 1 summary_text = " ".join(summary_sentences) return summary_text, df_features[['teks_kalimat', 'prediction', 'probability'] + feature_cols] def process_batch_file(file_path): # Initialize returns for consistent tuple size (8 items: 7 data + 1 status message) # output_csv, ui_df, df_results, fig1, fig2, fig3, fig4, status_msg empty_returns = (None, None, None, None, None, None, None, "No file uploaded.") if file_path is None: return empty_returns if hasattr(file_path, 'name'): file_path_str = file_path.name else: file_path_str = str(file_path) # Read File try: if file_path_str.endswith('.csv'): df = pd.read_csv(file_path_str) elif file_path_str.endswith('.xlsx'): df = pd.read_excel(file_path_str) else: return None, None, None, None, None, None, None, "Invalid file format. Please upload .csv or .xlsx" except Exception as e: print(f"Error reading file: {e}") return None, None, None, None, None, None, None, f"Error reading file: {str(e)}" # --- Column Detection & Logic --- cols = [c.lower() for c in df.columns] # Case A: Sentence-level data (raw corpus) -> Needs Aggregation # Check for ID columns AND Sentence columns is_sentence_level = False id_col = next((c for c in df.columns if c.lower() in ['id_berita', 'id', 'article_id']), None) sent_col = next((c for c in df.columns if c.lower() in ['teks_kalimat', 'sentence', 'kalimat', 'text']), None) if id_col and sent_col: is_sentence_level = True df_processed = pd.DataFrame(columns=['title', 'content']) if is_sentence_level: # Check for title column in raw data title_col = next((c for c in df.columns if c.lower() in ['id_judul', 'judul', 'title', 'headline']), None) # Aggregate # Group by ID grouped = df.groupby(id_col) processed_rows = [] for name, group in grouped: # Extract Title (first non-null) title_text = "Untitled" if title_col: possible_titles = group[title_col].dropna().unique() if len(possible_titles) > 0: title_text = possible_titles[0] # Combine sentences sentences = group[sent_col].dropna().astype(str).tolist() content_text = " ".join(sentences) processed_rows.append({'title': title_text, 'content': content_text, 'original_id': name}) df_processed = pd.DataFrame(processed_rows) status_msg = f"Detected sentence-level data. Aggregated {len(df)} sentences into {len(df_processed)} articles." else: # Case B: Article-level data -> Direct Mapping title_col = next((c for c in df.columns if c.lower() in ['title', 'judul', 'headline']), None) content_col = next((c for c in df.columns if c.lower() in ['content', 'isi', 'text', 'full_text', 'artikel']), None) if not title_col or not content_col: found_cols = ", ".join(df.columns) return None, None, None, None, None, None, None, f"Could not find required columns. Found: {found_cols}. Need 'title'/'judul' and 'content'/'isi'/'teks_kalimat'." df_processed = df.rename(columns={title_col: 'title', content_col: 'content'}) status_msg = f"Processed {len(df_processed)} articles." # --- Summarization Logic --- all_results = [] for idx, row in df_processed.iterrows(): title_raw = row['title'] content_raw = row['content'] # Validation if pd.isna(title_raw) or pd.isna(content_raw): continue title = str(title_raw).strip() content = str(content_raw).strip() if not title or not content or content.lower() == 'nan': continue features_list = feature_extractor.calculate_features(title, content) if not features_list: continue df_feat = pd.DataFrame(features_list) X_pred = df_feat[['f1_lead', 'f2_judul_sim', 'f3_freq_word', 'f4_sim_sent', 'f5_len_norm', 'f6_overlap']] if model: preds = model.predict(X_pred) if hasattr(model, "predict_proba"): probs = model.predict_proba(X_pred)[:, 1] else: probs = model.decision_function(X_pred) df_feat['prediction'] = preds df_feat['probability'] = probs summary_sents = df_feat[df_feat['prediction'] == 1]['teks_kalimat'].tolist() if not summary_sents: best_idx = np.argmax(probs) summary_sents = [df_feat.iloc[best_idx]['teks_kalimat']] summary_text = " ".join(summary_sents) result_row = { 'article_id': row.get('original_id', idx), 'title': title, 'original_content': content, 'generated_summary': summary_text, 'original_length': len(content.split()), 'summary_length': len(summary_text.split()), 'compression_ratio': len(summary_text.split()) / len(content.split()) if len(content.split()) > 0 else 0, 'num_sentences_original': len(features_list), 'num_sentences_summary': len(summary_sents) } all_results.append(result_row) if not all_results: return None, pd.DataFrame(), pd.DataFrame(), None, None, None, None, status_msg + " But no valid rows processed." df_results = pd.DataFrame(all_results) output_csv = "batch_results.csv" df_results.to_csv(output_csv, index=False) # Analysis Plots # 1. Ratio Distribution fig1, ax1 = plt.subplots(figsize=(6, 4)) ax1.hist(df_results['compression_ratio'], bins=10, color='skyblue', edgecolor='black') ax1.set_title('Distribution of Compression Ratios') ax1.set_xlabel('Ratio') ax1.set_ylabel('Frequency') plt.tight_layout() # 2. Length Scatter fig2, ax2 = plt.subplots(figsize=(6, 4)) ax2.scatter(df_results['original_length'], df_results['summary_length'], alpha=0.5) ax2.set_title('Original vs Summary Length (Words)') ax2.set_xlabel('Original Length') ax2.set_ylabel('Summary Length') plt.tight_layout() # 3. Sentence Count Comparison fig3, ax3 = plt.subplots(figsize=(6, 4)) indices = range(len(df_results)) if len(df_results) > 20: plot_df = df_results.head(20) indices = range(20) else: plot_df = df_results width = 0.35 # Use string casting for x-axis if article_ids are weird strings x_labels = [str(x) for x in plot_df['article_id']] ax3.bar(indices, plot_df['num_sentences_original'], width, label='Original') ax3.bar([i + width for i in indices], plot_df['num_sentences_summary'], width, label='Summary') ax3.set_title('Sentence Count Comparison (First 20)') ax3.set_xticks([i + width/2 for i in indices]) ax3.set_xticklabels(x_labels, rotation=45, ha='right') ax3.set_ylabel('Count') ax3.legend() plt.tight_layout() # 4. Word Cloud all_summary_text = " ".join(df_results['generated_summary'].tolist()) if all_summary_text.strip(): wordcloud = WordCloud(width=800, height=400, background_color='white').generate(all_summary_text) fig4, ax4 = plt.subplots(figsize=(10, 5)) ax4.imshow(wordcloud, interpolation='bilinear') ax4.axis('off') ax4.set_title('Word Cloud of Summaries') else: fig4 = None ui_df = df_results[['article_id', 'title', 'generated_summary', 'compression_ratio', 'num_sentences_original', 'num_sentences_summary']] return output_csv, ui_df, df_results, fig1, fig2, fig3, fig4, status_msg def get_template(): df = pd.DataFrame([{'title': 'Judul Berita', 'content': 'Isi berita lengkap...'}]) output_path = 'template.xlsx' df.to_excel(output_path, index=False) return output_path def show_details(evt: gr.SelectData, df): if df is None or evt.index[0] >= len(df): return "", "", "" row = df.iloc[evt.index[0]] return row['title'], row['original_content'], row['generated_summary'] # Gradio Interface with gr.Blocks(title="Balinese News Summarizer") as demo: gr.Markdown("# 📰 Balinese News Summarizer (SVM)") with gr.Tab("📝 Manual Input"): with gr.Row(): with gr.Column(): input_title = gr.Textbox(label="News Title", placeholder="Enter title here...") input_content = gr.Textbox(label="News Content", placeholder="Enter full news text here...", lines=10) btn_predict = gr.Button("Summarize", variant="primary") with gr.Column(): output_summary = gr.Textbox(label="Generated Summary", lines=5, interactive=False) output_df = gr.Dataframe(label="Sentence Analysis", headers=['teks_kalimat', 'prediction', 'probability', 'f1', 'f2', 'f3', 'f4', 'f5', 'f6']) btn_predict.click(fn=predict_summary, inputs=[input_title, input_content], outputs=[output_summary, output_df]) with gr.Tab("📂 Batch Analysis"): gr.Markdown("Upload a CSV/Excel file. Supports both **Article-level** (columns: title, content) and **Sentence-level** (columns: id_berita, id_judul, teks_kalimat) formats.") batch_results_state = gr.State() with gr.Row(): file_upload = gr.File(label="Upload File", file_types=['.csv', '.xlsx']) with gr.Column(): btn_template = gr.Button("Download Excel Template") btn_process = gr.Button("Process Batch", variant="primary") # Status Message status_output = gr.Textbox(label="Status", interactive=False) # Output Section with gr.Row(): download_result = gr.File(label="Download Results (CSV)") gr.Markdown("### Batch Results Preview (Select a row to view details below)") batch_output_df = gr.Dataframe(label="Results Table", interactive=False) # Detailed View with gr.Group(): gr.Markdown("### Selected Article Details") with gr.Row(): detail_title = gr.Textbox(label="Title", interactive=False) with gr.Row(): detail_original = gr.Textbox(label="Original Content", lines=5, interactive=False) detail_summary = gr.Textbox(label="Generated Summary", lines=5, interactive=False) # Plots gr.Markdown("### Analysis Visualization") with gr.Row(): plot1 = gr.Plot(label="Compression Ratio Distribution") plot2 = gr.Plot(label="Length Comparison") with gr.Row(): plot3 = gr.Plot(label="Sentence Count Comparison") with gr.Row(): plot4 = gr.Plot(label="Word Cloud") # Event Wiring btn_template.click(fn=get_template, inputs=None, outputs=file_upload) btn_process.click( fn=process_batch_file, inputs=file_upload, outputs=[download_result, batch_output_df, batch_results_state, plot1, plot2, plot3, plot4, status_output] ) # Row Selection Event batch_output_df.select( fn=show_details, inputs=[batch_results_state], outputs=[detail_title, detail_original, detail_summary] ) if __name__ == "__main__": demo.launch(server_name="0.0.0.0", server_port=7860)