Update app.py
Browse files
app.py
CHANGED
|
@@ -5,10 +5,11 @@ import pandas as pd
|
|
| 5 |
from datetime import datetime
|
| 6 |
import io
|
| 7 |
import re
|
| 8 |
-
import google.generativeai as genai
|
| 9 |
-
import faiss
|
| 10 |
-
import numpy as np
|
| 11 |
-
import time
|
|
|
|
| 12 |
|
| 13 |
# --- File Generation Libraries ---
|
| 14 |
from fpdf import FPDF
|
|
@@ -18,15 +19,14 @@ from docx.enum.text import WD_ALIGN_PARAGRAPH
|
|
| 18 |
from docx.shared import Pt
|
| 19 |
from docx.enum.text import WD_PARAGRAPH_ALIGNMENT
|
| 20 |
|
| 21 |
-
# Load environment variables
|
| 22 |
load_dotenv()
|
| 23 |
|
| 24 |
# --- RAG Setup ---
|
| 25 |
rag_initialized = False
|
| 26 |
gemini_llm_model = None
|
| 27 |
-
gemini_embedding_model_name = "models/embedding-001"
|
| 28 |
|
| 29 |
-
# Attempt to get API key from environment BEFORE trying to initialize
|
| 30 |
google_api_key = os.getenv("GOOGLE_API_KEY")
|
| 31 |
|
| 32 |
if not google_api_key:
|
|
@@ -41,7 +41,6 @@ else:
|
|
| 41 |
|
| 42 |
gemini_llm_model = genai.GenerativeModel(llm_model_id)
|
| 43 |
|
| 44 |
-
# Test embedding model to ensure it's accessible
|
| 45 |
test_embedding = genai.embed_content(model=gemini_embedding_model_name, content=["test"])
|
| 46 |
if test_embedding and test_embedding['embedding']:
|
| 47 |
rag_initialized = True
|
|
@@ -55,15 +54,15 @@ else:
|
|
| 55 |
print(f"Please verify your GOOGLE_API_KEY's validity and ensure the specified AI model ('{llm_model_id}' for LLM) and embedding model ('{gemini_embedding_model_name}') are available to your API key.")
|
| 56 |
print("RAG functionality will not work without a valid Google API key and accessible model.")
|
| 57 |
|
| 58 |
-
|
| 59 |
# Global variable to store the PARSED DATAFRAME for download functions
|
|
|
|
| 60 |
global_analysis_df_for_export = pd.DataFrame()
|
| 61 |
-
global_raw_markdown_for_export = ""
|
| 62 |
|
| 63 |
# --- Custom Document Structure ---
|
| 64 |
# We'll use a dictionary: {"page_content": "...", "metadata": {...}}
|
| 65 |
|
| 66 |
-
# --- Custom Text Splitter ---
|
| 67 |
def custom_character_text_splitter(text_documents, chunk_size=1000, chunk_overlap=200):
|
| 68 |
"""
|
| 69 |
A simple character-based text splitter.
|
|
@@ -75,29 +74,25 @@ def custom_character_text_splitter(text_documents, chunk_size=1000, chunk_overla
|
|
| 75 |
content = doc["page_content"]
|
| 76 |
metadata = doc["metadata"]
|
| 77 |
|
| 78 |
-
# If content is already smaller than chunk_size, just add it
|
| 79 |
if len(content) <= chunk_size:
|
| 80 |
chunks.append({"page_content": content, "metadata": metadata})
|
| 81 |
continue
|
| 82 |
|
| 83 |
-
# Split content into chunks with overlap
|
| 84 |
current_position = 0
|
| 85 |
while current_position < len(content):
|
| 86 |
chunk_end = min(current_position + chunk_size, len(content))
|
| 87 |
chunk_text = content[current_position:chunk_end]
|
| 88 |
chunks.append({"page_content": chunk_text, "metadata": metadata})
|
| 89 |
|
| 90 |
-
|
| 91 |
-
if chunk_end == len(content): # Reached end of content
|
| 92 |
break
|
| 93 |
current_position += (chunk_size - chunk_overlap)
|
| 94 |
-
# Ensure we don't start the next chunk beyond the content length
|
| 95 |
if current_position >= len(content):
|
| 96 |
break
|
| 97 |
return chunks
|
| 98 |
|
| 99 |
|
| 100 |
-
# --- Custom FAISS Integration ---
|
| 101 |
class CustomFAISSVectorStore:
|
| 102 |
def __init__(self, embedding_model_name, dimension=768): # Gemini embedding-001 has 768 dimensions
|
| 103 |
self.embedding_model_name = embedding_model_name
|
|
@@ -136,7 +131,6 @@ class CustomFAISSVectorStore:
|
|
| 136 |
if not query_text:
|
| 137 |
return []
|
| 138 |
|
| 139 |
-
# Embed query
|
| 140 |
try:
|
| 141 |
query_embedding_response = genai.embed_content(
|
| 142 |
model=self.embedding_model_name,
|
|
@@ -149,73 +143,71 @@ class CustomFAISSVectorStore:
|
|
| 149 |
if query_embedding.shape[1] != self.index.d:
|
| 150 |
raise ValueError(f"Query embedding dimension mismatch: Expected {self.index.d}, got {query_embedding.shape[1]}.")
|
| 151 |
|
| 152 |
-
# Perform search
|
| 153 |
distances, indices = self.index.search(query_embedding, k)
|
| 154 |
|
| 155 |
retrieved_docs = []
|
| 156 |
for i in indices[0]:
|
| 157 |
-
if 0 <= i < len(self.documents):
|
| 158 |
retrieved_docs.append(self.documents[i])
|
| 159 |
return retrieved_docs
|
| 160 |
|
| 161 |
|
| 162 |
-
def
|
| 163 |
"""
|
| 164 |
-
Loads and returns text from Excel or CSV files
|
| 165 |
-
|
| 166 |
-
|
| 167 |
"""
|
| 168 |
print(f"Attempting to load customer ticket data: {filename} with extension {file_extension}")
|
| 169 |
-
raw_documents = []
|
| 170 |
|
| 171 |
if file_extension in ['.xlsx', '.xls']:
|
| 172 |
try:
|
| 173 |
-
|
| 174 |
-
|
| 175 |
except Exception as e:
|
| 176 |
raise ValueError(f"Could not process Excel file '{filename}': {e}. Ensure it's a valid Excel format for ticket data.")
|
| 177 |
elif file_extension == '.csv':
|
| 178 |
try:
|
| 179 |
-
|
| 180 |
-
print(f"CSV file '{filename}' loaded with {len(df)} rows.")
|
| 181 |
except Exception as e:
|
| 182 |
raise ValueError(f"Could not process CSV file '{filename}': {e}. Ensure it's a valid CSV format for ticket data.")
|
| 183 |
else:
|
| 184 |
raise ValueError(f"Unsupported file type: {file_extension}. Only .xlsx, .xls, or .csv files are accepted for ticket data.")
|
| 185 |
|
| 186 |
-
|
| 187 |
-
|
| 188 |
-
|
| 189 |
-
|
| 190 |
-
|
| 191 |
-
ticket_id = f"Row {index+1}"
|
| 192 |
|
| 193 |
-
for
|
| 194 |
-
|
| 195 |
-
|
| 196 |
-
|
| 197 |
-
|
| 198 |
-
if
|
| 199 |
-
|
| 200 |
-
|
| 201 |
-
|
| 202 |
-
|
| 203 |
-
|
| 204 |
-
|
| 205 |
-
|
| 206 |
-
|
| 207 |
-
|
|
|
|
| 208 |
|
| 209 |
-
|
| 210 |
-
|
|
|
|
|
|
|
|
|
|
| 211 |
|
| 212 |
-
|
| 213 |
-
|
| 214 |
-
if
|
| 215 |
-
raise ValueError("
|
| 216 |
-
|
| 217 |
-
print(f"File '{filename}' processed into {len(raw_documents)} raw documents (one per ticket).")
|
| 218 |
-
return raw_documents
|
| 219 |
|
| 220 |
|
| 221 |
def generate_summary_html_by_category(df):
|
|
@@ -227,13 +219,15 @@ def generate_summary_html_by_category(df):
|
|
| 227 |
return "<p style='color:orange;'>No categories found or data is empty for detailed summarization. This could be due to issues with the AI's output parsing or content generation.</p>"
|
| 228 |
|
| 229 |
summary_html = ""
|
|
|
|
| 230 |
required_cols = ['Issue Description (Summary)', 'Sub-Category', 'Root Cause (Inferred)',
|
| 231 |
'Potential Resolution Steps', 'Future Prediction', 'Proactive Measures', 'Count']
|
| 232 |
for col in required_cols:
|
| 233 |
if col not in df.columns:
|
| 234 |
-
df[col] =
|
| 235 |
-
if col == 'Count': df[col] =
|
| 236 |
|
|
|
|
| 237 |
for category, category_df in df.groupby('Category'):
|
| 238 |
total_count = category_df['Count'].sum()
|
| 239 |
|
|
@@ -258,8 +252,7 @@ def generate_summary_html_by_category(df):
|
|
| 258 |
unique_root_causes = category_df['Root Cause (Inferred)']\
|
| 259 |
.astype(str)\
|
| 260 |
.drop_duplicates()\
|
| 261 |
-
.apply(lambda x: x.replace('*', ''))
|
| 262 |
-
.tolist()
|
| 263 |
for rc in unique_root_causes:
|
| 264 |
root_causes += f"<li>{rc}</li>"
|
| 265 |
root_causes += "</ul>" if unique_root_causes else "<p>Undetermined.</p>"
|
|
@@ -268,8 +261,7 @@ def generate_summary_html_by_category(df):
|
|
| 268 |
unique_resolutions = category_df['Potential Resolution Steps']\
|
| 269 |
.astype(str)\
|
| 270 |
.drop_duplicates()\
|
| 271 |
-
.apply(lambda x: x.replace('*', ''))
|
| 272 |
-
.tolist()
|
| 273 |
for res in unique_resolutions:
|
| 274 |
resolution_steps += f"<li>{res}</li>"
|
| 275 |
resolution_steps += "</ul>" if unique_resolutions else "<p>No specific steps identified.</p>"
|
|
@@ -321,49 +313,41 @@ def generate_summary_html_by_category(df):
|
|
| 321 |
def parse_markdown_table_to_df(markdown_string):
|
| 322 |
"""
|
| 323 |
Robustly parses a markdown table string into a pandas DataFrame.
|
|
|
|
| 324 |
"""
|
| 325 |
lines = markdown_string.strip().split('\n')
|
| 326 |
|
| 327 |
header_line = None
|
| 328 |
separator_line_index = -1
|
| 329 |
|
| 330 |
-
# Find header and separator lines
|
| 331 |
for i, line in enumerate(lines):
|
| 332 |
if '|' in line and '---' not in line and not line.strip().startswith('|---'):
|
| 333 |
header_line = line.strip()
|
| 334 |
elif '|---' in line:
|
| 335 |
separator_line_index = i
|
| 336 |
-
break
|
| 337 |
|
| 338 |
if header_line is None or separator_line_index == -1:
|
| 339 |
return pd.DataFrame(), "No valid Markdown table header or separator found."
|
| 340 |
|
| 341 |
-
# Extract data rows after the separator
|
| 342 |
data_lines = []
|
| 343 |
for line in lines[separator_line_index + 1:]:
|
| 344 |
if line.strip().startswith('|') and '|' in line.strip():
|
| 345 |
data_lines.append(line.strip())
|
| 346 |
else:
|
| 347 |
-
# Stop if we encounter a non-table line, assuming table has ended
|
| 348 |
break
|
| 349 |
|
| 350 |
if not data_lines:
|
| 351 |
return pd.DataFrame(), "No data rows found in the Markdown table."
|
| 352 |
|
| 353 |
-
# Process header
|
| 354 |
-
# Clean header line to split correctly and handle potential leading/trailing pipes
|
| 355 |
headers = [h.strip() for h in header_line.strip('|').split('|')]
|
| 356 |
-
|
| 357 |
-
headers = [h for h in headers if h]
|
| 358 |
|
| 359 |
-
# Process data rows
|
| 360 |
data = []
|
| 361 |
for line in data_lines:
|
| 362 |
-
# Clean data line similarly
|
| 363 |
row_values = [v.strip() for v in line.strip('|').split('|')]
|
| 364 |
-
row_values = [v for v in row_values if v]
|
| 365 |
|
| 366 |
-
# Ensure row has the same number of columns as headers
|
| 367 |
if len(row_values) == len(headers):
|
| 368 |
data.append(row_values)
|
| 369 |
else:
|
|
@@ -373,8 +357,8 @@ def parse_markdown_table_to_df(markdown_string):
|
|
| 373 |
return pd.DataFrame(), "No valid data rows parsed after cleaning."
|
| 374 |
|
| 375 |
df = pd.DataFrame(data, columns=headers)
|
| 376 |
-
df = df.dropna(axis=1, how='all')
|
| 377 |
-
df.columns = df.columns.str.strip()
|
| 378 |
|
| 379 |
return df, None
|
| 380 |
|
|
@@ -382,16 +366,14 @@ def parse_markdown_table_to_df(markdown_string):
|
|
| 382 |
def perform_ticket_analysis(file, user_query, progress=gr.Progress()):
|
| 383 |
"""
|
| 384 |
Main function to analyze customer tickets using custom RAG implementation and format output as rich HTML.
|
| 385 |
-
|
| 386 |
-
Includes progress updates for Gradio UI.
|
| 387 |
"""
|
| 388 |
global global_analysis_df_for_export
|
| 389 |
-
global global_raw_markdown_for_export
|
| 390 |
|
| 391 |
start_time = time.time()
|
| 392 |
error_message_markdown = ""
|
| 393 |
|
| 394 |
-
# Initialize progress bar
|
| 395 |
progress(0, desc="Starting analysis...")
|
| 396 |
|
| 397 |
if not rag_initialized:
|
|
@@ -406,145 +388,245 @@ def perform_ticket_analysis(file, user_query, progress=gr.Progress()):
|
|
| 406 |
error_message_markdown = "Please upload a customer ticket file."
|
| 407 |
return f"<p style='color:red;'>{error_message_markdown}</p>"
|
| 408 |
|
| 409 |
-
# Default comprehensive analysis if query is empty
|
| 410 |
-
if not user_query.strip():
|
| 411 |
-
user_query = "Perform a comprehensive analysis of the tickets. Provide categorization, sub-categories, count of issues, detailed description of issues, infer the root cause, suggest potential resolution steps, identify future prediction, and propose proactive measures. Ensure the output is a Markdown table with columns: 'Category', 'Sub-Category', 'Count', 'Issue Description (Summary)', 'Root Cause (Inferred)', 'Potential Resolution Steps', 'Future Prediction', 'Proactive Measures'."
|
| 412 |
-
|
| 413 |
file_path = file.name
|
| 414 |
filename = os.path.basename(file_path)
|
| 415 |
file_extension = os.path.splitext(filename)[1].lower()
|
| 416 |
|
| 417 |
-
raw_rag_output = ""
|
| 418 |
disclaimer_text_for_raw = "\n\n**Note:** 'Root Cause', 'Future Prediction', and 'Proactive Measures' are AI-inferred based on descriptions provided. For critical decisions, human expert review is essential."
|
| 419 |
|
| 420 |
try:
|
| 421 |
-
|
| 422 |
-
|
| 423 |
|
| 424 |
-
#
|
| 425 |
-
|
| 426 |
-
|
| 427 |
-
progress(0.3, desc="Chunking documents and preparing for embedding...")
|
| 428 |
-
# Split raw documents into chunks (each raw_document is often a full ticket, could be split if very long)
|
| 429 |
-
chunks = custom_character_text_splitter(raw_documents, chunk_size=1000, chunk_overlap=200)
|
| 430 |
|
| 431 |
-
|
| 432 |
-
|
| 433 |
-
return f"<p style='color:red;'>{error_message_markdown}</p>"
|
| 434 |
|
| 435 |
-
|
| 436 |
-
#
|
| 437 |
-
|
| 438 |
-
vector_store.add_documents(chunks) # Add chunks to the custom FAISS vector store
|
| 439 |
|
| 440 |
-
|
| 441 |
-
|
| 442 |
-
|
| 443 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 444 |
|
| 445 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 446 |
|
| 447 |
-
|
| 448 |
-
|
| 449 |
-
|
| 450 |
-
|
| 451 |
-
|
| 452 |
-
|
| 453 |
-
**CRITICAL INSTRUCTION: Analyze ALL of the provided customer tickets in the context.**
|
| 454 |
-
**Every single ticket MUST be accounted for and assigned to a category.**
|
| 455 |
-
**The sum of the 'Count' column for all categories in your output table MUST EXACTLY EQUAL the total number of individual tickets provided in the original input data ({total_tickets_count}).**
|
| 456 |
-
|
| 457 |
-
Group tickets into a specific number of **6 to 8 distinct main categories** based on common themes. Strive to differentiate themes even if they are subtly distinct, to ensure a comprehensive categorical breakdown.
|
| 458 |
-
For each main category, identify relevant **sub-categories** where appropriate.
|
| 459 |
-
Ensure that the count of tickets within each sub-category, when summed, matches the total count for its respective main category.
|
| 460 |
-
|
| 461 |
-
If a specific detail (like root cause) cannot be inferred from the available ticket data, state 'Undetermined' or 'N/A'.
|
| 462 |
-
If any tickets do not clearly fit into distinct categories, group them under a 'Miscellaneous' or 'Other' category to ensure all tickets are included in the overall count.
|
| 463 |
-
|
| 464 |
-
**Desired Output Format (Markdown Table - for robust parsing):**
|
| 465 |
-
| Category | Sub-Category | Count | Issue Description (Summary) | Root Cause (Inferred) | Potential Resolution Steps | Future Prediction | Proactive Measures |
|
| 466 |
-
|---|---|---|---|---|---|---|---|
|
| 467 |
-
| ... | ... | ... | ... | ... | ... | ... | ... |
|
| 468 |
|
| 469 |
-
|
|
|
|
| 470 |
|
| 471 |
-
|
| 472 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 473 |
|
| 474 |
-
|
| 475 |
-
|
|
|
|
|
|
|
|
|
|
| 476 |
|
| 477 |
-
|
|
|
|
|
|
|
|
|
|
| 478 |
|
| 479 |
-
|
| 480 |
-
|
| 481 |
-
|
| 482 |
-
|
| 483 |
-
|
| 484 |
-
|
| 485 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 486 |
|
| 487 |
except ValueError as e:
|
| 488 |
-
|
| 489 |
-
global_raw_markdown_for_export =
|
| 490 |
-
global_analysis_df_for_export = pd.DataFrame()
|
| 491 |
-
return f"<p style='color:red;'>{
|
| 492 |
except Exception as e:
|
| 493 |
-
|
| 494 |
print(f"Error during RAG analysis: {e}")
|
| 495 |
-
global_raw_markdown_for_export =
|
| 496 |
-
global_analysis_df_for_export = pd.DataFrame()
|
| 497 |
-
return f"<p style='color:red;'>{
|
| 498 |
-
|
| 499 |
-
# Store raw markdown for debugging if needed, but not primarily for export format
|
| 500 |
-
global_raw_markdown_for_export = raw_rag_output + disclaimer_text_for_raw
|
| 501 |
|
| 502 |
-
progress(0.
|
| 503 |
|
| 504 |
-
# --- HTML Formatting for On-Page Display (Non-Table Visual Summary) ---
|
| 505 |
-
analysis_df = pd.DataFrame()
|
| 506 |
summarized_analysis_html = ""
|
| 507 |
-
|
| 508 |
try:
|
| 509 |
-
|
| 510 |
-
analysis_df, parsing_error = parse_markdown_table_to_df(raw_rag_output)
|
| 511 |
-
|
| 512 |
-
if parsing_error:
|
| 513 |
-
print(f"Warning: {parsing_error}")
|
| 514 |
-
if raw_rag_output.strip():
|
| 515 |
-
# Fallback: if no table is found but AI output exists, show it as generic text
|
| 516 |
-
summarized_analysis_html = f"""
|
| 517 |
-
<div style="margin-bottom: 20px;">
|
| 518 |
-
<p style="color: #D32F2F;'>AI produced output but could not structure it into a table for the visual summary.</p>
|
| 519 |
-
<p style="color: #D32F2F;'>Here is the raw AI output:</p>
|
| 520 |
-
<pre style="background-color: #EEE; padding: 15px; border: 1px solid #DDD; overflow-x: auto; white-space: pre-wrap; word-wrap: break-word;">{raw_rag_output}</pre>
|
| 521 |
-
</div>
|
| 522 |
-
"""
|
| 523 |
-
else:
|
| 524 |
-
summarized_analysis_html = "<p style='color:orange;'>AI produced no structured output or could not be parsed.</p>"
|
| 525 |
-
elif not analysis_df.empty:
|
| 526 |
-
if 'Count' in analysis_df.columns:
|
| 527 |
-
analysis_df['Count'] = pd.to_numeric(analysis_df['Count'], errors='coerce').fillna(0).astype(int)
|
| 528 |
-
else:
|
| 529 |
-
analysis_df['Count'] = 1
|
| 530 |
-
print("Warning: 'Count' column not found in AI output. Defaulting counts to 1.")
|
| 531 |
-
|
| 532 |
summarized_analysis_html = generate_summary_html_by_category(analysis_df)
|
| 533 |
-
global_analysis_df_for_export = analysis_df # Store the *parsed* DF for export
|
| 534 |
else:
|
| 535 |
-
summarized_analysis_html = "<p style='color:orange;'>AI output parsed into an empty DataFrame
|
| 536 |
-
global_analysis_df_for_export = pd.DataFrame() # Clear DF if empty
|
| 537 |
|
| 538 |
except Exception as e:
|
| 539 |
-
print(f"Failed to
|
| 540 |
summarized_analysis_html = f"""
|
| 541 |
<div style="margin-bottom: 20px;">
|
| 542 |
<p style="color: #D32F2F;'>An unexpected error occurred while attempting to transform AI output into a visual summary: {e}</p>
|
| 543 |
-
<p style="color: #D32F2F;'>
|
| 544 |
-
<pre style="background-color: #EEE; padding: 15px; border: 1px solid #DDD; overflow-x: auto; white-space: pre-wrap; word-wrap: break-word;">{
|
| 545 |
</div>
|
| 546 |
"""
|
| 547 |
-
global_analysis_df_for_export = pd.DataFrame()
|
|
|
|
| 548 |
|
| 549 |
html_output = f"""
|
| 550 |
<div style="font-family: Arial, sans-serif; padding: 20px; background-color: #f9f9f9; border-radius: 10px; box-shadow: 0 2px 10px rgba(0,0,0,0.1);">
|
|
@@ -556,6 +638,12 @@ def perform_ticket_analysis(file, user_query, progress=gr.Progress()):
|
|
| 556 |
This report provides a data-driven analysis of recent customer support tickets, leveraging advanced AI to identify trends, root causes, and actionable insights.
|
| 557 |
Below is a summary of identified issues and a detailed breakdown.
|
| 558 |
</p>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 559 |
</div>
|
| 560 |
"""
|
| 561 |
|
|
@@ -607,7 +695,16 @@ def generate_pdf_report():
|
|
| 607 |
pdf.set_font("Arial", 'B', 14)
|
| 608 |
pdf.multi_cell(0, 10, "Overview & Key Insights")
|
| 609 |
pdf.set_font("Arial", size=10)
|
| 610 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 611 |
pdf.ln(10)
|
| 612 |
|
| 613 |
# Aggregated Issue Summaries
|
|
@@ -714,10 +811,6 @@ def generate_docx_report():
|
|
| 714 |
|
| 715 |
doc = DocxDocument()
|
| 716 |
|
| 717 |
-
# Set document-wide font (optional, but good practice)
|
| 718 |
-
# doc.styles['Normal'].font.name = 'Arial'
|
| 719 |
-
# doc.styles['Normal'].font.size = Pt(11)
|
| 720 |
-
|
| 721 |
# Title
|
| 722 |
heading = doc.add_heading("AI Customer Ticket Analysis Report", level=1)
|
| 723 |
heading.alignment = WD_ALIGN_PARAGRAPH.CENTER
|
|
@@ -730,10 +823,15 @@ def generate_docx_report():
|
|
| 730 |
|
| 731 |
# Overview Section
|
| 732 |
doc.add_heading("Overview & Key Insights", level=2)
|
|
|
|
|
|
|
|
|
|
| 733 |
doc.add_paragraph(
|
| 734 |
"This report provides a data-driven analysis of recent customer support tickets, leveraging advanced AI to identify trends, root causes, and actionable insights. "
|
| 735 |
"Below is a summary of identified issues and a detailed breakdown."
|
| 736 |
)
|
|
|
|
|
|
|
| 737 |
doc.add_paragraph("") # Spacer
|
| 738 |
|
| 739 |
# Aggregated Issue Summaries
|
|
@@ -856,7 +954,7 @@ with gr.Blocks(theme=gr.themes.Soft(), css="""
|
|
| 856 |
* **Dynamic Column Reading:** The tool will attempt to read **ALL** columns present in your file. Ensure column headers are clear and meaningful.
|
| 857 |
* **Minimum Columns:** A **minimum of two columns are highly recommended**: one for a unique `Ticket ID` (or `ID`) and one for the `Issue` (or `Description`).
|
| 858 |
* **Recommended Columns:** For richer analysis, include columns like `Status`, `Priority`, `CustomerName`, `Timestamp`, etc. The more relevant details provided, the better the AI's inferences.
|
| 859 |
-
* **
|
| 860 |
* **Privacy:** **DO NOT upload sensitive or confidential customer information.** This is a publicly hosted demo.
|
| 861 |
|
| 862 |
---
|
|
@@ -914,17 +1012,15 @@ with gr.Blocks(theme=gr.themes.Soft(), css="""
|
|
| 914 |
fn=perform_ticket_analysis,
|
| 915 |
inputs=[file_input, query_input],
|
| 916 |
outputs=output_html_display,
|
| 917 |
-
queue=True
|
| 918 |
)
|
| 919 |
|
| 920 |
-
# Use .then() to make the download buttons visible
|
| 921 |
-
# and call the report generation functions
|
| 922 |
pdf_btn.click(
|
| 923 |
fn=generate_pdf_report,
|
| 924 |
outputs=pdf_output_file,
|
| 925 |
api_name="download_pdf"
|
| 926 |
).then(
|
| 927 |
-
fn=lambda: gr.File(visible=True, value=None),
|
| 928 |
inputs=[],
|
| 929 |
outputs=pdf_output_file
|
| 930 |
)
|
|
@@ -934,7 +1030,7 @@ with gr.Blocks(theme=gr.themes.Soft(), css="""
|
|
| 934 |
outputs=docx_output_file,
|
| 935 |
api_name="download_docx"
|
| 936 |
).then(
|
| 937 |
-
fn=lambda: gr.File(visible=True, value=None),
|
| 938 |
inputs=[],
|
| 939 |
outputs=docx_output_file
|
| 940 |
)
|
|
|
|
| 5 |
from datetime import datetime
|
| 6 |
import io
|
| 7 |
import re
|
| 8 |
+
import google.generativeai as genai
|
| 9 |
+
import faiss
|
| 10 |
+
import numpy as np
|
| 11 |
+
import time
|
| 12 |
+
import json # For saving intermediate results if no DB is used, or for larger JSON processing
|
| 13 |
|
| 14 |
# --- File Generation Libraries ---
|
| 15 |
from fpdf import FPDF
|
|
|
|
| 19 |
from docx.shared import Pt
|
| 20 |
from docx.enum.text import WD_PARAGRAPH_ALIGNMENT
|
| 21 |
|
| 22 |
+
# Load environment variables
|
| 23 |
load_dotenv()
|
| 24 |
|
| 25 |
# --- RAG Setup ---
|
| 26 |
rag_initialized = False
|
| 27 |
gemini_llm_model = None
|
| 28 |
+
gemini_embedding_model_name = "models/embedding-001"
|
| 29 |
|
|
|
|
| 30 |
google_api_key = os.getenv("GOOGLE_API_KEY")
|
| 31 |
|
| 32 |
if not google_api_key:
|
|
|
|
| 41 |
|
| 42 |
gemini_llm_model = genai.GenerativeModel(llm_model_id)
|
| 43 |
|
|
|
|
| 44 |
test_embedding = genai.embed_content(model=gemini_embedding_model_name, content=["test"])
|
| 45 |
if test_embedding and test_embedding['embedding']:
|
| 46 |
rag_initialized = True
|
|
|
|
| 54 |
print(f"Please verify your GOOGLE_API_KEY's validity and ensure the specified AI model ('{llm_model_id}' for LLM) and embedding model ('{gemini_embedding_model_name}') are available to your API key.")
|
| 55 |
print("RAG functionality will not work without a valid Google API key and accessible model.")
|
| 56 |
|
|
|
|
| 57 |
# Global variable to store the PARSED DATAFRAME for download functions
|
| 58 |
+
# FOR VERY LARGE DATASETS, THIS WOULD BE REPLACED BY DATABASE CALLS
|
| 59 |
global_analysis_df_for_export = pd.DataFrame()
|
| 60 |
+
global_raw_markdown_for_export = ""
|
| 61 |
|
| 62 |
# --- Custom Document Structure ---
|
| 63 |
# We'll use a dictionary: {"page_content": "...", "metadata": {...}}
|
| 64 |
|
| 65 |
+
# --- Custom Text Splitter (might not be needed if tickets are already chunks) ---
|
| 66 |
def custom_character_text_splitter(text_documents, chunk_size=1000, chunk_overlap=200):
|
| 67 |
"""
|
| 68 |
A simple character-based text splitter.
|
|
|
|
| 74 |
content = doc["page_content"]
|
| 75 |
metadata = doc["metadata"]
|
| 76 |
|
|
|
|
| 77 |
if len(content) <= chunk_size:
|
| 78 |
chunks.append({"page_content": content, "metadata": metadata})
|
| 79 |
continue
|
| 80 |
|
|
|
|
| 81 |
current_position = 0
|
| 82 |
while current_position < len(content):
|
| 83 |
chunk_end = min(current_position + chunk_size, len(content))
|
| 84 |
chunk_text = content[current_position:chunk_end]
|
| 85 |
chunks.append({"page_content": chunk_text, "metadata": metadata})
|
| 86 |
|
| 87 |
+
if chunk_end == len(content):
|
|
|
|
| 88 |
break
|
| 89 |
current_position += (chunk_size - chunk_overlap)
|
|
|
|
| 90 |
if current_position >= len(content):
|
| 91 |
break
|
| 92 |
return chunks
|
| 93 |
|
| 94 |
|
| 95 |
+
# --- Custom FAISS Integration (for searching, not direct analysis flow for 800k tickets) ---
|
| 96 |
class CustomFAISSVectorStore:
|
| 97 |
def __init__(self, embedding_model_name, dimension=768): # Gemini embedding-001 has 768 dimensions
|
| 98 |
self.embedding_model_name = embedding_model_name
|
|
|
|
| 131 |
if not query_text:
|
| 132 |
return []
|
| 133 |
|
|
|
|
| 134 |
try:
|
| 135 |
query_embedding_response = genai.embed_content(
|
| 136 |
model=self.embedding_model_name,
|
|
|
|
| 143 |
if query_embedding.shape[1] != self.index.d:
|
| 144 |
raise ValueError(f"Query embedding dimension mismatch: Expected {self.index.d}, got {query_embedding.shape[1]}.")
|
| 145 |
|
|
|
|
| 146 |
distances, indices = self.index.search(query_embedding, k)
|
| 147 |
|
| 148 |
retrieved_docs = []
|
| 149 |
for i in indices[0]:
|
| 150 |
+
if 0 <= i < len(self.documents):
|
| 151 |
retrieved_docs.append(self.documents[i])
|
| 152 |
return retrieved_docs
|
| 153 |
|
| 154 |
|
| 155 |
+
def get_ticket_documents_iterator(file_path, file_extension, filename, chunksize=1000):
|
| 156 |
"""
|
| 157 |
+
Loads and returns text from Excel or CSV files in chunks (iterator).
|
| 158 |
+
For very large files, chunksize helps manage memory.
|
| 159 |
+
Yields list of dictionaries: {"page_content": "...", "metadata": {...}}
|
| 160 |
"""
|
| 161 |
print(f"Attempting to load customer ticket data: {filename} with extension {file_extension}")
|
|
|
|
| 162 |
|
| 163 |
if file_extension in ['.xlsx', '.xls']:
|
| 164 |
try:
|
| 165 |
+
# Use openpyxl engine for .xlsx, allows chunking
|
| 166 |
+
reader = pd.read_excel(file_path, engine='openpyxl', chunksize=chunksize)
|
| 167 |
except Exception as e:
|
| 168 |
raise ValueError(f"Could not process Excel file '{filename}': {e}. Ensure it's a valid Excel format for ticket data.")
|
| 169 |
elif file_extension == '.csv':
|
| 170 |
try:
|
| 171 |
+
reader = pd.read_csv(file_path, chunksize=chunksize)
|
|
|
|
| 172 |
except Exception as e:
|
| 173 |
raise ValueError(f"Could not process CSV file '{filename}': {e}. Ensure it's a valid CSV format for ticket data.")
|
| 174 |
else:
|
| 175 |
raise ValueError(f"Unsupported file type: {file_extension}. Only .xlsx, .xls, or .csv files are accepted for ticket data.")
|
| 176 |
|
| 177 |
+
total_rows_processed = 0
|
| 178 |
+
for i, df_chunk in enumerate(reader):
|
| 179 |
+
chunk_documents = []
|
| 180 |
+
if df_chunk.empty:
|
| 181 |
+
continue
|
|
|
|
| 182 |
|
| 183 |
+
for index, row in df_chunk.iterrows():
|
| 184 |
+
ticket_content_parts = []
|
| 185 |
+
ticket_id = f"Row {total_rows_processed + 1}" # Default ticket ID
|
| 186 |
+
|
| 187 |
+
for col_name, value in row.items():
|
| 188 |
+
if pd.notna(value) and str(value).strip() != '':
|
| 189 |
+
formatted_col_name = col_name.replace('_', ' ').title()
|
| 190 |
+
ticket_content_parts.append(f"{formatted_col_name}: {value}")
|
| 191 |
+
|
| 192 |
+
if col_name.lower() in ['ticketid', 'ticket_id', 'id']:
|
| 193 |
+
ticket_id = str(value).strip()
|
| 194 |
+
|
| 195 |
+
if not ticket_content_parts:
|
| 196 |
+
ticket_content = f"Ticket {ticket_id}: No detailed content provided."
|
| 197 |
+
else:
|
| 198 |
+
ticket_content = "\n".join(ticket_content_parts)
|
| 199 |
|
| 200 |
+
metadata = {"source": filename, "ticket_row_number": total_rows_processed + 1, "ticket_id": ticket_id}
|
| 201 |
+
metadata.update(row.to_dict()) # Include all original row data in metadata
|
| 202 |
+
|
| 203 |
+
chunk_documents.append({"page_content": ticket_content, "metadata": metadata})
|
| 204 |
+
total_rows_processed += 1
|
| 205 |
|
| 206 |
+
yield chunk_documents, total_rows_processed # Yield chunk and running total
|
| 207 |
+
|
| 208 |
+
if total_rows_processed == 0:
|
| 209 |
+
raise ValueError("The uploaded file contains no data or no rows could be parsed.")
|
| 210 |
+
print(f"Total rows processed from file: {total_rows_processed}")
|
|
|
|
|
|
|
| 211 |
|
| 212 |
|
| 213 |
def generate_summary_html_by_category(df):
|
|
|
|
| 219 |
return "<p style='color:orange;'>No categories found or data is empty for detailed summarization. This could be due to issues with the AI's output parsing or content generation.</p>"
|
| 220 |
|
| 221 |
summary_html = ""
|
| 222 |
+
# Ensure all required columns exist, even if with 'N/A' to prevent key errors
|
| 223 |
required_cols = ['Issue Description (Summary)', 'Sub-Category', 'Root Cause (Inferred)',
|
| 224 |
'Potential Resolution Steps', 'Future Prediction', 'Proactive Measures', 'Count']
|
| 225 |
for col in required_cols:
|
| 226 |
if col not in df.columns:
|
| 227 |
+
df[col] = 'N/A' # Default to N/A if column is missing
|
| 228 |
+
if col == 'Count': df[col] = 0 # Default count to 0 for missing count column
|
| 229 |
|
| 230 |
+
# Group by category, ensuring consistent column order for safety
|
| 231 |
for category, category_df in df.groupby('Category'):
|
| 232 |
total_count = category_df['Count'].sum()
|
| 233 |
|
|
|
|
| 252 |
unique_root_causes = category_df['Root Cause (Inferred)']\
|
| 253 |
.astype(str)\
|
| 254 |
.drop_duplicates()\
|
| 255 |
+
.apply(lambda x: x.replace('*', '')).tolist()
|
|
|
|
| 256 |
for rc in unique_root_causes:
|
| 257 |
root_causes += f"<li>{rc}</li>"
|
| 258 |
root_causes += "</ul>" if unique_root_causes else "<p>Undetermined.</p>"
|
|
|
|
| 261 |
unique_resolutions = category_df['Potential Resolution Steps']\
|
| 262 |
.astype(str)\
|
| 263 |
.drop_duplicates()\
|
| 264 |
+
.apply(lambda x: x.replace('*', '')).tolist()
|
|
|
|
| 265 |
for res in unique_resolutions:
|
| 266 |
resolution_steps += f"<li>{res}</li>"
|
| 267 |
resolution_steps += "</ul>" if unique_resolutions else "<p>No specific steps identified.</p>"
|
|
|
|
| 313 |
def parse_markdown_table_to_df(markdown_string):
|
| 314 |
"""
|
| 315 |
Robustly parses a markdown table string into a pandas DataFrame.
|
| 316 |
+
Assumes a standard markdown table with a header and separator line.
|
| 317 |
"""
|
| 318 |
lines = markdown_string.strip().split('\n')
|
| 319 |
|
| 320 |
header_line = None
|
| 321 |
separator_line_index = -1
|
| 322 |
|
|
|
|
| 323 |
for i, line in enumerate(lines):
|
| 324 |
if '|' in line and '---' not in line and not line.strip().startswith('|---'):
|
| 325 |
header_line = line.strip()
|
| 326 |
elif '|---' in line:
|
| 327 |
separator_line_index = i
|
| 328 |
+
break
|
| 329 |
|
| 330 |
if header_line is None or separator_line_index == -1:
|
| 331 |
return pd.DataFrame(), "No valid Markdown table header or separator found."
|
| 332 |
|
|
|
|
| 333 |
data_lines = []
|
| 334 |
for line in lines[separator_line_index + 1:]:
|
| 335 |
if line.strip().startswith('|') and '|' in line.strip():
|
| 336 |
data_lines.append(line.strip())
|
| 337 |
else:
|
|
|
|
| 338 |
break
|
| 339 |
|
| 340 |
if not data_lines:
|
| 341 |
return pd.DataFrame(), "No data rows found in the Markdown table."
|
| 342 |
|
|
|
|
|
|
|
| 343 |
headers = [h.strip() for h in header_line.strip('|').split('|')]
|
| 344 |
+
headers = [h for h in headers if h] # Remove empty strings
|
|
|
|
| 345 |
|
|
|
|
| 346 |
data = []
|
| 347 |
for line in data_lines:
|
|
|
|
| 348 |
row_values = [v.strip() for v in line.strip('|').split('|')]
|
| 349 |
+
row_values = [v for v in row_values if v]
|
| 350 |
|
|
|
|
| 351 |
if len(row_values) == len(headers):
|
| 352 |
data.append(row_values)
|
| 353 |
else:
|
|
|
|
| 357 |
return pd.DataFrame(), "No valid data rows parsed after cleaning."
|
| 358 |
|
| 359 |
df = pd.DataFrame(data, columns=headers)
|
| 360 |
+
df = df.dropna(axis=1, how='all')
|
| 361 |
+
df.columns = df.columns.str.strip()
|
| 362 |
|
| 363 |
return df, None
|
| 364 |
|
|
|
|
| 366 |
def perform_ticket_analysis(file, user_query, progress=gr.Progress()):
|
| 367 |
"""
|
| 368 |
Main function to analyze customer tickets using custom RAG implementation and format output as rich HTML.
|
| 369 |
+
Implements a two-phase iterative approach to handle large ticket sets robustly.
|
|
|
|
| 370 |
"""
|
| 371 |
global global_analysis_df_for_export
|
| 372 |
+
global global_raw_markdown_for_export
|
| 373 |
|
| 374 |
start_time = time.time()
|
| 375 |
error_message_markdown = ""
|
| 376 |
|
|
|
|
| 377 |
progress(0, desc="Starting analysis...")
|
| 378 |
|
| 379 |
if not rag_initialized:
|
|
|
|
| 388 |
error_message_markdown = "Please upload a customer ticket file."
|
| 389 |
return f"<p style='color:red;'>{error_message_markdown}</p>"
|
| 390 |
|
|
|
|
|
|
|
|
|
|
|
|
|
| 391 |
file_path = file.name
|
| 392 |
filename = os.path.basename(file_path)
|
| 393 |
file_extension = os.path.splitext(filename)[1].lower()
|
| 394 |
|
|
|
|
| 395 |
disclaimer_text_for_raw = "\n\n**Note:** 'Root Cause', 'Future Prediction', and 'Proactive Measures' are AI-inferred based on descriptions provided. For critical decisions, human expert review is essential."
|
| 396 |
|
| 397 |
try:
|
| 398 |
+
# --- PHASE 1: Initial Categorization of ALL Tickets ---
|
| 399 |
+
progress(0.1, desc="Phase 1/2: Initializing categorization...")
|
| 400 |
|
| 401 |
+
# Use an iterator to handle potentially very large files
|
| 402 |
+
ticket_iterator = get_ticket_documents_iterator(file_path, file_extension, filename, chunksize=5000) # Process 5000 tickets at a time
|
|
|
|
|
|
|
|
|
|
|
|
|
| 403 |
|
| 404 |
+
all_categorized_tickets = []
|
| 405 |
+
total_tickets_count = 0
|
|
|
|
| 406 |
|
| 407 |
+
# Simulate a small database for categorized tickets
|
| 408 |
+
# In a real large-scale system, this would be a database connection.
|
| 409 |
+
categorized_tickets_db = []
|
|
|
|
| 410 |
|
| 411 |
+
for i, (chunk_documents, current_total_processed) in enumerate(ticket_iterator):
|
| 412 |
+
total_tickets_count = current_total_processed
|
| 413 |
+
progress(0.1 + (0.4 * (i+1) / (total_tickets_count / 5000 + 1)), desc=f"Phase 1/2: Categorizing tickets... {total_tickets_count} processed so far.")
|
| 414 |
+
|
| 415 |
+
if not chunk_documents:
|
| 416 |
+
continue
|
| 417 |
+
|
| 418 |
+
categorization_context = ""
|
| 419 |
+
ticket_id_map = {} # To map original ticket_id to content for later lookup within this chunk
|
| 420 |
+
for doc in chunk_documents:
|
| 421 |
+
ticket_id_for_prompt = str(doc["metadata"].get("ticket_id", f"Row {doc['metadata']['ticket_row_number']}"))
|
| 422 |
+
categorization_context += f"--- Ticket ID: {ticket_id_for_prompt} ---\n{doc['page_content']}\n\n"
|
| 423 |
+
ticket_id_map[ticket_id_for_prompt] = doc["page_content"]
|
| 424 |
+
|
| 425 |
+
categorization_prompt = f"""
|
| 426 |
+
You are an expert customer support ticket analyst. Your task is to categorize ALL provided customer tickets in this batch.
|
| 427 |
+
For each ticket, identify a concise main category (e.g., 'Technical Issue', 'Billing Inquiry', 'Feature Request', 'Account Management', 'Performance Problem', 'Data Access', 'General Inquiry').
|
| 428 |
+
Aim for **6 to 8 distinct main categories**.
|
| 429 |
+
Your output MUST be a simple list, one ticket per line, in the format:
|
| 430 |
+
Ticket ID: [Original Ticket ID from metadata] - Category: [Determined Category] - Issue Summary: [Very brief 1-2 sentence summary of the core issue]
|
| 431 |
+
|
| 432 |
+
Example:
|
| 433 |
+
Ticket ID: TKT-001 - Category: Technical Issue - Issue Summary: User cannot log in due to password error.
|
| 434 |
+
Ticket ID: INV-567 - Category: Billing Inquiry - Issue Summary: Customer dispute charge for duplicate service.
|
| 435 |
+
|
| 436 |
+
Context (Customer Tickets - analyze ALL of these, total {len(chunk_documents)} tickets in this batch):
|
| 437 |
+
{categorization_context}
|
| 438 |
+
"""
|
| 439 |
+
|
| 440 |
+
try:
|
| 441 |
+
# Add retry logic and error handling for API calls here
|
| 442 |
+
categorization_response = gemini_llm_model.generate_content(categorization_prompt)
|
| 443 |
+
categorization_raw_output = categorization_response.text
|
| 444 |
+
|
| 445 |
+
# Parse categorization output
|
| 446 |
+
for line in categorization_raw_output.split('\n'):
|
| 447 |
+
line = line.strip()
|
| 448 |
+
match = re.match(r'Ticket ID: (.*?) - Category: (.*?) - Issue Summary: (.*)', line)
|
| 449 |
+
if match:
|
| 450 |
+
ticket_id = match.group(1).strip()
|
| 451 |
+
category = match.group(2).strip()
|
| 452 |
+
issue_summary = match.group(3).strip()
|
| 453 |
+
original_content = ticket_id_map.get(ticket_id, "Original content not found for this ID.")
|
| 454 |
+
categorized_tickets_db.append({
|
| 455 |
+
"ticket_id": ticket_id,
|
| 456 |
+
"category": category,
|
| 457 |
+
"issue_summary_short": issue_summary,
|
| 458 |
+
"original_content": original_content
|
| 459 |
+
})
|
| 460 |
+
elif line:
|
| 461 |
+
print(f"Warning: Line did not match categorization format in chunk {i}: {line}")
|
| 462 |
+
# Fallback parsing for malformed lines
|
| 463 |
+
parts = line.split("Category:")
|
| 464 |
+
ticket_id_part = parts[0].replace("Ticket ID:", "").strip() if len(parts) > 0 else f"Unparsed_Line_{len(categorized_tickets_db)+1}"
|
| 465 |
+
category_part = parts[1].split("Issue Summary:")[0].strip() if len(parts) > 1 and "Issue Summary:" in parts[1] else "Uncategorized/Parsing Error"
|
| 466 |
+
issue_summary_part = parts[1].split("Issue Summary:", 1)[1].strip() if len(parts) > 1 and "Issue Summary:" in parts[1] else line
|
| 467 |
+
original_content = ticket_id_map.get(ticket_id_part, "Original content not found / Raw line.")
|
| 468 |
+
categorized_tickets_db.append({
|
| 469 |
+
"ticket_id": ticket_id_part,
|
| 470 |
+
"category": category_part,
|
| 471 |
+
"issue_summary_short": issue_summary_part,
|
| 472 |
+
"original_content": original_content
|
| 473 |
+
})
|
| 474 |
+
|
| 475 |
+
except Exception as e:
|
| 476 |
+
print(f"Error during categorization of chunk {i}: {e}")
|
| 477 |
+
# For a real system, you'd log this, and potentially save the raw chunk for re-processing.
|
| 478 |
+
# For this demo, we'll just skip the chunk or add a generic error entry.
|
| 479 |
+
for doc in chunk_documents:
|
| 480 |
+
ticket_id = str(doc["metadata"].get("ticket_id", f"Row {doc['metadata']['ticket_row_number']}"))
|
| 481 |
+
categorized_tickets_db.append({
|
| 482 |
+
"ticket_id": ticket_id,
|
| 483 |
+
"category": "Error_Processing_Chunk",
|
| 484 |
+
"issue_summary_short": f"Failed to categorize due to error: {e}",
|
| 485 |
+
"original_content": doc["page_content"]
|
| 486 |
+
})
|
| 487 |
|
| 488 |
+
# After processing all chunks, convert to DataFrame for easier manipulation
|
| 489 |
+
final_categorized_df = pd.DataFrame(categorized_tickets_db)
|
| 490 |
+
if final_categorized_df.empty:
|
| 491 |
+
error_message_markdown = "AI failed to categorize any tickets. Please ensure the ticket data is clear."
|
| 492 |
+
return f"<p style='color:red;'>{error_message_markdown}</p>"
|
| 493 |
+
|
| 494 |
+
# --- PHASE 2: Detailed Analysis per Category ---
|
| 495 |
+
progress(0.5, desc=f"Phase 2/2: Detailing analysis for categories...")
|
| 496 |
|
| 497 |
+
final_analysis_rows = []
|
| 498 |
+
unique_categories = final_categorized_df['category'].unique()
|
| 499 |
+
|
| 500 |
+
# Define a batch size for processing categories for detailed analysis if there are many categories
|
| 501 |
+
# For 6-8 categories, this internal batching might not be strictly necessary, but good practice.
|
| 502 |
+
category_batch_size = 5
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 503 |
|
| 504 |
+
for i, category_chunk in enumerate(np.array_split(unique_categories, max(1, len(unique_categories) // category_batch_size))):
|
| 505 |
+
progress((0.5 + (0.45 * i / len(np.array_split(unique_categories, max(1, len(unique_categories) // category_batch_size))))), desc=f"Phase 2/2: Detailing analysis for category batch {i+1}...")
|
| 506 |
|
| 507 |
+
for category in category_chunk:
|
| 508 |
+
category_specific_tickets_df = final_categorized_df[final_categorized_df['category'] == category]
|
| 509 |
+
|
| 510 |
+
# Combine relevant content for the category. For 800k tickets, this concatenation could still be huge.
|
| 511 |
+
# Consider summarizing `original_content` for each ticket in Phase 1 if context window is an issue.
|
| 512 |
+
category_tickets_content = "\n\n".join(category_specific_tickets_df['original_content'].tolist())
|
| 513 |
+
|
| 514 |
+
category_count = len(category_specific_tickets_df)
|
| 515 |
+
|
| 516 |
+
detail_prompt = f"""
|
| 517 |
+
You are an expert customer support ticket analyst.
|
| 518 |
+
Analyze the following customer tickets, which all belong to the '{category}' category.
|
| 519 |
+
Your output MUST be a **single Markdown table row** that summarizes the common themes, root causes, resolution steps, future predictions, and proactive measures *for this specific category of tickets*.
|
| 520 |
|
| 521 |
+
**Ensure the 'Count' in your table row EXACTLY matches the number of tickets provided for this category ({category_count}).**
|
| 522 |
+
|
| 523 |
+
Keep 'Issue Description (Summary)' concise (max 3-4 sentences), summarizing common themes *across these tickets*.
|
| 524 |
+
If a specific detail (like root cause) cannot be inferred from the available ticket data, state 'Undetermined' or 'N/A'.
|
| 525 |
+
Do not make up information.
|
| 526 |
|
| 527 |
+
**Desired Output Format (Markdown Table - single row):**
|
| 528 |
+
| Category | Sub-Category | Count | Issue Description (Summary) | Root Cause (Inferred) | Potential Resolution Steps | Future Prediction | Proactive Measures |
|
| 529 |
+
|---|---|---|---|---|---|---|---|
|
| 530 |
+
| ... | ... | ... | ... | ... | ... | ... | ... |
|
| 531 |
|
| 532 |
+
Context (Tickets for Category '{category}'):
|
| 533 |
+
{category_tickets_content}
|
| 534 |
+
"""
|
| 535 |
+
|
| 536 |
+
try:
|
| 537 |
+
# Add retry logic and error handling for API calls here
|
| 538 |
+
detail_response = gemini_llm_model.generate_content(detail_prompt)
|
| 539 |
+
detail_raw_output = detail_response.text
|
| 540 |
+
|
| 541 |
+
detailed_df, parsing_err = parse_markdown_table_to_df(detail_raw_output)
|
| 542 |
+
if parsing_err or detailed_df.empty:
|
| 543 |
+
print(f"Warning: Parsing error or empty DataFrame for category '{category}': {parsing_err or 'Empty DataFrame'}")
|
| 544 |
+
print(f"Raw AI output for detail: {detail_raw_output}")
|
| 545 |
+
default_row = {
|
| 546 |
+
"Category": category,
|
| 547 |
+
"Sub-Category": "N/A",
|
| 548 |
+
"Count": category_count,
|
| 549 |
+
"Issue Description (Summary)": category_specific_tickets_df['issue_summary_short'].iloc[0] if not category_specific_tickets_df.empty else "Error in summary generation/parsing for this category.",
|
| 550 |
+
"Root Cause (Inferred)": "Undetermined due to parsing error.",
|
| 551 |
+
"Potential Resolution Steps": "N/A",
|
| 552 |
+
"Future Prediction": "N/A",
|
| 553 |
+
"Proactive Measures": "N/A"
|
| 554 |
+
}
|
| 555 |
+
final_analysis_rows.append(default_row)
|
| 556 |
+
else:
|
| 557 |
+
detailed_row = detailed_df.iloc[0].to_dict()
|
| 558 |
+
detailed_row['Category'] = category
|
| 559 |
+
|
| 560 |
+
if 'Count' in detailed_row:
|
| 561 |
+
try:
|
| 562 |
+
ai_reported_count = pd.to_numeric(detailed_row['Count'], errors='coerce')
|
| 563 |
+
if pd.isna(ai_reported_count) or int(ai_reported_count) != category_count:
|
| 564 |
+
print(f"Warning: AI-reported count {ai_reported_count} for '{category}' does not match actual count {category_count}. Correcting.")
|
| 565 |
+
detailed_row['Count'] = category_count
|
| 566 |
+
except:
|
| 567 |
+
detailed_row['Count'] = category_count
|
| 568 |
+
else:
|
| 569 |
+
detailed_row['Count'] = category_count
|
| 570 |
+
|
| 571 |
+
final_analysis_rows.append(detailed_row)
|
| 572 |
+
|
| 573 |
+
except Exception as e:
|
| 574 |
+
print(f"Error detailing analysis for category '{category}': {e}")
|
| 575 |
+
default_row = {
|
| 576 |
+
"Category": category,
|
| 577 |
+
"Sub-Category": "N/A",
|
| 578 |
+
"Count": category_count,
|
| 579 |
+
"Issue Description (Summary)": f"Error during detailed analysis for this category: {e}",
|
| 580 |
+
"Root Cause (Inferred)": "Undetermined due to error.",
|
| 581 |
+
"Potential Resolution Steps": "N/A",
|
| 582 |
+
"Future Prediction": "N/A",
|
| 583 |
+
"Proactive Measures": "N/A"
|
| 584 |
+
}
|
| 585 |
+
final_analysis_rows.append(default_row)
|
| 586 |
+
|
| 587 |
+
analysis_df = pd.DataFrame(final_analysis_rows)
|
| 588 |
+
if 'Count' in analysis_df.columns:
|
| 589 |
+
analysis_df['Count'] = pd.to_numeric(analysis_df['Count'], errors='coerce').fillna(0).astype(int)
|
| 590 |
+
|
| 591 |
+
summarized_count = analysis_df['Count'].sum()
|
| 592 |
+
if summarized_count != total_tickets_count:
|
| 593 |
+
print(f"CRITICAL WARNING: Total count in summarized DF ({summarized_count}) does not match original tickets ({total_tickets_count}).")
|
| 594 |
+
|
| 595 |
+
global_analysis_df_for_export = analysis_df.copy()
|
| 596 |
+
global_raw_markdown_for_export = analysis_df.to_markdown(index=False) + disclaimer_text_for_raw
|
| 597 |
|
| 598 |
except ValueError as e:
|
| 599 |
+
error_message_markdown = f"Ticket Analysis Error: {e}"
|
| 600 |
+
global_raw_markdown_for_export = error_message_markdown + disclaimer_text_for_raw
|
| 601 |
+
global_analysis_df_for_export = pd.DataFrame()
|
| 602 |
+
return f"<p style='color:red;'>{error_message_markdown}</p>"
|
| 603 |
except Exception as e:
|
| 604 |
+
error_message_markdown = f"An unexpected error occurred during analysis: {e}. Check server logs for details."
|
| 605 |
print(f"Error during RAG analysis: {e}")
|
| 606 |
+
global_raw_markdown_for_export = error_message_markdown + disclaimer_text_for_raw
|
| 607 |
+
global_analysis_df_for_export = pd.DataFrame()
|
| 608 |
+
return f"<p style='color:red;'>{error_message_markdown}</p>"
|
|
|
|
|
|
|
|
|
|
| 609 |
|
| 610 |
+
progress(0.98, desc="Generating summary report for display...")
|
| 611 |
|
|
|
|
|
|
|
| 612 |
summarized_analysis_html = ""
|
|
|
|
| 613 |
try:
|
| 614 |
+
if not analysis_df.empty:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 615 |
summarized_analysis_html = generate_summary_html_by_category(analysis_df)
|
|
|
|
| 616 |
else:
|
| 617 |
+
summarized_analysis_html = "<p style='color:orange;'>AI output parsed into an empty DataFrame or no categories were generated. Please check the uploaded file and try again.</p>"
|
|
|
|
| 618 |
|
| 619 |
except Exception as e:
|
| 620 |
+
print(f"Failed to generate HTML summary from DataFrame: {e}")
|
| 621 |
summarized_analysis_html = f"""
|
| 622 |
<div style="margin-bottom: 20px;">
|
| 623 |
<p style="color: #D32F2F;'>An unexpected error occurred while attempting to transform AI output into a visual summary: {e}</p>
|
| 624 |
+
<p style="color: #D32F2F;'>Raw AI output (if available):</p>
|
| 625 |
+
<pre style="background-color: #EEE; padding: 15px; border: 1px solid #DDD; overflow-x: auto; white-space: pre-wrap; word-wrap: break-word;">{global_raw_markdown_for_export}</pre>
|
| 626 |
</div>
|
| 627 |
"""
|
| 628 |
+
global_analysis_df_for_export = pd.DataFrame()
|
| 629 |
+
|
| 630 |
|
| 631 |
html_output = f"""
|
| 632 |
<div style="font-family: Arial, sans-serif; padding: 20px; background-color: #f9f9f9; border-radius: 10px; box-shadow: 0 2px 10px rgba(0,0,0,0.1);">
|
|
|
|
| 638 |
This report provides a data-driven analysis of recent customer support tickets, leveraging advanced AI to identify trends, root causes, and actionable insights.
|
| 639 |
Below is a summary of identified issues and a detailed breakdown.
|
| 640 |
</p>
|
| 641 |
+
<p style="font-size: 1.0em; line-height: 1.6; color: #424242;">
|
| 642 |
+
Total Original Tickets in File: <strong>{total_tickets_count}</strong>.<br>
|
| 643 |
+
Total Tickets Summarized in Report: <strong>{summarized_count}</strong>.<br>
|
| 644 |
+
Total Categories Identified: <strong>{len(unique_categories) if 'unique_categories' in locals() else 0}</strong>.
|
| 645 |
+
{"<p style='color:red; font-weight:bold;'>WARNING: The number of tickets summarized in the report does not match the total original tickets. This may indicate a parsing or AI processing issue. Please review the raw output if available.</p>" if summarized_count != total_tickets_count else ""}
|
| 646 |
+
</p>
|
| 647 |
</div>
|
| 648 |
"""
|
| 649 |
|
|
|
|
| 695 |
pdf.set_font("Arial", 'B', 14)
|
| 696 |
pdf.multi_cell(0, 10, "Overview & Key Insights")
|
| 697 |
pdf.set_font("Arial", size=10)
|
| 698 |
+
total_tickets_processed = df['Count'].sum() if 'Count' in df.columns else 0
|
| 699 |
+
total_categories_identified = len(df['Category'].unique()) if 'Category' in df.columns else 0
|
| 700 |
+
|
| 701 |
+
overview_text = (
|
| 702 |
+
"This report provides a data-driven analysis of recent customer support tickets, leveraging advanced AI to identify trends, root causes, and actionable insights. "
|
| 703 |
+
"Below is a summary of identified issues and a detailed breakdown.\n\n"
|
| 704 |
+
f"Total Tickets Summarized in Report: {total_tickets_processed}.\n"
|
| 705 |
+
f"Total Categories Identified: {total_categories_identified}."
|
| 706 |
+
)
|
| 707 |
+
pdf.multi_cell(0, 7, overview_text)
|
| 708 |
pdf.ln(10)
|
| 709 |
|
| 710 |
# Aggregated Issue Summaries
|
|
|
|
| 811 |
|
| 812 |
doc = DocxDocument()
|
| 813 |
|
|
|
|
|
|
|
|
|
|
|
|
|
| 814 |
# Title
|
| 815 |
heading = doc.add_heading("AI Customer Ticket Analysis Report", level=1)
|
| 816 |
heading.alignment = WD_ALIGN_PARAGRAPH.CENTER
|
|
|
|
| 823 |
|
| 824 |
# Overview Section
|
| 825 |
doc.add_heading("Overview & Key Insights", level=2)
|
| 826 |
+
total_tickets_processed = df['Count'].sum() if 'Count' in df.columns else 0
|
| 827 |
+
total_categories_identified = len(df['Category'].unique()) if 'Category' in df.columns else 0
|
| 828 |
+
|
| 829 |
doc.add_paragraph(
|
| 830 |
"This report provides a data-driven analysis of recent customer support tickets, leveraging advanced AI to identify trends, root causes, and actionable insights. "
|
| 831 |
"Below is a summary of identified issues and a detailed breakdown."
|
| 832 |
)
|
| 833 |
+
doc.add_paragraph(f"Total Tickets Summarized in Report: {total_tickets_processed}.")
|
| 834 |
+
doc.add_paragraph(f"Total Categories Identified: {total_categories_identified}.")
|
| 835 |
doc.add_paragraph("") # Spacer
|
| 836 |
|
| 837 |
# Aggregated Issue Summaries
|
|
|
|
| 954 |
* **Dynamic Column Reading:** The tool will attempt to read **ALL** columns present in your file. Ensure column headers are clear and meaningful.
|
| 955 |
* **Minimum Columns:** A **minimum of two columns are highly recommended**: one for a unique `Ticket ID` (or `ID`) and one for the `Issue` (or `Description`).
|
| 956 |
* **Recommended Columns:** For richer analysis, include columns like `Status`, `Priority`, `CustomerName`, `Timestamp`, etc. The more relevant details provided, the better the AI's inferences.
|
| 957 |
+
* **Scale Handling:** For very large files (e.g., 60,000+ tickets), the system will process data in **batches** to manage memory and API limits. Be aware that processing can take a significant amount of time.
|
| 958 |
* **Privacy:** **DO NOT upload sensitive or confidential customer information.** This is a publicly hosted demo.
|
| 959 |
|
| 960 |
---
|
|
|
|
| 1012 |
fn=perform_ticket_analysis,
|
| 1013 |
inputs=[file_input, query_input],
|
| 1014 |
outputs=output_html_display,
|
| 1015 |
+
queue=True
|
| 1016 |
)
|
| 1017 |
|
|
|
|
|
|
|
| 1018 |
pdf_btn.click(
|
| 1019 |
fn=generate_pdf_report,
|
| 1020 |
outputs=pdf_output_file,
|
| 1021 |
api_name="download_pdf"
|
| 1022 |
).then(
|
| 1023 |
+
fn=lambda: gr.File(visible=True, value=None),
|
| 1024 |
inputs=[],
|
| 1025 |
outputs=pdf_output_file
|
| 1026 |
)
|
|
|
|
| 1030 |
outputs=docx_output_file,
|
| 1031 |
api_name="download_docx"
|
| 1032 |
).then(
|
| 1033 |
+
fn=lambda: gr.File(visible=True, value=None),
|
| 1034 |
inputs=[],
|
| 1035 |
outputs=docx_output_file
|
| 1036 |
)
|