Neom99 commited on
Commit
c5da91f
·
verified ·
1 Parent(s): 997cdd8

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +294 -198
app.py CHANGED
@@ -5,10 +5,11 @@ import pandas as pd
5
  from datetime import datetime
6
  import io
7
  import re
8
- import google.generativeai as genai # Direct Google Generative AI client
9
- import faiss # FAISS for vector operations
10
- import numpy as np # For numerical operations, especially with embeddings
11
- import time # Import the time module for tracking elapsed time
 
12
 
13
  # --- File Generation Libraries ---
14
  from fpdf import FPDF
@@ -18,15 +19,14 @@ from docx.enum.text import WD_ALIGN_PARAGRAPH
18
  from docx.shared import Pt
19
  from docx.enum.text import WD_PARAGRAPH_ALIGNMENT
20
 
21
- # Load environment variables (for API keys when running locally)
22
  load_dotenv()
23
 
24
  # --- RAG Setup ---
25
  rag_initialized = False
26
  gemini_llm_model = None
27
- gemini_embedding_model_name = "models/embedding-001" # Consistent embedding model name
28
 
29
- # Attempt to get API key from environment BEFORE trying to initialize
30
  google_api_key = os.getenv("GOOGLE_API_KEY")
31
 
32
  if not google_api_key:
@@ -41,7 +41,6 @@ else:
41
 
42
  gemini_llm_model = genai.GenerativeModel(llm_model_id)
43
 
44
- # Test embedding model to ensure it's accessible
45
  test_embedding = genai.embed_content(model=gemini_embedding_model_name, content=["test"])
46
  if test_embedding and test_embedding['embedding']:
47
  rag_initialized = True
@@ -55,15 +54,15 @@ else:
55
  print(f"Please verify your GOOGLE_API_KEY's validity and ensure the specified AI model ('{llm_model_id}' for LLM) and embedding model ('{gemini_embedding_model_name}') are available to your API key.")
56
  print("RAG functionality will not work without a valid Google API key and accessible model.")
57
 
58
-
59
  # Global variable to store the PARSED DATAFRAME for download functions
 
60
  global_analysis_df_for_export = pd.DataFrame()
61
- global_raw_markdown_for_export = "" # Keep raw for debugging/reference if needed
62
 
63
  # --- Custom Document Structure ---
64
  # We'll use a dictionary: {"page_content": "...", "metadata": {...}}
65
 
66
- # --- Custom Text Splitter ---
67
  def custom_character_text_splitter(text_documents, chunk_size=1000, chunk_overlap=200):
68
  """
69
  A simple character-based text splitter.
@@ -75,29 +74,25 @@ def custom_character_text_splitter(text_documents, chunk_size=1000, chunk_overla
75
  content = doc["page_content"]
76
  metadata = doc["metadata"]
77
 
78
- # If content is already smaller than chunk_size, just add it
79
  if len(content) <= chunk_size:
80
  chunks.append({"page_content": content, "metadata": metadata})
81
  continue
82
 
83
- # Split content into chunks with overlap
84
  current_position = 0
85
  while current_position < len(content):
86
  chunk_end = min(current_position + chunk_size, len(content))
87
  chunk_text = content[current_position:chunk_end]
88
  chunks.append({"page_content": chunk_text, "metadata": metadata})
89
 
90
- # Move current_position for the next chunk
91
- if chunk_end == len(content): # Reached end of content
92
  break
93
  current_position += (chunk_size - chunk_overlap)
94
- # Ensure we don't start the next chunk beyond the content length
95
  if current_position >= len(content):
96
  break
97
  return chunks
98
 
99
 
100
- # --- Custom FAISS Integration ---
101
  class CustomFAISSVectorStore:
102
  def __init__(self, embedding_model_name, dimension=768): # Gemini embedding-001 has 768 dimensions
103
  self.embedding_model_name = embedding_model_name
@@ -136,7 +131,6 @@ class CustomFAISSVectorStore:
136
  if not query_text:
137
  return []
138
 
139
- # Embed query
140
  try:
141
  query_embedding_response = genai.embed_content(
142
  model=self.embedding_model_name,
@@ -149,73 +143,71 @@ class CustomFAISSVectorStore:
149
  if query_embedding.shape[1] != self.index.d:
150
  raise ValueError(f"Query embedding dimension mismatch: Expected {self.index.d}, got {query_embedding.shape[1]}.")
151
 
152
- # Perform search
153
  distances, indices = self.index.search(query_embedding, k)
154
 
155
  retrieved_docs = []
156
  for i in indices[0]:
157
- if 0 <= i < len(self.documents): # Ensure valid index
158
  retrieved_docs.append(self.documents[i])
159
  return retrieved_docs
160
 
161
 
162
- def get_document_content_from_tickets(file_path, file_extension, filename):
163
  """
164
- Loads and returns text from Excel or CSV files, treating each row as a customer ticket.
165
- Dynamically reads all columns and includes them in the document content.
166
- Returns a list of dictionaries: {"page_content": "...", "metadata": {...}}
167
  """
168
  print(f"Attempting to load customer ticket data: {filename} with extension {file_extension}")
169
- raw_documents = []
170
 
171
  if file_extension in ['.xlsx', '.xls']:
172
  try:
173
- df = pd.read_excel(file_path)
174
- print(f"Excel file '{filename}' loaded with {len(df)} rows.")
175
  except Exception as e:
176
  raise ValueError(f"Could not process Excel file '{filename}': {e}. Ensure it's a valid Excel format for ticket data.")
177
  elif file_extension == '.csv':
178
  try:
179
- df = pd.read_csv(file_path)
180
- print(f"CSV file '{filename}' loaded with {len(df)} rows.")
181
  except Exception as e:
182
  raise ValueError(f"Could not process CSV file '{filename}': {e}. Ensure it's a valid CSV format for ticket data.")
183
  else:
184
  raise ValueError(f"Unsupported file type: {file_extension}. Only .xlsx, .xls, or .csv files are accepted for ticket data.")
185
 
186
- if df.empty:
187
- raise ValueError("The uploaded file contains no data or no rows could be parsed.")
188
-
189
- for index, row in df.iterrows():
190
- ticket_content_parts = []
191
- ticket_id = f"Row {index+1}"
192
 
193
- for col_name, value in row.items():
194
- if pd.notna(value) and str(value).strip() != '':
195
- formatted_col_name = col_name.replace('_', ' ').title()
196
- ticket_content_parts.append(f"{formatted_col_name}: {value}")
197
-
198
- if col_name.lower() in ['ticketid', 'ticket_id', 'id']:
199
- ticket_id = value
200
- elif col_name.lower() in ['issue', 'description', 'problem'] and len(str(value)) > 10:
201
- pass
202
-
203
- if not ticket_content_parts:
204
- # If a row has no content, we still might want to include it with a note
205
- ticket_content = f"Ticket {ticket_id}: No detailed content provided."
206
- else:
207
- ticket_content = "\n".join(ticket_content_parts)
 
208
 
209
- metadata = {"source": filename, "ticket_row_number": index+1, "ticket_id": ticket_id}
210
- metadata.update(row.to_dict())
 
 
 
211
 
212
- raw_documents.append({"page_content": ticket_content, "metadata": metadata})
213
-
214
- if not raw_documents:
215
- raise ValueError("No usable ticket data extracted from the document.")
216
-
217
- print(f"File '{filename}' processed into {len(raw_documents)} raw documents (one per ticket).")
218
- return raw_documents
219
 
220
 
221
  def generate_summary_html_by_category(df):
@@ -227,13 +219,15 @@ def generate_summary_html_by_category(df):
227
  return "<p style='color:orange;'>No categories found or data is empty for detailed summarization. This could be due to issues with the AI's output parsing or content generation.</p>"
228
 
229
  summary_html = ""
 
230
  required_cols = ['Issue Description (Summary)', 'Sub-Category', 'Root Cause (Inferred)',
231
  'Potential Resolution Steps', 'Future Prediction', 'Proactive Measures', 'Count']
232
  for col in required_cols:
233
  if col not in df.columns:
234
- df[col] = df[col] if col in df.columns else 'N/A'
235
- if col == 'Count': df[col] = 1
236
 
 
237
  for category, category_df in df.groupby('Category'):
238
  total_count = category_df['Count'].sum()
239
 
@@ -258,8 +252,7 @@ def generate_summary_html_by_category(df):
258
  unique_root_causes = category_df['Root Cause (Inferred)']\
259
  .astype(str)\
260
  .drop_duplicates()\
261
- .apply(lambda x: x.replace('*', ''))\
262
- .tolist()
263
  for rc in unique_root_causes:
264
  root_causes += f"<li>{rc}</li>"
265
  root_causes += "</ul>" if unique_root_causes else "<p>Undetermined.</p>"
@@ -268,8 +261,7 @@ def generate_summary_html_by_category(df):
268
  unique_resolutions = category_df['Potential Resolution Steps']\
269
  .astype(str)\
270
  .drop_duplicates()\
271
- .apply(lambda x: x.replace('*', ''))\
272
- .tolist()
273
  for res in unique_resolutions:
274
  resolution_steps += f"<li>{res}</li>"
275
  resolution_steps += "</ul>" if unique_resolutions else "<p>No specific steps identified.</p>"
@@ -321,49 +313,41 @@ def generate_summary_html_by_category(df):
321
  def parse_markdown_table_to_df(markdown_string):
322
  """
323
  Robustly parses a markdown table string into a pandas DataFrame.
 
324
  """
325
  lines = markdown_string.strip().split('\n')
326
 
327
  header_line = None
328
  separator_line_index = -1
329
 
330
- # Find header and separator lines
331
  for i, line in enumerate(lines):
332
  if '|' in line and '---' not in line and not line.strip().startswith('|---'):
333
  header_line = line.strip()
334
  elif '|---' in line:
335
  separator_line_index = i
336
- break # Found separator, headers should be just above it
337
 
338
  if header_line is None or separator_line_index == -1:
339
  return pd.DataFrame(), "No valid Markdown table header or separator found."
340
 
341
- # Extract data rows after the separator
342
  data_lines = []
343
  for line in lines[separator_line_index + 1:]:
344
  if line.strip().startswith('|') and '|' in line.strip():
345
  data_lines.append(line.strip())
346
  else:
347
- # Stop if we encounter a non-table line, assuming table has ended
348
  break
349
 
350
  if not data_lines:
351
  return pd.DataFrame(), "No data rows found in the Markdown table."
352
 
353
- # Process header
354
- # Clean header line to split correctly and handle potential leading/trailing pipes
355
  headers = [h.strip() for h in header_line.strip('|').split('|')]
356
- # Remove any empty strings from headers (e.g., if input had || between columns)
357
- headers = [h for h in headers if h]
358
 
359
- # Process data rows
360
  data = []
361
  for line in data_lines:
362
- # Clean data line similarly
363
  row_values = [v.strip() for v in line.strip('|').split('|')]
364
- row_values = [v for v in row_values if v] # Remove empty strings
365
 
366
- # Ensure row has the same number of columns as headers
367
  if len(row_values) == len(headers):
368
  data.append(row_values)
369
  else:
@@ -373,8 +357,8 @@ def parse_markdown_table_to_df(markdown_string):
373
  return pd.DataFrame(), "No valid data rows parsed after cleaning."
374
 
375
  df = pd.DataFrame(data, columns=headers)
376
- df = df.dropna(axis=1, how='all') # Drop columns that are entirely NaN
377
- df.columns = df.columns.str.strip() # Strip whitespace from column names
378
 
379
  return df, None
380
 
@@ -382,16 +366,14 @@ def parse_markdown_table_to_df(markdown_string):
382
  def perform_ticket_analysis(file, user_query, progress=gr.Progress()):
383
  """
384
  Main function to analyze customer tickets using custom RAG implementation and format output as rich HTML.
385
- Updates a global variable with the raw markdown result for download functions.
386
- Includes progress updates for Gradio UI.
387
  """
388
  global global_analysis_df_for_export
389
- global global_raw_markdown_for_export # Keep this for troubleshooting raw output
390
 
391
  start_time = time.time()
392
  error_message_markdown = ""
393
 
394
- # Initialize progress bar
395
  progress(0, desc="Starting analysis...")
396
 
397
  if not rag_initialized:
@@ -406,145 +388,245 @@ def perform_ticket_analysis(file, user_query, progress=gr.Progress()):
406
  error_message_markdown = "Please upload a customer ticket file."
407
  return f"<p style='color:red;'>{error_message_markdown}</p>"
408
 
409
- # Default comprehensive analysis if query is empty
410
- if not user_query.strip():
411
- user_query = "Perform a comprehensive analysis of the tickets. Provide categorization, sub-categories, count of issues, detailed description of issues, infer the root cause, suggest potential resolution steps, identify future prediction, and propose proactive measures. Ensure the output is a Markdown table with columns: 'Category', 'Sub-Category', 'Count', 'Issue Description (Summary)', 'Root Cause (Inferred)', 'Potential Resolution Steps', 'Future Prediction', 'Proactive Measures'."
412
-
413
  file_path = file.name
414
  filename = os.path.basename(file_path)
415
  file_extension = os.path.splitext(filename)[1].lower()
416
 
417
- raw_rag_output = ""
418
  disclaimer_text_for_raw = "\n\n**Note:** 'Root Cause', 'Future Prediction', and 'Proactive Measures' are AI-inferred based on descriptions provided. For critical decisions, human expert review is essential."
419
 
420
  try:
421
- progress(0.1, desc="Loading and processing ticket data...")
422
- raw_documents = get_document_content_from_tickets(file_path, file_extension, filename)
423
 
424
- # Original count of tickets for prompt insertion
425
- total_tickets_count = len(raw_documents)
426
-
427
- progress(0.3, desc="Chunking documents and preparing for embedding...")
428
- # Split raw documents into chunks (each raw_document is often a full ticket, could be split if very long)
429
- chunks = custom_character_text_splitter(raw_documents, chunk_size=1000, chunk_overlap=200)
430
 
431
- if not chunks:
432
- error_message_markdown = "No usable text extracted or chunks created from the ticket data after splitting."
433
- return f"<p style='color:red;'>{error_message_markdown}</p>"
434
 
435
- progress(0.5, desc="Building vector store and embedding data...")
436
- # Build FAISS index
437
- vector_store = CustomFAISSVectorStore(embedding_model_name=gemini_embedding_model_name)
438
- vector_store.add_documents(chunks) # Add chunks to the custom FAISS vector store
439
 
440
- # Retrieve relevant chunks for the context
441
- # Given the instruction to analyze ALL tickets, we'll retrieve all chunks.
442
- # Gemini 1.5 Flash has a large context window, so passing all chunks is feasible.
443
- retrieved_chunks = vector_store.search(user_query, k=len(chunks))
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
444
 
445
- context_content = "\n\n".join([c["page_content"] for c in retrieved_chunks])
 
 
 
 
 
 
 
446
 
447
- # Construct the prompt with the retrieved context and strong instructions
448
- template = f"""
449
- You are an expert customer support ticket analyst. Your goal is to provide a comprehensive and accurate analysis of customer tickets
450
- based *only* on the provided context. The context contains individual customer tickets, each representing a row
451
- from a data file. Each row contains various columns and their values.
452
-
453
- **CRITICAL INSTRUCTION: Analyze ALL of the provided customer tickets in the context.**
454
- **Every single ticket MUST be accounted for and assigned to a category.**
455
- **The sum of the 'Count' column for all categories in your output table MUST EXACTLY EQUAL the total number of individual tickets provided in the original input data ({total_tickets_count}).**
456
-
457
- Group tickets into a specific number of **6 to 8 distinct main categories** based on common themes. Strive to differentiate themes even if they are subtly distinct, to ensure a comprehensive categorical breakdown.
458
- For each main category, identify relevant **sub-categories** where appropriate.
459
- Ensure that the count of tickets within each sub-category, when summed, matches the total count for its respective main category.
460
-
461
- If a specific detail (like root cause) cannot be inferred from the available ticket data, state 'Undetermined' or 'N/A'.
462
- If any tickets do not clearly fit into distinct categories, group them under a 'Miscellaneous' or 'Other' category to ensure all tickets are included in the overall count.
463
-
464
- **Desired Output Format (Markdown Table - for robust parsing):**
465
- | Category | Sub-Category | Count | Issue Description (Summary) | Root Cause (Inferred) | Potential Resolution Steps | Future Prediction | Proactive Measures |
466
- |---|---|---|---|---|---|---|---|
467
- | ... | ... | ... | ... | ... | ... | ... | ... |
468
 
469
- For 'Issue Description (Summary)', provide a clear and concise summary of the **common themes and types of issues** within that category. Do not truncate or limit the description if a comprehensive overview is needed, but ensure it remains a summary of the *category's* issues, not a list of individual ticket details.
 
470
 
471
- If the information for a column is not explicitly present or inferable for *all* analyzed tickets in a category, indicate 'N/A'.
472
- Do not make up information.
 
 
 
 
 
 
 
 
 
 
 
473
 
474
- Context (Customer Tickets - analyze ALL of these, total {total_tickets_count} original tickets data provided as context):
475
- {context_content}
 
 
 
476
 
477
- Question: {user_query}
 
 
 
478
 
479
- Your detailed ticket analysis for ALL tickets, ensuring total count consistency:
480
- """
481
-
482
- progress(0.7, desc="Calling AI for comprehensive analysis...")
483
- # Generate content using the LLM directly
484
- response = gemini_llm_model.generate_content(template)
485
- raw_rag_output = response.text
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
486
 
487
  except ValueError as e:
488
- raw_rag_output = f"Ticket Analysis Error: {e}"
489
- global_raw_markdown_for_export = raw_rag_output + disclaimer_text_for_raw
490
- global_analysis_df_for_export = pd.DataFrame() # Clear DF on error
491
- return f"<p style='color:red;'>{raw_rag_output}</p>"
492
  except Exception as e:
493
- raw_rag_output = f"An unexpected error occurred during analysis: {e}. Check logs for details."
494
  print(f"Error during RAG analysis: {e}")
495
- global_raw_markdown_for_export = raw_rag_output + disclaimer_text_for_raw
496
- global_analysis_df_for_export = pd.DataFrame() # Clear DF on error
497
- return f"<p style='color:red;'>{raw_rag_output}</p>"
498
-
499
- # Store raw markdown for debugging if needed, but not primarily for export format
500
- global_raw_markdown_for_export = raw_rag_output + disclaimer_text_for_raw
501
 
502
- progress(0.9, desc="Parsing AI output and generating summary report...")
503
 
504
- # --- HTML Formatting for On-Page Display (Non-Table Visual Summary) ---
505
- analysis_df = pd.DataFrame()
506
  summarized_analysis_html = ""
507
-
508
  try:
509
- # Use the more robust parsing function
510
- analysis_df, parsing_error = parse_markdown_table_to_df(raw_rag_output)
511
-
512
- if parsing_error:
513
- print(f"Warning: {parsing_error}")
514
- if raw_rag_output.strip():
515
- # Fallback: if no table is found but AI output exists, show it as generic text
516
- summarized_analysis_html = f"""
517
- <div style="margin-bottom: 20px;">
518
- <p style="color: #D32F2F;'>AI produced output but could not structure it into a table for the visual summary.</p>
519
- <p style="color: #D32F2F;'>Here is the raw AI output:</p>
520
- <pre style="background-color: #EEE; padding: 15px; border: 1px solid #DDD; overflow-x: auto; white-space: pre-wrap; word-wrap: break-word;">{raw_rag_output}</pre>
521
- </div>
522
- """
523
- else:
524
- summarized_analysis_html = "<p style='color:orange;'>AI produced no structured output or could not be parsed.</p>"
525
- elif not analysis_df.empty:
526
- if 'Count' in analysis_df.columns:
527
- analysis_df['Count'] = pd.to_numeric(analysis_df['Count'], errors='coerce').fillna(0).astype(int)
528
- else:
529
- analysis_df['Count'] = 1
530
- print("Warning: 'Count' column not found in AI output. Defaulting counts to 1.")
531
-
532
  summarized_analysis_html = generate_summary_html_by_category(analysis_df)
533
- global_analysis_df_for_export = analysis_df # Store the *parsed* DF for export
534
  else:
535
- summarized_analysis_html = "<p style='color:orange;'>AI output parsed into an empty DataFrame. This might mean the table had headers but no data rows.</p>"
536
- global_analysis_df_for_export = pd.DataFrame() # Clear DF if empty
537
 
538
  except Exception as e:
539
- print(f"Failed to parse AI markdown table into DataFrame for HTML display or generate summary: {e}")
540
  summarized_analysis_html = f"""
541
  <div style="margin-bottom: 20px;">
542
  <p style="color: #D32F2F;'>An unexpected error occurred while attempting to transform AI output into a visual summary: {e}</p>
543
- <p style="color: #D32F2F;'>Here is the raw AI output (could not be structured):</p>
544
- <pre style="background-color: #EEE; padding: 15px; border: 1px solid #DDD; overflow-x: auto; white-space: pre-wrap; word-wrap: break-word;">{raw_rag_output}</pre>
545
  </div>
546
  """
547
- global_analysis_df_for_export = pd.DataFrame() # Clear DF on error
 
548
 
549
  html_output = f"""
550
  <div style="font-family: Arial, sans-serif; padding: 20px; background-color: #f9f9f9; border-radius: 10px; box-shadow: 0 2px 10px rgba(0,0,0,0.1);">
@@ -556,6 +638,12 @@ def perform_ticket_analysis(file, user_query, progress=gr.Progress()):
556
  This report provides a data-driven analysis of recent customer support tickets, leveraging advanced AI to identify trends, root causes, and actionable insights.
557
  Below is a summary of identified issues and a detailed breakdown.
558
  </p>
 
 
 
 
 
 
559
  </div>
560
  """
561
 
@@ -607,7 +695,16 @@ def generate_pdf_report():
607
  pdf.set_font("Arial", 'B', 14)
608
  pdf.multi_cell(0, 10, "Overview & Key Insights")
609
  pdf.set_font("Arial", size=10)
610
- pdf.multi_cell(0, 7, "This report provides a data-driven analysis of recent customer support tickets, leveraging advanced AI to identify trends, root causes, and actionable insights. Below is a summary of identified issues and a detailed breakdown.")
 
 
 
 
 
 
 
 
 
611
  pdf.ln(10)
612
 
613
  # Aggregated Issue Summaries
@@ -714,10 +811,6 @@ def generate_docx_report():
714
 
715
  doc = DocxDocument()
716
 
717
- # Set document-wide font (optional, but good practice)
718
- # doc.styles['Normal'].font.name = 'Arial'
719
- # doc.styles['Normal'].font.size = Pt(11)
720
-
721
  # Title
722
  heading = doc.add_heading("AI Customer Ticket Analysis Report", level=1)
723
  heading.alignment = WD_ALIGN_PARAGRAPH.CENTER
@@ -730,10 +823,15 @@ def generate_docx_report():
730
 
731
  # Overview Section
732
  doc.add_heading("Overview & Key Insights", level=2)
 
 
 
733
  doc.add_paragraph(
734
  "This report provides a data-driven analysis of recent customer support tickets, leveraging advanced AI to identify trends, root causes, and actionable insights. "
735
  "Below is a summary of identified issues and a detailed breakdown."
736
  )
 
 
737
  doc.add_paragraph("") # Spacer
738
 
739
  # Aggregated Issue Summaries
@@ -856,7 +954,7 @@ with gr.Blocks(theme=gr.themes.Soft(), css="""
856
  * **Dynamic Column Reading:** The tool will attempt to read **ALL** columns present in your file. Ensure column headers are clear and meaningful.
857
  * **Minimum Columns:** A **minimum of two columns are highly recommended**: one for a unique `Ticket ID` (or `ID`) and one for the `Issue` (or `Description`).
858
  * **Recommended Columns:** For richer analysis, include columns like `Status`, `Priority`, `CustomerName`, `Timestamp`, etc. The more relevant details provided, the better the AI's inferences.
859
- * **Size Recommendation:** For optimal performance on the free tier, we recommend files with up to **~1000 rows**. Larger files might experience slower processing or hit API rate limits.
860
  * **Privacy:** **DO NOT upload sensitive or confidential customer information.** This is a publicly hosted demo.
861
 
862
  ---
@@ -914,17 +1012,15 @@ with gr.Blocks(theme=gr.themes.Soft(), css="""
914
  fn=perform_ticket_analysis,
915
  inputs=[file_input, query_input],
916
  outputs=output_html_display,
917
- queue=True # Enable queuing for progress updates to work properly
918
  )
919
 
920
- # Use .then() to make the download buttons visible
921
- # and call the report generation functions
922
  pdf_btn.click(
923
  fn=generate_pdf_report,
924
  outputs=pdf_output_file,
925
  api_name="download_pdf"
926
  ).then(
927
- fn=lambda: gr.File(visible=True, value=None), # Make visible and clear previous file
928
  inputs=[],
929
  outputs=pdf_output_file
930
  )
@@ -934,7 +1030,7 @@ with gr.Blocks(theme=gr.themes.Soft(), css="""
934
  outputs=docx_output_file,
935
  api_name="download_docx"
936
  ).then(
937
- fn=lambda: gr.File(visible=True, value=None), # Make visible and clear previous file
938
  inputs=[],
939
  outputs=docx_output_file
940
  )
 
5
  from datetime import datetime
6
  import io
7
  import re
8
+ import google.generativeai as genai
9
+ import faiss
10
+ import numpy as np
11
+ import time
12
+ import json # For saving intermediate results if no DB is used, or for larger JSON processing
13
 
14
  # --- File Generation Libraries ---
15
  from fpdf import FPDF
 
19
  from docx.shared import Pt
20
  from docx.enum.text import WD_PARAGRAPH_ALIGNMENT
21
 
22
+ # Load environment variables
23
  load_dotenv()
24
 
25
  # --- RAG Setup ---
26
  rag_initialized = False
27
  gemini_llm_model = None
28
+ gemini_embedding_model_name = "models/embedding-001"
29
 
 
30
  google_api_key = os.getenv("GOOGLE_API_KEY")
31
 
32
  if not google_api_key:
 
41
 
42
  gemini_llm_model = genai.GenerativeModel(llm_model_id)
43
 
 
44
  test_embedding = genai.embed_content(model=gemini_embedding_model_name, content=["test"])
45
  if test_embedding and test_embedding['embedding']:
46
  rag_initialized = True
 
54
  print(f"Please verify your GOOGLE_API_KEY's validity and ensure the specified AI model ('{llm_model_id}' for LLM) and embedding model ('{gemini_embedding_model_name}') are available to your API key.")
55
  print("RAG functionality will not work without a valid Google API key and accessible model.")
56
 
 
57
  # Global variable to store the PARSED DATAFRAME for download functions
58
+ # FOR VERY LARGE DATASETS, THIS WOULD BE REPLACED BY DATABASE CALLS
59
  global_analysis_df_for_export = pd.DataFrame()
60
+ global_raw_markdown_for_export = ""
61
 
62
  # --- Custom Document Structure ---
63
  # We'll use a dictionary: {"page_content": "...", "metadata": {...}}
64
 
65
+ # --- Custom Text Splitter (might not be needed if tickets are already chunks) ---
66
  def custom_character_text_splitter(text_documents, chunk_size=1000, chunk_overlap=200):
67
  """
68
  A simple character-based text splitter.
 
74
  content = doc["page_content"]
75
  metadata = doc["metadata"]
76
 
 
77
  if len(content) <= chunk_size:
78
  chunks.append({"page_content": content, "metadata": metadata})
79
  continue
80
 
 
81
  current_position = 0
82
  while current_position < len(content):
83
  chunk_end = min(current_position + chunk_size, len(content))
84
  chunk_text = content[current_position:chunk_end]
85
  chunks.append({"page_content": chunk_text, "metadata": metadata})
86
 
87
+ if chunk_end == len(content):
 
88
  break
89
  current_position += (chunk_size - chunk_overlap)
 
90
  if current_position >= len(content):
91
  break
92
  return chunks
93
 
94
 
95
+ # --- Custom FAISS Integration (for searching, not direct analysis flow for 800k tickets) ---
96
  class CustomFAISSVectorStore:
97
  def __init__(self, embedding_model_name, dimension=768): # Gemini embedding-001 has 768 dimensions
98
  self.embedding_model_name = embedding_model_name
 
131
  if not query_text:
132
  return []
133
 
 
134
  try:
135
  query_embedding_response = genai.embed_content(
136
  model=self.embedding_model_name,
 
143
  if query_embedding.shape[1] != self.index.d:
144
  raise ValueError(f"Query embedding dimension mismatch: Expected {self.index.d}, got {query_embedding.shape[1]}.")
145
 
 
146
  distances, indices = self.index.search(query_embedding, k)
147
 
148
  retrieved_docs = []
149
  for i in indices[0]:
150
+ if 0 <= i < len(self.documents):
151
  retrieved_docs.append(self.documents[i])
152
  return retrieved_docs
153
 
154
 
155
+ def get_ticket_documents_iterator(file_path, file_extension, filename, chunksize=1000):
156
  """
157
+ Loads and returns text from Excel or CSV files in chunks (iterator).
158
+ For very large files, chunksize helps manage memory.
159
+ Yields list of dictionaries: {"page_content": "...", "metadata": {...}}
160
  """
161
  print(f"Attempting to load customer ticket data: {filename} with extension {file_extension}")
 
162
 
163
  if file_extension in ['.xlsx', '.xls']:
164
  try:
165
+ # Use openpyxl engine for .xlsx, allows chunking
166
+ reader = pd.read_excel(file_path, engine='openpyxl', chunksize=chunksize)
167
  except Exception as e:
168
  raise ValueError(f"Could not process Excel file '{filename}': {e}. Ensure it's a valid Excel format for ticket data.")
169
  elif file_extension == '.csv':
170
  try:
171
+ reader = pd.read_csv(file_path, chunksize=chunksize)
 
172
  except Exception as e:
173
  raise ValueError(f"Could not process CSV file '{filename}': {e}. Ensure it's a valid CSV format for ticket data.")
174
  else:
175
  raise ValueError(f"Unsupported file type: {file_extension}. Only .xlsx, .xls, or .csv files are accepted for ticket data.")
176
 
177
+ total_rows_processed = 0
178
+ for i, df_chunk in enumerate(reader):
179
+ chunk_documents = []
180
+ if df_chunk.empty:
181
+ continue
 
182
 
183
+ for index, row in df_chunk.iterrows():
184
+ ticket_content_parts = []
185
+ ticket_id = f"Row {total_rows_processed + 1}" # Default ticket ID
186
+
187
+ for col_name, value in row.items():
188
+ if pd.notna(value) and str(value).strip() != '':
189
+ formatted_col_name = col_name.replace('_', ' ').title()
190
+ ticket_content_parts.append(f"{formatted_col_name}: {value}")
191
+
192
+ if col_name.lower() in ['ticketid', 'ticket_id', 'id']:
193
+ ticket_id = str(value).strip()
194
+
195
+ if not ticket_content_parts:
196
+ ticket_content = f"Ticket {ticket_id}: No detailed content provided."
197
+ else:
198
+ ticket_content = "\n".join(ticket_content_parts)
199
 
200
+ metadata = {"source": filename, "ticket_row_number": total_rows_processed + 1, "ticket_id": ticket_id}
201
+ metadata.update(row.to_dict()) # Include all original row data in metadata
202
+
203
+ chunk_documents.append({"page_content": ticket_content, "metadata": metadata})
204
+ total_rows_processed += 1
205
 
206
+ yield chunk_documents, total_rows_processed # Yield chunk and running total
207
+
208
+ if total_rows_processed == 0:
209
+ raise ValueError("The uploaded file contains no data or no rows could be parsed.")
210
+ print(f"Total rows processed from file: {total_rows_processed}")
 
 
211
 
212
 
213
  def generate_summary_html_by_category(df):
 
219
  return "<p style='color:orange;'>No categories found or data is empty for detailed summarization. This could be due to issues with the AI's output parsing or content generation.</p>"
220
 
221
  summary_html = ""
222
+ # Ensure all required columns exist, even if with 'N/A' to prevent key errors
223
  required_cols = ['Issue Description (Summary)', 'Sub-Category', 'Root Cause (Inferred)',
224
  'Potential Resolution Steps', 'Future Prediction', 'Proactive Measures', 'Count']
225
  for col in required_cols:
226
  if col not in df.columns:
227
+ df[col] = 'N/A' # Default to N/A if column is missing
228
+ if col == 'Count': df[col] = 0 # Default count to 0 for missing count column
229
 
230
+ # Group by category, ensuring consistent column order for safety
231
  for category, category_df in df.groupby('Category'):
232
  total_count = category_df['Count'].sum()
233
 
 
252
  unique_root_causes = category_df['Root Cause (Inferred)']\
253
  .astype(str)\
254
  .drop_duplicates()\
255
+ .apply(lambda x: x.replace('*', '')).tolist()
 
256
  for rc in unique_root_causes:
257
  root_causes += f"<li>{rc}</li>"
258
  root_causes += "</ul>" if unique_root_causes else "<p>Undetermined.</p>"
 
261
  unique_resolutions = category_df['Potential Resolution Steps']\
262
  .astype(str)\
263
  .drop_duplicates()\
264
+ .apply(lambda x: x.replace('*', '')).tolist()
 
265
  for res in unique_resolutions:
266
  resolution_steps += f"<li>{res}</li>"
267
  resolution_steps += "</ul>" if unique_resolutions else "<p>No specific steps identified.</p>"
 
313
  def parse_markdown_table_to_df(markdown_string):
314
  """
315
  Robustly parses a markdown table string into a pandas DataFrame.
316
+ Assumes a standard markdown table with a header and separator line.
317
  """
318
  lines = markdown_string.strip().split('\n')
319
 
320
  header_line = None
321
  separator_line_index = -1
322
 
 
323
  for i, line in enumerate(lines):
324
  if '|' in line and '---' not in line and not line.strip().startswith('|---'):
325
  header_line = line.strip()
326
  elif '|---' in line:
327
  separator_line_index = i
328
+ break
329
 
330
  if header_line is None or separator_line_index == -1:
331
  return pd.DataFrame(), "No valid Markdown table header or separator found."
332
 
 
333
  data_lines = []
334
  for line in lines[separator_line_index + 1:]:
335
  if line.strip().startswith('|') and '|' in line.strip():
336
  data_lines.append(line.strip())
337
  else:
 
338
  break
339
 
340
  if not data_lines:
341
  return pd.DataFrame(), "No data rows found in the Markdown table."
342
 
 
 
343
  headers = [h.strip() for h in header_line.strip('|').split('|')]
344
+ headers = [h for h in headers if h] # Remove empty strings
 
345
 
 
346
  data = []
347
  for line in data_lines:
 
348
  row_values = [v.strip() for v in line.strip('|').split('|')]
349
+ row_values = [v for v in row_values if v]
350
 
 
351
  if len(row_values) == len(headers):
352
  data.append(row_values)
353
  else:
 
357
  return pd.DataFrame(), "No valid data rows parsed after cleaning."
358
 
359
  df = pd.DataFrame(data, columns=headers)
360
+ df = df.dropna(axis=1, how='all')
361
+ df.columns = df.columns.str.strip()
362
 
363
  return df, None
364
 
 
366
  def perform_ticket_analysis(file, user_query, progress=gr.Progress()):
367
  """
368
  Main function to analyze customer tickets using custom RAG implementation and format output as rich HTML.
369
+ Implements a two-phase iterative approach to handle large ticket sets robustly.
 
370
  """
371
  global global_analysis_df_for_export
372
+ global global_raw_markdown_for_export
373
 
374
  start_time = time.time()
375
  error_message_markdown = ""
376
 
 
377
  progress(0, desc="Starting analysis...")
378
 
379
  if not rag_initialized:
 
388
  error_message_markdown = "Please upload a customer ticket file."
389
  return f"<p style='color:red;'>{error_message_markdown}</p>"
390
 
 
 
 
 
391
  file_path = file.name
392
  filename = os.path.basename(file_path)
393
  file_extension = os.path.splitext(filename)[1].lower()
394
 
 
395
  disclaimer_text_for_raw = "\n\n**Note:** 'Root Cause', 'Future Prediction', and 'Proactive Measures' are AI-inferred based on descriptions provided. For critical decisions, human expert review is essential."
396
 
397
  try:
398
+ # --- PHASE 1: Initial Categorization of ALL Tickets ---
399
+ progress(0.1, desc="Phase 1/2: Initializing categorization...")
400
 
401
+ # Use an iterator to handle potentially very large files
402
+ ticket_iterator = get_ticket_documents_iterator(file_path, file_extension, filename, chunksize=5000) # Process 5000 tickets at a time
 
 
 
 
403
 
404
+ all_categorized_tickets = []
405
+ total_tickets_count = 0
 
406
 
407
+ # Simulate a small database for categorized tickets
408
+ # In a real large-scale system, this would be a database connection.
409
+ categorized_tickets_db = []
 
410
 
411
+ for i, (chunk_documents, current_total_processed) in enumerate(ticket_iterator):
412
+ total_tickets_count = current_total_processed
413
+ progress(0.1 + (0.4 * (i+1) / (total_tickets_count / 5000 + 1)), desc=f"Phase 1/2: Categorizing tickets... {total_tickets_count} processed so far.")
414
+
415
+ if not chunk_documents:
416
+ continue
417
+
418
+ categorization_context = ""
419
+ ticket_id_map = {} # To map original ticket_id to content for later lookup within this chunk
420
+ for doc in chunk_documents:
421
+ ticket_id_for_prompt = str(doc["metadata"].get("ticket_id", f"Row {doc['metadata']['ticket_row_number']}"))
422
+ categorization_context += f"--- Ticket ID: {ticket_id_for_prompt} ---\n{doc['page_content']}\n\n"
423
+ ticket_id_map[ticket_id_for_prompt] = doc["page_content"]
424
+
425
+ categorization_prompt = f"""
426
+ You are an expert customer support ticket analyst. Your task is to categorize ALL provided customer tickets in this batch.
427
+ For each ticket, identify a concise main category (e.g., 'Technical Issue', 'Billing Inquiry', 'Feature Request', 'Account Management', 'Performance Problem', 'Data Access', 'General Inquiry').
428
+ Aim for **6 to 8 distinct main categories**.
429
+ Your output MUST be a simple list, one ticket per line, in the format:
430
+ Ticket ID: [Original Ticket ID from metadata] - Category: [Determined Category] - Issue Summary: [Very brief 1-2 sentence summary of the core issue]
431
+
432
+ Example:
433
+ Ticket ID: TKT-001 - Category: Technical Issue - Issue Summary: User cannot log in due to password error.
434
+ Ticket ID: INV-567 - Category: Billing Inquiry - Issue Summary: Customer dispute charge for duplicate service.
435
+
436
+ Context (Customer Tickets - analyze ALL of these, total {len(chunk_documents)} tickets in this batch):
437
+ {categorization_context}
438
+ """
439
+
440
+ try:
441
+ # Add retry logic and error handling for API calls here
442
+ categorization_response = gemini_llm_model.generate_content(categorization_prompt)
443
+ categorization_raw_output = categorization_response.text
444
+
445
+ # Parse categorization output
446
+ for line in categorization_raw_output.split('\n'):
447
+ line = line.strip()
448
+ match = re.match(r'Ticket ID: (.*?) - Category: (.*?) - Issue Summary: (.*)', line)
449
+ if match:
450
+ ticket_id = match.group(1).strip()
451
+ category = match.group(2).strip()
452
+ issue_summary = match.group(3).strip()
453
+ original_content = ticket_id_map.get(ticket_id, "Original content not found for this ID.")
454
+ categorized_tickets_db.append({
455
+ "ticket_id": ticket_id,
456
+ "category": category,
457
+ "issue_summary_short": issue_summary,
458
+ "original_content": original_content
459
+ })
460
+ elif line:
461
+ print(f"Warning: Line did not match categorization format in chunk {i}: {line}")
462
+ # Fallback parsing for malformed lines
463
+ parts = line.split("Category:")
464
+ ticket_id_part = parts[0].replace("Ticket ID:", "").strip() if len(parts) > 0 else f"Unparsed_Line_{len(categorized_tickets_db)+1}"
465
+ category_part = parts[1].split("Issue Summary:")[0].strip() if len(parts) > 1 and "Issue Summary:" in parts[1] else "Uncategorized/Parsing Error"
466
+ issue_summary_part = parts[1].split("Issue Summary:", 1)[1].strip() if len(parts) > 1 and "Issue Summary:" in parts[1] else line
467
+ original_content = ticket_id_map.get(ticket_id_part, "Original content not found / Raw line.")
468
+ categorized_tickets_db.append({
469
+ "ticket_id": ticket_id_part,
470
+ "category": category_part,
471
+ "issue_summary_short": issue_summary_part,
472
+ "original_content": original_content
473
+ })
474
+
475
+ except Exception as e:
476
+ print(f"Error during categorization of chunk {i}: {e}")
477
+ # For a real system, you'd log this, and potentially save the raw chunk for re-processing.
478
+ # For this demo, we'll just skip the chunk or add a generic error entry.
479
+ for doc in chunk_documents:
480
+ ticket_id = str(doc["metadata"].get("ticket_id", f"Row {doc['metadata']['ticket_row_number']}"))
481
+ categorized_tickets_db.append({
482
+ "ticket_id": ticket_id,
483
+ "category": "Error_Processing_Chunk",
484
+ "issue_summary_short": f"Failed to categorize due to error: {e}",
485
+ "original_content": doc["page_content"]
486
+ })
487
 
488
+ # After processing all chunks, convert to DataFrame for easier manipulation
489
+ final_categorized_df = pd.DataFrame(categorized_tickets_db)
490
+ if final_categorized_df.empty:
491
+ error_message_markdown = "AI failed to categorize any tickets. Please ensure the ticket data is clear."
492
+ return f"<p style='color:red;'>{error_message_markdown}</p>"
493
+
494
+ # --- PHASE 2: Detailed Analysis per Category ---
495
+ progress(0.5, desc=f"Phase 2/2: Detailing analysis for categories...")
496
 
497
+ final_analysis_rows = []
498
+ unique_categories = final_categorized_df['category'].unique()
499
+
500
+ # Define a batch size for processing categories for detailed analysis if there are many categories
501
+ # For 6-8 categories, this internal batching might not be strictly necessary, but good practice.
502
+ category_batch_size = 5
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
503
 
504
+ for i, category_chunk in enumerate(np.array_split(unique_categories, max(1, len(unique_categories) // category_batch_size))):
505
+ progress((0.5 + (0.45 * i / len(np.array_split(unique_categories, max(1, len(unique_categories) // category_batch_size))))), desc=f"Phase 2/2: Detailing analysis for category batch {i+1}...")
506
 
507
+ for category in category_chunk:
508
+ category_specific_tickets_df = final_categorized_df[final_categorized_df['category'] == category]
509
+
510
+ # Combine relevant content for the category. For 800k tickets, this concatenation could still be huge.
511
+ # Consider summarizing `original_content` for each ticket in Phase 1 if context window is an issue.
512
+ category_tickets_content = "\n\n".join(category_specific_tickets_df['original_content'].tolist())
513
+
514
+ category_count = len(category_specific_tickets_df)
515
+
516
+ detail_prompt = f"""
517
+ You are an expert customer support ticket analyst.
518
+ Analyze the following customer tickets, which all belong to the '{category}' category.
519
+ Your output MUST be a **single Markdown table row** that summarizes the common themes, root causes, resolution steps, future predictions, and proactive measures *for this specific category of tickets*.
520
 
521
+ **Ensure the 'Count' in your table row EXACTLY matches the number of tickets provided for this category ({category_count}).**
522
+
523
+ Keep 'Issue Description (Summary)' concise (max 3-4 sentences), summarizing common themes *across these tickets*.
524
+ If a specific detail (like root cause) cannot be inferred from the available ticket data, state 'Undetermined' or 'N/A'.
525
+ Do not make up information.
526
 
527
+ **Desired Output Format (Markdown Table - single row):**
528
+ | Category | Sub-Category | Count | Issue Description (Summary) | Root Cause (Inferred) | Potential Resolution Steps | Future Prediction | Proactive Measures |
529
+ |---|---|---|---|---|---|---|---|
530
+ | ... | ... | ... | ... | ... | ... | ... | ... |
531
 
532
+ Context (Tickets for Category '{category}'):
533
+ {category_tickets_content}
534
+ """
535
+
536
+ try:
537
+ # Add retry logic and error handling for API calls here
538
+ detail_response = gemini_llm_model.generate_content(detail_prompt)
539
+ detail_raw_output = detail_response.text
540
+
541
+ detailed_df, parsing_err = parse_markdown_table_to_df(detail_raw_output)
542
+ if parsing_err or detailed_df.empty:
543
+ print(f"Warning: Parsing error or empty DataFrame for category '{category}': {parsing_err or 'Empty DataFrame'}")
544
+ print(f"Raw AI output for detail: {detail_raw_output}")
545
+ default_row = {
546
+ "Category": category,
547
+ "Sub-Category": "N/A",
548
+ "Count": category_count,
549
+ "Issue Description (Summary)": category_specific_tickets_df['issue_summary_short'].iloc[0] if not category_specific_tickets_df.empty else "Error in summary generation/parsing for this category.",
550
+ "Root Cause (Inferred)": "Undetermined due to parsing error.",
551
+ "Potential Resolution Steps": "N/A",
552
+ "Future Prediction": "N/A",
553
+ "Proactive Measures": "N/A"
554
+ }
555
+ final_analysis_rows.append(default_row)
556
+ else:
557
+ detailed_row = detailed_df.iloc[0].to_dict()
558
+ detailed_row['Category'] = category
559
+
560
+ if 'Count' in detailed_row:
561
+ try:
562
+ ai_reported_count = pd.to_numeric(detailed_row['Count'], errors='coerce')
563
+ if pd.isna(ai_reported_count) or int(ai_reported_count) != category_count:
564
+ print(f"Warning: AI-reported count {ai_reported_count} for '{category}' does not match actual count {category_count}. Correcting.")
565
+ detailed_row['Count'] = category_count
566
+ except:
567
+ detailed_row['Count'] = category_count
568
+ else:
569
+ detailed_row['Count'] = category_count
570
+
571
+ final_analysis_rows.append(detailed_row)
572
+
573
+ except Exception as e:
574
+ print(f"Error detailing analysis for category '{category}': {e}")
575
+ default_row = {
576
+ "Category": category,
577
+ "Sub-Category": "N/A",
578
+ "Count": category_count,
579
+ "Issue Description (Summary)": f"Error during detailed analysis for this category: {e}",
580
+ "Root Cause (Inferred)": "Undetermined due to error.",
581
+ "Potential Resolution Steps": "N/A",
582
+ "Future Prediction": "N/A",
583
+ "Proactive Measures": "N/A"
584
+ }
585
+ final_analysis_rows.append(default_row)
586
+
587
+ analysis_df = pd.DataFrame(final_analysis_rows)
588
+ if 'Count' in analysis_df.columns:
589
+ analysis_df['Count'] = pd.to_numeric(analysis_df['Count'], errors='coerce').fillna(0).astype(int)
590
+
591
+ summarized_count = analysis_df['Count'].sum()
592
+ if summarized_count != total_tickets_count:
593
+ print(f"CRITICAL WARNING: Total count in summarized DF ({summarized_count}) does not match original tickets ({total_tickets_count}).")
594
+
595
+ global_analysis_df_for_export = analysis_df.copy()
596
+ global_raw_markdown_for_export = analysis_df.to_markdown(index=False) + disclaimer_text_for_raw
597
 
598
  except ValueError as e:
599
+ error_message_markdown = f"Ticket Analysis Error: {e}"
600
+ global_raw_markdown_for_export = error_message_markdown + disclaimer_text_for_raw
601
+ global_analysis_df_for_export = pd.DataFrame()
602
+ return f"<p style='color:red;'>{error_message_markdown}</p>"
603
  except Exception as e:
604
+ error_message_markdown = f"An unexpected error occurred during analysis: {e}. Check server logs for details."
605
  print(f"Error during RAG analysis: {e}")
606
+ global_raw_markdown_for_export = error_message_markdown + disclaimer_text_for_raw
607
+ global_analysis_df_for_export = pd.DataFrame()
608
+ return f"<p style='color:red;'>{error_message_markdown}</p>"
 
 
 
609
 
610
+ progress(0.98, desc="Generating summary report for display...")
611
 
 
 
612
  summarized_analysis_html = ""
 
613
  try:
614
+ if not analysis_df.empty:
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
615
  summarized_analysis_html = generate_summary_html_by_category(analysis_df)
 
616
  else:
617
+ summarized_analysis_html = "<p style='color:orange;'>AI output parsed into an empty DataFrame or no categories were generated. Please check the uploaded file and try again.</p>"
 
618
 
619
  except Exception as e:
620
+ print(f"Failed to generate HTML summary from DataFrame: {e}")
621
  summarized_analysis_html = f"""
622
  <div style="margin-bottom: 20px;">
623
  <p style="color: #D32F2F;'>An unexpected error occurred while attempting to transform AI output into a visual summary: {e}</p>
624
+ <p style="color: #D32F2F;'>Raw AI output (if available):</p>
625
+ <pre style="background-color: #EEE; padding: 15px; border: 1px solid #DDD; overflow-x: auto; white-space: pre-wrap; word-wrap: break-word;">{global_raw_markdown_for_export}</pre>
626
  </div>
627
  """
628
+ global_analysis_df_for_export = pd.DataFrame()
629
+
630
 
631
  html_output = f"""
632
  <div style="font-family: Arial, sans-serif; padding: 20px; background-color: #f9f9f9; border-radius: 10px; box-shadow: 0 2px 10px rgba(0,0,0,0.1);">
 
638
  This report provides a data-driven analysis of recent customer support tickets, leveraging advanced AI to identify trends, root causes, and actionable insights.
639
  Below is a summary of identified issues and a detailed breakdown.
640
  </p>
641
+ <p style="font-size: 1.0em; line-height: 1.6; color: #424242;">
642
+ Total Original Tickets in File: <strong>{total_tickets_count}</strong>.<br>
643
+ Total Tickets Summarized in Report: <strong>{summarized_count}</strong>.<br>
644
+ Total Categories Identified: <strong>{len(unique_categories) if 'unique_categories' in locals() else 0}</strong>.
645
+ {"<p style='color:red; font-weight:bold;'>WARNING: The number of tickets summarized in the report does not match the total original tickets. This may indicate a parsing or AI processing issue. Please review the raw output if available.</p>" if summarized_count != total_tickets_count else ""}
646
+ </p>
647
  </div>
648
  """
649
 
 
695
  pdf.set_font("Arial", 'B', 14)
696
  pdf.multi_cell(0, 10, "Overview & Key Insights")
697
  pdf.set_font("Arial", size=10)
698
+ total_tickets_processed = df['Count'].sum() if 'Count' in df.columns else 0
699
+ total_categories_identified = len(df['Category'].unique()) if 'Category' in df.columns else 0
700
+
701
+ overview_text = (
702
+ "This report provides a data-driven analysis of recent customer support tickets, leveraging advanced AI to identify trends, root causes, and actionable insights. "
703
+ "Below is a summary of identified issues and a detailed breakdown.\n\n"
704
+ f"Total Tickets Summarized in Report: {total_tickets_processed}.\n"
705
+ f"Total Categories Identified: {total_categories_identified}."
706
+ )
707
+ pdf.multi_cell(0, 7, overview_text)
708
  pdf.ln(10)
709
 
710
  # Aggregated Issue Summaries
 
811
 
812
  doc = DocxDocument()
813
 
 
 
 
 
814
  # Title
815
  heading = doc.add_heading("AI Customer Ticket Analysis Report", level=1)
816
  heading.alignment = WD_ALIGN_PARAGRAPH.CENTER
 
823
 
824
  # Overview Section
825
  doc.add_heading("Overview & Key Insights", level=2)
826
+ total_tickets_processed = df['Count'].sum() if 'Count' in df.columns else 0
827
+ total_categories_identified = len(df['Category'].unique()) if 'Category' in df.columns else 0
828
+
829
  doc.add_paragraph(
830
  "This report provides a data-driven analysis of recent customer support tickets, leveraging advanced AI to identify trends, root causes, and actionable insights. "
831
  "Below is a summary of identified issues and a detailed breakdown."
832
  )
833
+ doc.add_paragraph(f"Total Tickets Summarized in Report: {total_tickets_processed}.")
834
+ doc.add_paragraph(f"Total Categories Identified: {total_categories_identified}.")
835
  doc.add_paragraph("") # Spacer
836
 
837
  # Aggregated Issue Summaries
 
954
  * **Dynamic Column Reading:** The tool will attempt to read **ALL** columns present in your file. Ensure column headers are clear and meaningful.
955
  * **Minimum Columns:** A **minimum of two columns are highly recommended**: one for a unique `Ticket ID` (or `ID`) and one for the `Issue` (or `Description`).
956
  * **Recommended Columns:** For richer analysis, include columns like `Status`, `Priority`, `CustomerName`, `Timestamp`, etc. The more relevant details provided, the better the AI's inferences.
957
+ * **Scale Handling:** For very large files (e.g., 60,000+ tickets), the system will process data in **batches** to manage memory and API limits. Be aware that processing can take a significant amount of time.
958
  * **Privacy:** **DO NOT upload sensitive or confidential customer information.** This is a publicly hosted demo.
959
 
960
  ---
 
1012
  fn=perform_ticket_analysis,
1013
  inputs=[file_input, query_input],
1014
  outputs=output_html_display,
1015
+ queue=True
1016
  )
1017
 
 
 
1018
  pdf_btn.click(
1019
  fn=generate_pdf_report,
1020
  outputs=pdf_output_file,
1021
  api_name="download_pdf"
1022
  ).then(
1023
+ fn=lambda: gr.File(visible=True, value=None),
1024
  inputs=[],
1025
  outputs=pdf_output_file
1026
  )
 
1030
  outputs=docx_output_file,
1031
  api_name="download_docx"
1032
  ).then(
1033
+ fn=lambda: gr.File(visible=True, value=None),
1034
  inputs=[],
1035
  outputs=docx_output_file
1036
  )