Neom99 commited on
Commit
dcd9524
·
verified ·
1 Parent(s): c08fe92

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +57 -13
app.py CHANGED
@@ -27,7 +27,7 @@ rag_initialized = False
27
  gemini_llm_model = None
28
  gemini_embedding_model_name = "models/embedding-001"
29
 
30
- google_api_key = os.getenv("GOOGLE_API_KEY")
31
 
32
  if not google_api_key:
33
  print("\n" + "="*80)
@@ -419,20 +419,29 @@ def perform_ticket_analysis(file, user_query, progress=gr.Progress()):
419
 
420
  start_time = time.time()
421
  error_message_markdown = ""
 
422
 
 
 
 
 
 
423
  progress(0, desc="Starting analysis...")
424
 
425
  if not rag_initialized:
426
  error_message_markdown = "RAG system not initialized. Please ensure your GOOGLE_API_KEY is configured correctly and the specified AI model is accessible."
427
- return f"<p style='color:red;'>{error_message_markdown}</p>"
 
428
 
429
  if gemini_llm_model is None:
430
  error_message_markdown = "Google Generative AI LLM was not initialized. Please check startup logs."
431
- return f"<p style='color:red;'>{error_message_markdown}</p>"
 
432
 
433
  if file is None:
434
  error_message_markdown = "Please upload a customer ticket file."
435
- return f"<p style='color:red;'>{error_message_markdown}</p>"
 
436
 
437
  file_path = file.name
438
  filename = os.path.basename(file_path)
@@ -442,6 +451,7 @@ def perform_ticket_analysis(file, user_query, progress=gr.Progress()):
442
 
443
  try:
444
  # --- PHASE 1: Initial Categorization of ALL Tickets ---
 
445
  progress(0.1, desc="Phase 1/2: Initializing categorization...")
446
 
447
  # Use an iterator to handle potentially very large files
@@ -458,7 +468,9 @@ def perform_ticket_analysis(file, user_query, progress=gr.Progress()):
458
  total_tickets_count = current_total_processed
459
  # Adjust progress based on actual total, or a rough estimate if total_tickets_count is not yet final
460
  estimated_total_chunks = max(1, total_tickets_count // 5000 + 1) # Crude estimate
461
- progress(0.1 + (0.4 * (i+1) / estimated_total_chunks), desc=f"Phase 1/2: Categorizing tickets... {total_tickets_count} processed so far.")
 
 
462
 
463
  if not chunk_documents:
464
  continue
@@ -490,8 +502,10 @@ def perform_ticket_analysis(file, user_query, progress=gr.Progress()):
490
 
491
  try:
492
  # Add retry logic and error handling for API calls here
 
493
  categorization_response = gemini_llm_model.generate_content(categorization_prompt)
494
  categorization_raw_output = categorization_response.text
 
495
 
496
  # Parse categorization output
497
  for line in categorization_raw_output.split('\n'):
@@ -510,6 +524,7 @@ def perform_ticket_analysis(file, user_query, progress=gr.Progress()):
510
  })
511
  elif line:
512
  print(f"Warning: Line did not match categorization format in chunk {i}: {line}")
 
513
  # Fallback parsing for malformed lines
514
  parts = line.split("Category:")
515
  ticket_id_part = parts[0].replace("Ticket ID:", "").strip() if len(parts) > 0 else f"Unparsed_Line_{len(categorized_tickets_db)+1}"
@@ -524,6 +539,7 @@ def perform_ticket_analysis(file, user_query, progress=gr.Progress()):
524
  })
525
 
526
  except Exception as e:
 
527
  print(f"Error during categorization of chunk {i}: {e}")
528
  # For a real system, you'd log this, and potentially save the raw chunk for re-processing.
529
  # For this demo, we'll just skip the chunk or add a generic error entry.
@@ -540,20 +556,25 @@ def perform_ticket_analysis(file, user_query, progress=gr.Progress()):
540
  final_categorized_df = pd.DataFrame(categorized_tickets_db)
541
  if final_categorized_df.empty:
542
  error_message_markdown = "AI failed to categorize any tickets. Please ensure the ticket data is clear."
543
- return f"<p style='color:red;'>{error_message_markdown}</p>"
 
544
 
545
  # --- PHASE 2: Detailed Analysis per Category ---
 
546
  progress(0.5, desc=f"Phase 2/2: Detailing analysis for categories...")
547
 
548
  final_analysis_rows = []
549
  unique_categories = final_categorized_df['category'].unique()
 
550
 
551
  # Define a batch size for processing categories for detailed analysis if there are many categories
552
  # For 6-8 categories, this internal batching might not be strictly necessary, but good practice.
553
  category_batch_size = 5
554
 
555
  for i, category_chunk in enumerate(np.array_split(unique_categories, max(1, len(unique_categories) // category_batch_size))):
556
- progress((0.5 + (0.45 * i / len(np.array_split(unique_categories, max(1, len(unique_categories) // category_batch_size))))), desc=f"Phase 2/2: Detailing analysis for category batch {i+1}...")
 
 
557
 
558
  for category in category_chunk:
559
  category_specific_tickets_df = final_categorized_df[final_categorized_df['category'] == category]
@@ -563,6 +584,7 @@ def perform_ticket_analysis(file, user_query, progress=gr.Progress()):
563
  category_tickets_content = "\n\n".join(category_specific_tickets_df['original_content'].tolist())
564
 
565
  category_count = len(category_specific_tickets_df)
 
566
 
567
  detail_prompt = f"""
568
  You are an expert customer support ticket analyst.
@@ -585,12 +607,14 @@ def perform_ticket_analysis(file, user_query, progress=gr.Progress()):
585
  """
586
 
587
  try:
588
- # Add retry logic and error handling for API calls here
589
  detail_response = gemini_llm_model.generate_content(detail_prompt)
590
  detail_raw_output = detail_response.text
 
591
 
592
  detailed_df, parsing_err = parse_markdown_table_to_df(detail_raw_output)
593
  if parsing_err or detailed_df.empty:
 
594
  print(f"Warning: Parsing error or empty DataFrame for category '{category}': {parsing_err or 'Empty DataFrame'}")
595
  print(f"Raw AI output for detail: {detail_raw_output}")
596
  default_row = {
@@ -612,6 +636,7 @@ def perform_ticket_analysis(file, user_query, progress=gr.Progress()):
612
  try:
613
  ai_reported_count = pd.to_numeric(detailed_row['Count'], errors='coerce')
614
  if pd.isna(ai_reported_count) or int(ai_reported_count) != category_count:
 
615
  print(f"Warning: AI-reported count {ai_reported_count} for '{category}' does not match actual count {category_count}. Correcting.")
616
  detailed_row['Count'] = category_count
617
  except:
@@ -622,6 +647,7 @@ def perform_ticket_analysis(file, user_query, progress=gr.Progress()):
622
  final_analysis_rows.append(detailed_row)
623
 
624
  except Exception as e:
 
625
  print(f"Error detailing analysis for category '{category}': {e}")
626
  default_row = {
627
  "Category": category,
@@ -641,6 +667,7 @@ def perform_ticket_analysis(file, user_query, progress=gr.Progress()):
641
 
642
  summarized_count = analysis_df['Count'].sum()
643
  if summarized_count != total_tickets_count:
 
644
  print(f"CRITICAL WARNING: Total count in summarized DF ({summarized_count}) does not match original tickets ({total_tickets_count}).")
645
 
646
  global_analysis_df_for_export = analysis_df.copy()
@@ -648,26 +675,32 @@ def perform_ticket_analysis(file, user_query, progress=gr.Progress()):
648
 
649
  except ValueError as e:
650
  error_message_markdown = f"Ticket Analysis Error: {e}"
 
651
  global_raw_markdown_for_export = error_message_markdown + disclaimer_text_for_raw
652
  global_analysis_df_for_export = pd.DataFrame()
653
- return f"<p style='color:red;'>{error_message_markdown}</p>"
654
  except Exception as e:
655
  error_message_markdown = f"An unexpected error occurred during analysis: {e}. Check server logs for details."
 
656
  print(f"Error during RAG analysis: {e}")
657
  global_raw_markdown_for_export = error_message_markdown + disclaimer_text_for_raw
658
  global_analysis_df_for_export = pd.DataFrame()
659
- return f"<p style='color:red;'>{error_message_markdown}</p>"
660
 
 
661
  progress(0.98, desc="Generating summary report for display...")
662
 
663
  summarized_analysis_html = ""
664
  try:
665
  if not analysis_df.empty:
666
  summarized_analysis_html = generate_summary_html_by_category(analysis_df)
 
667
  else:
668
  summarized_analysis_html = "<p style='color:orange;'>AI output parsed into an empty DataFrame or no categories were generated. Please check the uploaded file and try again.</p>"
 
669
 
670
  except Exception as e:
 
671
  print(f"Failed to generate HTML summary from DataFrame: {e}")
672
  summarized_analysis_html = f"""
673
  <div style="margin-bottom: 20px;">
@@ -717,8 +750,9 @@ def perform_ticket_analysis(file, user_query, progress=gr.Progress()):
717
  </div>
718
  </div>
719
  """
 
720
  progress(1, desc="Analysis complete!")
721
- return html_output
722
 
723
 
724
  # --- Output Download Functions (using global_analysis_df_for_export) ---
@@ -1038,9 +1072,18 @@ with gr.Blocks(theme=gr.themes.Soft(), css="""
1038
  lines=2,
1039
  interactive=True
1040
  )
1041
-
1042
  submit_btn = gr.Button("Analyze Tickets", variant="primary")
1043
 
 
 
 
 
 
 
 
 
 
1044
  output_html_display = gr.HTML(
1045
  label="AI Ticket Analysis Result"
1046
  )
@@ -1059,10 +1102,11 @@ with gr.Blocks(theme=gr.themes.Soft(), css="""
1059
  </p>
1060
  """)
1061
 
 
1062
  submit_btn.click(
1063
  fn=perform_ticket_analysis,
1064
  inputs=[file_input, query_input],
1065
- outputs=output_html_display,
1066
  queue=True
1067
  )
1068
 
 
27
  gemini_llm_model = None
28
  gemini_embedding_model_name = "models/embedding-001"
29
 
30
+ google_api_key = os.getenv("GOOGLE_API_key")
31
 
32
  if not google_api_key:
33
  print("\n" + "="*80)
 
419
 
420
  start_time = time.time()
421
  error_message_markdown = ""
422
+ log_messages = [] # Initialize log messages list
423
 
424
+ def append_log_message(message):
425
+ timestamp = datetime.now().strftime("%H:%M:%S")
426
+ log_messages.append(f"[{timestamp}] {message}")
427
+
428
+ append_log_message("Starting analysis...")
429
  progress(0, desc="Starting analysis...")
430
 
431
  if not rag_initialized:
432
  error_message_markdown = "RAG system not initialized. Please ensure your GOOGLE_API_KEY is configured correctly and the specified AI model is accessible."
433
+ append_log_message(f"Error: {error_message_markdown}")
434
+ return f"<p style='color:red;'>{error_message_markdown}</p>", "\n".join(log_messages)
435
 
436
  if gemini_llm_model is None:
437
  error_message_markdown = "Google Generative AI LLM was not initialized. Please check startup logs."
438
+ append_log_message(f"Error: {error_message_markdown}")
439
+ return f"<p style='color:red;'>{error_message_markdown}</p>", "\n".join(log_messages)
440
 
441
  if file is None:
442
  error_message_markdown = "Please upload a customer ticket file."
443
+ append_log_message(f"Error: {error_message_markdown}")
444
+ return f"<p style='color:red;'>{error_message_markdown}</p>", "\n".join(log_messages)
445
 
446
  file_path = file.name
447
  filename = os.path.basename(file_path)
 
451
 
452
  try:
453
  # --- PHASE 1: Initial Categorization of ALL Tickets ---
454
+ append_log_message("Phase 1/2: Initializing categorization...")
455
  progress(0.1, desc="Phase 1/2: Initializing categorization...")
456
 
457
  # Use an iterator to handle potentially very large files
 
468
  total_tickets_count = current_total_processed
469
  # Adjust progress based on actual total, or a rough estimate if total_tickets_count is not yet final
470
  estimated_total_chunks = max(1, total_tickets_count // 5000 + 1) # Crude estimate
471
+ progress_desc = f"Phase 1/2: Categorizing tickets... {total_tickets_count} processed so far."
472
+ append_log_message(progress_desc)
473
+ progress(0.1 + (0.4 * (i+1) / estimated_total_chunks), desc=progress_desc)
474
 
475
  if not chunk_documents:
476
  continue
 
502
 
503
  try:
504
  # Add retry logic and error handling for API calls here
505
+ append_log_message(f"Calling LLM for categorization of {len(chunk_documents)} tickets...")
506
  categorization_response = gemini_llm_model.generate_content(categorization_prompt)
507
  categorization_raw_output = categorization_response.text
508
+ append_log_message("LLM categorization response received. Parsing...")
509
 
510
  # Parse categorization output
511
  for line in categorization_raw_output.split('\n'):
 
524
  })
525
  elif line:
526
  print(f"Warning: Line did not match categorization format in chunk {i}: {line}")
527
+ append_log_message(f"Warning: Malformed categorization line in chunk {i}: {line}")
528
  # Fallback parsing for malformed lines
529
  parts = line.split("Category:")
530
  ticket_id_part = parts[0].replace("Ticket ID:", "").strip() if len(parts) > 0 else f"Unparsed_Line_{len(categorized_tickets_db)+1}"
 
539
  })
540
 
541
  except Exception as e:
542
+ append_log_message(f"Error during categorization of chunk {i}: {e}")
543
  print(f"Error during categorization of chunk {i}: {e}")
544
  # For a real system, you'd log this, and potentially save the raw chunk for re-processing.
545
  # For this demo, we'll just skip the chunk or add a generic error entry.
 
556
  final_categorized_df = pd.DataFrame(categorized_tickets_db)
557
  if final_categorized_df.empty:
558
  error_message_markdown = "AI failed to categorize any tickets. Please ensure the ticket data is clear."
559
+ append_log_message(f"Error: {error_message_markdown}")
560
+ return f"<p style='color:red;'>{error_message_markdown}</p>", "\n".join(log_messages)
561
 
562
  # --- PHASE 2: Detailed Analysis per Category ---
563
+ append_log_message("Phase 2/2: Detailing analysis for categories...")
564
  progress(0.5, desc=f"Phase 2/2: Detailing analysis for categories...")
565
 
566
  final_analysis_rows = []
567
  unique_categories = final_categorized_df['category'].unique()
568
+ append_log_message(f"Identified {len(unique_categories)} unique categories for detailed analysis.")
569
 
570
  # Define a batch size for processing categories for detailed analysis if there are many categories
571
  # For 6-8 categories, this internal batching might not be strictly necessary, but good practice.
572
  category_batch_size = 5
573
 
574
  for i, category_chunk in enumerate(np.array_split(unique_categories, max(1, len(unique_categories) // category_batch_size))):
575
+ progress_desc = f"Phase 2/2: Detailing analysis for category batch {i+1}..."
576
+ append_log_message(progress_desc)
577
+ progress((0.5 + (0.45 * i / len(np.array_split(unique_categories, max(1, len(unique_categories) // category_batch_size))))), desc=progress_desc)
578
 
579
  for category in category_chunk:
580
  category_specific_tickets_df = final_categorized_df[final_categorized_df['category'] == category]
 
584
  category_tickets_content = "\n\n".join(category_specific_tickets_df['original_content'].tolist())
585
 
586
  category_count = len(category_specific_tickets_df)
587
+ append_log_message(f"Analyzing category: '{category}' with {category_count} tickets.")
588
 
589
  detail_prompt = f"""
590
  You are an expert customer support ticket analyst.
 
607
  """
608
 
609
  try:
610
+ append_log_message(f"Calling LLM for detailed analysis of category '{category}'...")
611
  detail_response = gemini_llm_model.generate_content(detail_prompt)
612
  detail_raw_output = detail_response.text
613
+ append_log_message(f"LLM detailed response for '{category}' received. Parsing...")
614
 
615
  detailed_df, parsing_err = parse_markdown_table_to_df(detail_raw_output)
616
  if parsing_err or detailed_df.empty:
617
+ append_log_message(f"Warning: Parsing error or empty DataFrame for category '{category}': {parsing_err or 'Empty DataFrame'}")
618
  print(f"Warning: Parsing error or empty DataFrame for category '{category}': {parsing_err or 'Empty DataFrame'}")
619
  print(f"Raw AI output for detail: {detail_raw_output}")
620
  default_row = {
 
636
  try:
637
  ai_reported_count = pd.to_numeric(detailed_row['Count'], errors='coerce')
638
  if pd.isna(ai_reported_count) or int(ai_reported_count) != category_count:
639
+ append_log_message(f"Warning: AI-reported count {ai_reported_count} for '{category}' does not match actual count {category_count}. Correcting.")
640
  print(f"Warning: AI-reported count {ai_reported_count} for '{category}' does not match actual count {category_count}. Correcting.")
641
  detailed_row['Count'] = category_count
642
  except:
 
647
  final_analysis_rows.append(detailed_row)
648
 
649
  except Exception as e:
650
+ append_log_message(f"Error detailing analysis for category '{category}': {e}")
651
  print(f"Error detailing analysis for category '{category}': {e}")
652
  default_row = {
653
  "Category": category,
 
667
 
668
  summarized_count = analysis_df['Count'].sum()
669
  if summarized_count != total_tickets_count:
670
+ append_log_message(f"CRITICAL WARNING: Total count in summarized DF ({summarized_count}) does not match original tickets ({total_tickets_count}).")
671
  print(f"CRITICAL WARNING: Total count in summarized DF ({summarized_count}) does not match original tickets ({total_tickets_count}).")
672
 
673
  global_analysis_df_for_export = analysis_df.copy()
 
675
 
676
  except ValueError as e:
677
  error_message_markdown = f"Ticket Analysis Error: {e}"
678
+ append_log_message(f"Error: {error_message_markdown}")
679
  global_raw_markdown_for_export = error_message_markdown + disclaimer_text_for_raw
680
  global_analysis_df_for_export = pd.DataFrame()
681
+ return f"<p style='color:red;'>{error_message_markdown}</p>", "\n".join(log_messages)
682
  except Exception as e:
683
  error_message_markdown = f"An unexpected error occurred during analysis: {e}. Check server logs for details."
684
+ append_log_message(f"Error: {error_message_markdown}")
685
  print(f"Error during RAG analysis: {e}")
686
  global_raw_markdown_for_export = error_message_markdown + disclaimer_text_for_raw
687
  global_analysis_df_for_export = pd.DataFrame()
688
+ return f"<p style='color:red;'>{error_message_markdown}</p>", "\n".join(log_messages)
689
 
690
+ append_log_message("Generating summary report for display...")
691
  progress(0.98, desc="Generating summary report for display...")
692
 
693
  summarized_analysis_html = ""
694
  try:
695
  if not analysis_df.empty:
696
  summarized_analysis_html = generate_summary_html_by_category(analysis_df)
697
+ append_log_message("HTML summary generated successfully.")
698
  else:
699
  summarized_analysis_html = "<p style='color:orange;'>AI output parsed into an empty DataFrame or no categories were generated. Please check the uploaded file and try again.</p>"
700
+ append_log_message("Warning: No categories generated, HTML summary might be empty.")
701
 
702
  except Exception as e:
703
+ append_log_message(f"Failed to generate HTML summary from DataFrame: {e}")
704
  print(f"Failed to generate HTML summary from DataFrame: {e}")
705
  summarized_analysis_html = f"""
706
  <div style="margin-bottom: 20px;">
 
750
  </div>
751
  </div>
752
  """
753
+ append_log_message("Analysis complete! Displaying results.")
754
  progress(1, desc="Analysis complete!")
755
+ return html_output, "\n".join(log_messages)
756
 
757
 
758
  # --- Output Download Functions (using global_analysis_df_for_export) ---
 
1072
  lines=2,
1073
  interactive=True
1074
  )
1075
+
1076
  submit_btn = gr.Button("Analyze Tickets", variant="primary")
1077
 
1078
+ # New Textbox for detailed processing log
1079
+ progress_log = gr.Textbox(
1080
+ label="Processing Log",
1081
+ interactive=False,
1082
+ lines=5, # Show 5 lines at a time
1083
+ autoscroll=True, # Automatically scroll to the latest message
1084
+ elem_id="progress-log-box" # Add an ID for potential CSS styling if needed
1085
+ )
1086
+
1087
  output_html_display = gr.HTML(
1088
  label="AI Ticket Analysis Result"
1089
  )
 
1102
  </p>
1103
  """)
1104
 
1105
+ # Modified submit_btn.click to update both HTML display and the new progress_log
1106
  submit_btn.click(
1107
  fn=perform_ticket_analysis,
1108
  inputs=[file_input, query_input],
1109
+ outputs=[output_html_display, progress_log], # Now returns two outputs
1110
  queue=True
1111
  )
1112