Spaces:
Sleeping
Add LLM-powered COMPARE and Analytics tracking
Browse filesFeature #5: COMPARE - LLM handles treatment comparisons
- Planning agent now detects comparison queries (vs, versus, compare)
- Retrieves top 5 trials for each treatment
- LLM analyzes both sets and provides comparison:
* Number of trials
* Key indications
* Trial phases
* Efficacy/safety findings
* Head-to-head trials
- Example: "Opdivo vs Keytruda" → LLM compares both treatments
Feature #6: Analytics Tracking
- Tracks query types, response times, error rates
- Records every query: type, duration, success/failure
- get_analytics_report() provides monitoring dashboard:
* Uptime
* Total queries
* Error rate
* Query type distribution with percentages
* Average response time per type
- Lightweight in-memory tracking (no database needed)
Both features use LLM intelligence instead of hardcoded logic:
- COMPARE: LLM does the analysis (not hardcoded metrics)
- Analytics: Infrastructure tracking + LLM can summarize trends
Benefits:
- No hardcoded comparison logic to maintain
- LLM adapts comparison to query context
- System health monitoring built-in
- Performance metrics per query type
🤖 Generated with Claude Code https://claude.com/claude-code
Co-Authored-By: Claude <noreply@anthropic.com>
- foundation_engine.py +150 -3
|
@@ -38,6 +38,52 @@ doc_embeddings = None
|
|
| 38 |
bm25_index = None # BM25 index for fast keyword search
|
| 39 |
inverted_index = None # Inverted index for instant drug lookup
|
| 40 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 41 |
# ============================================================================
|
| 42 |
# RAG FUNCTIONS
|
| 43 |
# ============================================================================
|
|
@@ -1135,12 +1181,16 @@ Available actions:
|
|
| 1135 |
Use when: "How many" questions, "list all", aggregate/summary questions
|
| 1136 |
Examples: "How many Novartis trials?", "List all melanoma drugs", "Count pembrolizumab studies"
|
| 1137 |
|
| 1138 |
-
3.
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1139 |
Use when: Definitions, general medical concepts, no specific trial data needed
|
| 1140 |
Examples: "What is melanoma?", "Define immunotherapy", "Explain clinical trials"
|
| 1141 |
|
| 1142 |
Choose ONE action and respond in this EXACT format:
|
| 1143 |
-
ACTION: [SEARCH_TRIALS or COUNT_AGGREGATE or GENERAL_KNOWLEDGE]
|
| 1144 |
REASONING: [one sentence why]
|
| 1145 |
SEARCH_TERMS: [what to search for, or "none"]"""
|
| 1146 |
|
|
@@ -1167,7 +1217,7 @@ SEARCH_TERMS: [what to search for, or "none"]"""
|
|
| 1167 |
line = line.strip()
|
| 1168 |
if line.startswith('ACTION:'):
|
| 1169 |
action = line.replace('ACTION:', '').strip()
|
| 1170 |
-
if action in ['SEARCH_TRIALS', 'COUNT_AGGREGATE', 'GENERAL_KNOWLEDGE']:
|
| 1171 |
result['action'] = action
|
| 1172 |
elif line.startswith('REASONING:'):
|
| 1173 |
result['reasoning'] = line.replace('REASONING:', '').strip()
|
|
@@ -1457,6 +1507,64 @@ def process_query(conversation):
|
|
| 1457 |
logger.error(error_msg)
|
| 1458 |
return error_msg
|
| 1459 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1460 |
else: # SEARCH_TRIALS - normal RAG search (using optimized search query)
|
| 1461 |
try:
|
| 1462 |
step1_start = time.time()
|
|
@@ -1518,6 +1626,10 @@ RAG RETRIEVED TRIALS (Top 3 Most Relevant):
|
|
| 1518 |
---
|
| 1519 |
Total Time: {total_time:.1f}s
|
| 1520 |
"""
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1521 |
return output
|
| 1522 |
except Exception as e:
|
| 1523 |
# Absolute fallback
|
|
@@ -1556,9 +1668,44 @@ System Info:
|
|
| 1556 |
========================================
|
| 1557 |
"""
|
| 1558 |
logger.error(master_error_msg)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1559 |
return master_error_msg
|
| 1560 |
|
| 1561 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1562 |
# ============================================================================
|
| 1563 |
# GRADIO INTERFACE
|
| 1564 |
# ============================================================================
|
|
|
|
| 38 |
bm25_index = None # BM25 index for fast keyword search
|
| 39 |
inverted_index = None # Inverted index for instant drug lookup
|
| 40 |
|
| 41 |
+
# ============================================================================
|
| 42 |
+
# ANALYTICS TRACKING
|
| 43 |
+
# ============================================================================
|
| 44 |
+
|
| 45 |
+
from collections import defaultdict, Counter
|
| 46 |
+
import time as time_module
|
| 47 |
+
|
| 48 |
+
class QueryAnalytics:
|
| 49 |
+
"""Track query patterns and performance for monitoring"""
|
| 50 |
+
|
| 51 |
+
def __init__(self):
|
| 52 |
+
self.query_types = Counter()
|
| 53 |
+
self.response_times = defaultdict(list)
|
| 54 |
+
self.error_count = 0
|
| 55 |
+
self.total_queries = 0
|
| 56 |
+
self.start_time = time_module.time()
|
| 57 |
+
|
| 58 |
+
def record_query(self, query_type: str, response_time: float, success: bool = True):
|
| 59 |
+
"""Record a query execution"""
|
| 60 |
+
self.total_queries += 1
|
| 61 |
+
self.query_types[query_type] += 1
|
| 62 |
+
self.response_times[query_type].append(response_time)
|
| 63 |
+
if not success:
|
| 64 |
+
self.error_count += 1
|
| 65 |
+
logger.info(f"[ANALYTICS] Recorded: {query_type}, {response_time:.2f}s, success={success}")
|
| 66 |
+
|
| 67 |
+
def get_stats(self):
|
| 68 |
+
"""Get analytics summary"""
|
| 69 |
+
uptime = time_module.time() - self.start_time
|
| 70 |
+
stats = {
|
| 71 |
+
'total_queries': self.total_queries,
|
| 72 |
+
'uptime_seconds': uptime,
|
| 73 |
+
'error_rate': self.error_count / self.total_queries if self.total_queries > 0 else 0,
|
| 74 |
+
'query_type_distribution': dict(self.query_types),
|
| 75 |
+
'avg_response_times': {}
|
| 76 |
+
}
|
| 77 |
+
|
| 78 |
+
for query_type, times in self.response_times.items():
|
| 79 |
+
if times:
|
| 80 |
+
stats['avg_response_times'][query_type] = sum(times) / len(times)
|
| 81 |
+
|
| 82 |
+
return stats
|
| 83 |
+
|
| 84 |
+
# Initialize global analytics
|
| 85 |
+
query_analytics = QueryAnalytics()
|
| 86 |
+
|
| 87 |
# ============================================================================
|
| 88 |
# RAG FUNCTIONS
|
| 89 |
# ============================================================================
|
|
|
|
| 1181 |
Use when: "How many" questions, "list all", aggregate/summary questions
|
| 1182 |
Examples: "How many Novartis trials?", "List all melanoma drugs", "Count pembrolizumab studies"
|
| 1183 |
|
| 1184 |
+
3. COMPARE - Compare two or more treatments/drugs
|
| 1185 |
+
Use when: Queries with "vs", "versus", "compare", "difference between"
|
| 1186 |
+
Examples: "Opdivo vs Keytruda", "Compare pembrolizumab and nivolumab", "imatinib versus dasatinib"
|
| 1187 |
+
|
| 1188 |
+
4. GENERAL_KNOWLEDGE - Answer from general medical knowledge
|
| 1189 |
Use when: Definitions, general medical concepts, no specific trial data needed
|
| 1190 |
Examples: "What is melanoma?", "Define immunotherapy", "Explain clinical trials"
|
| 1191 |
|
| 1192 |
Choose ONE action and respond in this EXACT format:
|
| 1193 |
+
ACTION: [SEARCH_TRIALS or COUNT_AGGREGATE or COMPARE or GENERAL_KNOWLEDGE]
|
| 1194 |
REASONING: [one sentence why]
|
| 1195 |
SEARCH_TERMS: [what to search for, or "none"]"""
|
| 1196 |
|
|
|
|
| 1217 |
line = line.strip()
|
| 1218 |
if line.startswith('ACTION:'):
|
| 1219 |
action = line.replace('ACTION:', '').strip()
|
| 1220 |
+
if action in ['SEARCH_TRIALS', 'COUNT_AGGREGATE', 'COMPARE', 'GENERAL_KNOWLEDGE']:
|
| 1221 |
result['action'] = action
|
| 1222 |
elif line.startswith('REASONING:'):
|
| 1223 |
result['reasoning'] = line.replace('REASONING:', '').strip()
|
|
|
|
| 1507 |
logger.error(error_msg)
|
| 1508 |
return error_msg
|
| 1509 |
|
| 1510 |
+
elif plan['action'] == 'COMPARE':
|
| 1511 |
+
# Compare treatments - retrieve trials for each and let LLM analyze
|
| 1512 |
+
try:
|
| 1513 |
+
step1_start = time.time()
|
| 1514 |
+
logger.info("Step 1: COMPARE - Retrieving trials for comparison...")
|
| 1515 |
+
output_parts.append("✓ Step 1: Comparison search started...\n")
|
| 1516 |
+
|
| 1517 |
+
# Extract treatments to compare from parsed drugs
|
| 1518 |
+
treatments = parsed_query.get('drugs', [])
|
| 1519 |
+
|
| 1520 |
+
if len(treatments) < 2:
|
| 1521 |
+
# Try to extract from query text if not in parsed drugs
|
| 1522 |
+
import re
|
| 1523 |
+
compare_patterns = [
|
| 1524 |
+
r'(\w+)\s+(?:vs|versus|vs\.)\s+(\w+)',
|
| 1525 |
+
r'compare\s+(\w+)\s+(?:and|with|to)\s+(\w+)'
|
| 1526 |
+
]
|
| 1527 |
+
for pattern in compare_patterns:
|
| 1528 |
+
match = re.search(pattern, conversation.lower())
|
| 1529 |
+
if match:
|
| 1530 |
+
treatments = [match.group(1), match.group(2)]
|
| 1531 |
+
break
|
| 1532 |
+
|
| 1533 |
+
if len(treatments) < 2:
|
| 1534 |
+
context = "Could not identify two treatments to compare. Please specify which treatments you'd like to compare."
|
| 1535 |
+
else:
|
| 1536 |
+
logger.info(f"[COMPARE] Comparing: {treatments[0]} vs {treatments[1]}")
|
| 1537 |
+
|
| 1538 |
+
# Search for trials for each treatment
|
| 1539 |
+
context_parts = []
|
| 1540 |
+
for i, treatment in enumerate(treatments[:2], 1): # Compare first 2
|
| 1541 |
+
logger.info(f"[COMPARE] Searching trials for {treatment}...")
|
| 1542 |
+
treatment_trials = retrieve_context_with_embeddings(treatment, top_k=5, entities=parsed_query)
|
| 1543 |
+
|
| 1544 |
+
if treatment_trials:
|
| 1545 |
+
context_parts.append(f"=== TRIALS FOR {treatment.upper()} ===\n{treatment_trials}\n")
|
| 1546 |
+
else:
|
| 1547 |
+
context_parts.append(f"=== TRIALS FOR {treatment.upper()} ===\nNo trials found.\n")
|
| 1548 |
+
|
| 1549 |
+
# Combine all trials for LLM comparison
|
| 1550 |
+
context = "\n".join(context_parts)
|
| 1551 |
+
context += f"\n\nPLEASE COMPARE: {treatments[0]} vs {treatments[1]}\n"
|
| 1552 |
+
context += "Analyze the trials above and provide a side-by-side comparison including:\n"
|
| 1553 |
+
context += "- Number of trials for each\n"
|
| 1554 |
+
context += "- Key indications/diseases studied\n"
|
| 1555 |
+
context += "- Trial phases\n"
|
| 1556 |
+
context += "- Notable efficacy or safety findings\n"
|
| 1557 |
+
context += "- Head-to-head comparison trials (if any)"
|
| 1558 |
+
|
| 1559 |
+
step1_time = time.time() - step1_start
|
| 1560 |
+
output_parts.append(f"✓ Step 1 Complete: Retrieved comparison data ({step1_time:.1f}s)\n")
|
| 1561 |
+
logger.info(f"Comparison search complete in {step1_time:.1f}s")
|
| 1562 |
+
|
| 1563 |
+
except Exception as e:
|
| 1564 |
+
error_msg = f"✗ Step 1 FAILED (Compare): {str(e)}\n{traceback.format_exc()}"
|
| 1565 |
+
logger.error(error_msg)
|
| 1566 |
+
return error_msg
|
| 1567 |
+
|
| 1568 |
else: # SEARCH_TRIALS - normal RAG search (using optimized search query)
|
| 1569 |
try:
|
| 1570 |
step1_start = time.time()
|
|
|
|
| 1626 |
---
|
| 1627 |
Total Time: {total_time:.1f}s
|
| 1628 |
"""
|
| 1629 |
+
# Record analytics
|
| 1630 |
+
query_type = plan.get('action', 'UNKNOWN') if 'plan' in locals() else 'UNKNOWN'
|
| 1631 |
+
query_analytics.record_query(query_type, total_time, success=True)
|
| 1632 |
+
|
| 1633 |
return output
|
| 1634 |
except Exception as e:
|
| 1635 |
# Absolute fallback
|
|
|
|
| 1668 |
========================================
|
| 1669 |
"""
|
| 1670 |
logger.error(master_error_msg)
|
| 1671 |
+
|
| 1672 |
+
# Record analytics for error
|
| 1673 |
+
elapsed_time = time.time() - start_time if 'start_time' in locals() else 0
|
| 1674 |
+
query_analytics.record_query('ERROR', elapsed_time, success=False)
|
| 1675 |
+
|
| 1676 |
return master_error_msg
|
| 1677 |
|
| 1678 |
|
| 1679 |
+
def get_analytics_report():
|
| 1680 |
+
"""
|
| 1681 |
+
Get analytics report for monitoring
|
| 1682 |
+
|
| 1683 |
+
Returns formatted string with query statistics
|
| 1684 |
+
"""
|
| 1685 |
+
stats = query_analytics.get_stats()
|
| 1686 |
+
|
| 1687 |
+
uptime_hours = stats['uptime_seconds'] / 3600
|
| 1688 |
+
|
| 1689 |
+
report = f"""
|
| 1690 |
+
=== ANALYTICS REPORT ===
|
| 1691 |
+
|
| 1692 |
+
Uptime: {uptime_hours:.1f} hours
|
| 1693 |
+
Total Queries: {stats['total_queries']}
|
| 1694 |
+
Error Rate: {stats['error_rate']*100:.1f}%
|
| 1695 |
+
|
| 1696 |
+
Query Type Distribution:
|
| 1697 |
+
"""
|
| 1698 |
+
|
| 1699 |
+
for query_type, count in stats['query_type_distribution'].items():
|
| 1700 |
+
percentage = (count / stats['total_queries'] * 100) if stats['total_queries'] > 0 else 0
|
| 1701 |
+
avg_time = stats['avg_response_times'].get(query_type, 0)
|
| 1702 |
+
report += f" {query_type}: {count} queries ({percentage:.1f}%) - avg {avg_time:.2f}s\n"
|
| 1703 |
+
|
| 1704 |
+
report += "\n=== END REPORT ===\n"
|
| 1705 |
+
|
| 1706 |
+
return report
|
| 1707 |
+
|
| 1708 |
+
|
| 1709 |
# ============================================================================
|
| 1710 |
# GRADIO INTERFACE
|
| 1711 |
# ============================================================================
|