Buckets:
AnonymousSpottedToad/candidate_bucket / codebase /python /ViperJuice_Code-Index-MCP /examples /demo_mcp_comparison.py
| #!/usr/bin/env python3 | |
| """ | |
| Demo script showing MCP vs Direct Search comparison with token usage and performance metrics. | |
| This script demonstrates: | |
| 1. Token usage differences between MCP and direct search | |
| 2. Performance comparisons for different query types | |
| 3. Cost analysis across different LLM models | |
| 4. Visual comparison charts | |
| Usage: | |
| python demo_mcp_comparison.py [--workspace PATH] [--quick] | |
| """ | |
| import sys | |
| import os | |
| import time | |
| import json | |
| import argparse | |
| from pathlib import Path | |
| from typing import Dict, List, Any, Tuple | |
| # Add parent directory to path for imports | |
| sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) | |
| from mcp_server.utils.token_counter import TokenCounter, quick_estimate, compare_model_costs | |
| from mcp_server.utils.direct_searcher import DirectSearcher, quick_search | |
| from mcp_server.visualization.quick_charts import QuickCharts | |
| from tests.comparison.test_data import TestDataGenerator, QueryType | |
| class MCPComparisonDemo: | |
| """Demo class for comparing MCP vs direct search approaches.""" | |
| def __init__(self, workspace_path: str = "."): | |
| self.workspace_path = Path(workspace_path) | |
| self.token_counter = TokenCounter() | |
| self.direct_searcher = DirectSearcher() | |
| self.charts = QuickCharts(figsize=(12, 6)) | |
| # Results storage | |
| self.comparison_results = [] | |
| def run_symbol_search_comparison(self, symbol: str = "main") -> Dict[str, Any]: | |
| """Compare symbol search between MCP and direct search.""" | |
| print(f"\n=== Symbol Search Comparison: '{symbol}' ===") | |
| # Direct search using ripgrep | |
| direct_pattern = f"\\b{symbol}\\b" | |
| print(f"Running ripgrep search for pattern: {direct_pattern}") | |
| direct_result = self.direct_searcher.search_pattern( | |
| direct_pattern, | |
| str(self.workspace_path), | |
| use_ripgrep=True | |
| ) | |
| # Simulate MCP search (in a real scenario, this would use the MCP client) | |
| print("Simulating MCP symbol search...") | |
| mcp_start = time.time() | |
| # MCP would return structured results - simulate this | |
| mcp_query = f"symbol:{symbol}" | |
| self.token_counter.add_input_tokens(mcp_query, model="claude-3-sonnet") | |
| # Simulate MCP response (structured JSON) | |
| mcp_response = { | |
| "results": [ | |
| { | |
| "symbol": symbol, | |
| "file": f"src/{symbol}.py", | |
| "line": 10, | |
| "kind": "function", | |
| "signature": f"def {symbol}():" | |
| } | |
| for _ in range(min(len(direct_result.get("results", [])), 10)) | |
| ], | |
| "total_matches": len(direct_result.get("results", [])), | |
| "search_time_ms": 25 | |
| } | |
| mcp_response_text = json.dumps(mcp_response, indent=2) | |
| self.token_counter.add_output_tokens(mcp_response_text, model="claude-3-sonnet") | |
| mcp_elapsed = time.time() - mcp_start | |
| # Compare results | |
| comparison = { | |
| "query": symbol, | |
| "query_type": "symbol_search", | |
| "mcp": { | |
| "matches": len(mcp_response["results"]), | |
| "elapsed_time": mcp_elapsed, | |
| "tokens": { | |
| "input": self.token_counter.input_tokens, | |
| "output": self.token_counter.output_tokens, | |
| "total": self.token_counter.total_tokens | |
| } | |
| }, | |
| "direct": { | |
| "matches": direct_result["match_count"], | |
| "elapsed_time": direct_result["elapsed_time"], | |
| "tool": direct_result["tool"] | |
| } | |
| } | |
| self._print_comparison(comparison) | |
| self.comparison_results.append(comparison) | |
| return comparison | |
| def run_pattern_search_comparison(self, pattern: str = "TODO|FIXME") -> Dict[str, Any]: | |
| """Compare pattern search between approaches.""" | |
| print(f"\n=== Pattern Search Comparison: '{pattern}' ===") | |
| # Reset token counter for this comparison | |
| self.token_counter.reset() | |
| # Direct search | |
| print(f"Running direct search for pattern: {pattern}") | |
| direct_result = self.direct_searcher.search_pattern( | |
| pattern, | |
| str(self.workspace_path), | |
| use_ripgrep=True | |
| ) | |
| # Simulate MCP pattern search | |
| print("Simulating MCP pattern search...") | |
| mcp_start = time.time() | |
| mcp_query = f"pattern:{pattern}" | |
| self.token_counter.add_input_tokens(mcp_query, model="claude-3-sonnet") | |
| # MCP would return matches with context | |
| mcp_response = { | |
| "results": [ | |
| { | |
| "file": match["file"], | |
| "line": match["line"], | |
| "content": match["content"], | |
| "context": { | |
| "before": "...", | |
| "after": "..." | |
| } | |
| } | |
| for match in direct_result.get("results", [])[:20] | |
| ], | |
| "total_matches": direct_result["match_count"], | |
| "pattern": pattern | |
| } | |
| mcp_response_text = json.dumps(mcp_response, indent=2) | |
| self.token_counter.add_output_tokens(mcp_response_text, model="claude-3-sonnet") | |
| mcp_elapsed = time.time() - mcp_start | |
| # Compare results | |
| comparison = { | |
| "query": pattern, | |
| "query_type": "pattern_search", | |
| "mcp": { | |
| "matches": len(mcp_response["results"]), | |
| "elapsed_time": mcp_elapsed, | |
| "tokens": { | |
| "input": self.token_counter.input_tokens, | |
| "output": self.token_counter.output_tokens, | |
| "total": self.token_counter.total_tokens | |
| } | |
| }, | |
| "direct": { | |
| "matches": direct_result["match_count"], | |
| "elapsed_time": direct_result["elapsed_time"], | |
| "tool": direct_result["tool"] | |
| } | |
| } | |
| self._print_comparison(comparison) | |
| self.comparison_results.append(comparison) | |
| return comparison | |
| def demonstrate_token_counting(self): | |
| """Show token counting across different models.""" | |
| print("\n=== Token Counting Demonstration ===") | |
| sample_query = "Find all Python classes that inherit from BaseModel" | |
| sample_response = """Found 5 classes: | |
| 1. UserModel in models/user.py:15 | |
| 2. ProductModel in models/product.py:8 | |
| 3. OrderModel in models/order.py:22 | |
| 4. ConfigModel in core/config.py:10 | |
| 5. ValidationModel in utils/validation.py:45""" | |
| print(f"Query: {sample_query}") | |
| print(f"Response preview: {sample_response[:100]}...") | |
| # Compare costs across models | |
| print("\nToken counts and costs for this query:") | |
| costs = compare_model_costs(sample_query + sample_response, is_output=True) | |
| for model, (tokens, cost) in costs.items(): | |
| print(f" {model}: {tokens} tokens, ${cost:.4f}") | |
| def generate_comparison_charts(self, output_dir: str = "comparison_charts"): | |
| """Generate visualization charts from comparison results.""" | |
| print(f"\n=== Generating Comparison Charts ===") | |
| os.makedirs(output_dir, exist_ok=True) | |
| # Token usage comparison | |
| token_data = {} | |
| for result in self.comparison_results: | |
| query_type = result["query_type"] | |
| token_data[query_type] = result["mcp"]["tokens"]["total"] | |
| if token_data: | |
| self.charts.token_usage_comparison( | |
| token_data, | |
| title="Token Usage by Query Type", | |
| output_path=f"{output_dir}/token_usage.png" | |
| ) | |
| print(f"Saved token usage chart to {output_dir}/token_usage.png") | |
| # Latency comparison | |
| latency_data = { | |
| "MCP": sum(r["mcp"]["elapsed_time"] for r in self.comparison_results) * 1000, | |
| "Direct Search": sum(r["direct"]["elapsed_time"] for r in self.comparison_results) * 1000 | |
| } | |
| self.charts.latency_comparison( | |
| latency_data, | |
| title="Total Search Latency Comparison", | |
| unit="ms", | |
| output_path=f"{output_dir}/latency_comparison.png" | |
| ) | |
| print(f"Saved latency chart to {output_dir}/latency_comparison.png") | |
| # Multi-metric comparison | |
| if self.comparison_results: | |
| metrics = { | |
| "Queries": {}, | |
| "Matches": {}, | |
| "Time (ms)": {} | |
| } | |
| for result in self.comparison_results: | |
| query = result["query"][:20] + "..." if len(result["query"]) > 20 else result["query"] | |
| metrics["Queries"][query] = 1 | |
| metrics["Matches"][query] = result["mcp"]["matches"] | |
| metrics["Time (ms)"][query] = result["mcp"]["elapsed_time"] * 1000 | |
| self.charts.multi_metric_comparison( | |
| metrics, | |
| "MCP Performance Metrics", | |
| output_path=f"{output_dir}/multi_metrics.png" | |
| ) | |
| print(f"Saved multi-metric chart to {output_dir}/multi_metrics.png") | |
| def generate_report(self) -> Dict[str, Any]: | |
| """Generate final comparison report.""" | |
| total_mcp_tokens = sum(r["mcp"]["tokens"]["total"] for r in self.comparison_results) | |
| total_mcp_time = sum(r["mcp"]["elapsed_time"] for r in self.comparison_results) | |
| total_direct_time = sum(r["direct"]["elapsed_time"] for r in self.comparison_results) | |
| # Calculate costs for different models | |
| model_costs = {} | |
| for model in ["gpt-4", "claude-3-sonnet", "gpt-3.5-turbo"]: | |
| counter = TokenCounter() | |
| for result in self.comparison_results: | |
| # Rough estimation based on our token counts | |
| counter.input_tokens = result["mcp"]["tokens"]["input"] | |
| counter.output_tokens = result["mcp"]["tokens"]["output"] | |
| cost = counter.estimate_cost(model=model) | |
| model_costs[model] = model_costs.get(model, 0) + cost | |
| report = { | |
| "summary": { | |
| "total_comparisons": len(self.comparison_results), | |
| "total_mcp_tokens": total_mcp_tokens, | |
| "total_mcp_time_seconds": total_mcp_time, | |
| "total_direct_time_seconds": total_direct_time, | |
| "speedup_factor": total_direct_time / total_mcp_time if total_mcp_time > 0 else 0 | |
| }, | |
| "cost_analysis": model_costs, | |
| "recommendations": self._generate_recommendations(), | |
| "detailed_results": self.comparison_results | |
| } | |
| return report | |
| def _print_comparison(self, comparison: Dict[str, Any]): | |
| """Print formatted comparison results.""" | |
| print(f"\nResults for '{comparison['query']}':") | |
| print(f" MCP: {comparison['mcp']['matches']} matches in {comparison['mcp']['elapsed_time']:.3f}s") | |
| print(f" Tokens: {comparison['mcp']['tokens']['total']} " | |
| f"(input: {comparison['mcp']['tokens']['input']}, " | |
| f"output: {comparison['mcp']['tokens']['output']})") | |
| print(f" Direct ({comparison['direct']['tool']}): " | |
| f"{comparison['direct']['matches']} matches in {comparison['direct']['elapsed_time']:.3f}s") | |
| def _generate_recommendations(self) -> List[str]: | |
| """Generate recommendations based on comparison results.""" | |
| recommendations = [] | |
| if not self.comparison_results: | |
| return ["No comparison data available"] | |
| avg_speedup = sum( | |
| r["direct"]["elapsed_time"] / r["mcp"]["elapsed_time"] | |
| for r in self.comparison_results if r["mcp"]["elapsed_time"] > 0 | |
| ) / len(self.comparison_results) | |
| total_tokens = sum(r["mcp"]["tokens"]["total"] for r in self.comparison_results) | |
| if avg_speedup < 1: | |
| recommendations.append( | |
| "MCP is faster on average for these queries - benefits from pre-indexing" | |
| ) | |
| else: | |
| recommendations.append( | |
| f"Direct search is {avg_speedup:.1f}x faster for these simple queries" | |
| ) | |
| if total_tokens > 1000: | |
| recommendations.append( | |
| f"High token usage ({total_tokens} tokens) - consider direct search for cost savings" | |
| ) | |
| else: | |
| recommendations.append( | |
| f"Moderate token usage ({total_tokens} tokens) - acceptable for enhanced features" | |
| ) | |
| recommendations.extend([ | |
| "Use MCP for: semantic search, cross-file references, type information", | |
| "Use direct search for: simple pattern matching, one-off searches, small codebases" | |
| ]) | |
| return recommendations | |
| def main(): | |
| parser = argparse.ArgumentParser( | |
| description="Demo MCP vs Direct Search comparison" | |
| ) | |
| parser.add_argument( | |
| "--workspace", | |
| default=".", | |
| help="Workspace path to search in" | |
| ) | |
| parser.add_argument( | |
| "--quick", | |
| action="store_true", | |
| help="Run quick demo with fewer comparisons" | |
| ) | |
| args = parser.parse_args() | |
| # Create demo instance | |
| demo = MCPComparisonDemo(args.workspace) | |
| print("=== MCP vs Direct Search Comparison Demo ===") | |
| print(f"Workspace: {os.path.abspath(args.workspace)}") | |
| # Demonstrate token counting | |
| demo.demonstrate_token_counting() | |
| # Run comparisons | |
| if args.quick: | |
| # Quick demo - just two comparisons | |
| demo.run_symbol_search_comparison("main") | |
| demo.run_pattern_search_comparison("TODO") | |
| else: | |
| # Full demo | |
| symbols = ["main", "test", "init", "config"] | |
| patterns = ["TODO|FIXME", "import.*os", "class.*:"] | |
| for symbol in symbols: | |
| demo.run_symbol_search_comparison(symbol) | |
| for pattern in patterns: | |
| demo.run_pattern_search_comparison(pattern) | |
| # Generate charts | |
| demo.generate_comparison_charts() | |
| # Generate and print report | |
| report = demo.generate_report() | |
| print("\n=== FINAL REPORT ===") | |
| print(f"Total comparisons: {report['summary']['total_comparisons']}") | |
| print(f"Total MCP tokens used: {report['summary']['total_mcp_tokens']}") | |
| print(f"Average speedup factor: {report['summary']['speedup_factor']:.2f}x") | |
| print("\nEstimated costs per search:") | |
| for model, cost in report['cost_analysis'].items(): | |
| print(f" {model}: ${cost:.4f}") | |
| print("\nRecommendations:") | |
| for rec in report['recommendations']: | |
| print(f" • {rec}") | |
| # Save detailed report | |
| with open("comparison_report.json", "w") as f: | |
| json.dump(report, f, indent=2) | |
| print("\nDetailed report saved to comparison_report.json") | |
| if __name__ == "__main__": | |
| main() |
Xet Storage Details
- Size:
- 15.4 kB
- Xet hash:
- 69c9ee778b3c210de1922596c49be61fd173977342ed628e23663ddcd7c65865
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.