Joachim M. Vis commited on
Commit
7fd2b63
·
verified ·
1 Parent(s): fadc0bc

Upload 3 files

Browse files
examples/enhanced_web_search_example.py ADDED
@@ -0,0 +1,84 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Example script demonstrating how to use the EnhancedWebSearchTool for GAIA assessment questions.
3
+
4
+ This script shows how to use the EnhancedWebSearchTool to search for information
5
+ related to specific GAIA assessment questions.
6
+ """
7
+
8
+ import os
9
+ import sys
10
+ import logging
11
+ import json
12
+ from typing import Dict, Any, List
13
+
14
+ # Add the parent directory to the path so we can import the agent module
15
+ sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
16
+
17
+ from agent.tool_registry import create_default_registry, unified_search
18
+ from tools.web_tools import create_enhanced_web_search
19
+
20
+ # Configure logging
21
+ logging.basicConfig(
22
+ level=logging.INFO,
23
+ format="%(asctime)s - %(name)s - %(levelname)s - %(message)s"
24
+ )
25
+ logger = logging.getLogger("enhanced_web_search_example")
26
+
27
+ def main():
28
+ """Run the enhanced web search example."""
29
+
30
+ # Check required environment variables
31
+ required_vars = ["PERPLEXITY_API_KEY"]
32
+ missing_vars = [var for var in required_vars if not os.environ.get(var)]
33
+
34
+ if missing_vars:
35
+ logger.error(f"Missing required environment variables: {', '.join(missing_vars)}")
36
+ logger.error("Please set these environment variables before running the example.")
37
+ sys.exit(1)
38
+
39
+ # Create the registry
40
+ logger.info("Creating tool registry...")
41
+ registry = create_default_registry()
42
+
43
+ # GAIA assessment questions
44
+ questions = [
45
+ "How many studio albums were published by Mercedes Sosa between 2000 and 2009 (included)?",
46
+ "Who nominated the only Featured Article on English Wikipedia about a dinosaur that was promoted in November 2016?",
47
+ "On June 6, 2023, an article by Carolyn Collins Petersen was published in Universe Today. This article mentions a team that produced a paper about their observations, linked at the bottom of the article. Find this paper. Under what NASA award number was the work performed by R. G. Arendt supported by?",
48
+ "Who won the Malko Competition for Young Conductors in 2021?"
49
+ ]
50
+
51
+ # Search for each question
52
+ for i, question in enumerate(questions):
53
+ logger.info(f"\n\nQuestion {i+1}: {question}")
54
+
55
+ # Search using unified_search
56
+ logger.info("Searching using unified_search...")
57
+ results = unified_search(registry, question)
58
+
59
+ # Print the results
60
+ if results and results.get("results"):
61
+ logger.info(f"Found {len(results['results'])} results")
62
+ logger.info(f"Providers used: {results.get('metadata', {}).get('providers_used', [])}")
63
+
64
+ # Print the top result
65
+ if results["results"]:
66
+ top_result = results["results"][0]
67
+ logger.info("\nTop result:")
68
+ logger.info(f"Title: {top_result.get('title', 'N/A')}")
69
+ logger.info(f"Source: {top_result.get('source', 'N/A')}")
70
+ logger.info(f"Relevance score: {top_result.get('relevance_score', 'N/A')}")
71
+ logger.info(f"Snippet: {top_result.get('snippet', 'N/A')}")
72
+
73
+ # If we have Perplexity content, print it
74
+ perplexity_content = results.get("metadata", {}).get("perplexity_content")
75
+ if perplexity_content:
76
+ logger.info("\nPerplexity content:")
77
+ logger.info(perplexity_content[:500] + "..." if len(perplexity_content) > 500 else perplexity_content)
78
+ else:
79
+ logger.warning("No results found")
80
+
81
+ logger.info("\nExample completed successfully!")
82
+
83
+ if __name__ == "__main__":
84
+ main()
examples/unified_search_example.py ADDED
@@ -0,0 +1,373 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env python
2
+ """
3
+ Unified Search Example
4
+
5
+ This script demonstrates how to use all three search tools:
6
+ 1. BrowserSearchTool for direct browser-based searches
7
+ 2. LibrarySearchTool for searches using imported libraries
8
+ 3. ApiSearchTool for searches using external API services
9
+
10
+ The example includes:
11
+ - Importing and initializing each tool
12
+ - Example queries for each tool type (academic, complex reasoning, simple factual)
13
+ - Demonstrating how to use them individually and together
14
+ - Showing how to interpret and process the results
15
+ """
16
+
17
+ import os
18
+ import json
19
+ import logging
20
+ from typing import Dict, Any, List, Optional
21
+
22
+ # Configure logging
23
+ logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s')
24
+ logger = logging.getLogger("unified_search_example")
25
+
26
+ # Import the tool registry and search tools
27
+ from agent.tool_registry import ToolRegistry, create_default_registry
28
+ from tools.web_tools import (
29
+ LibrarySearchTool,
30
+ ApiSearchTool,
31
+ create_library_search,
32
+ create_api_search
33
+ )
34
+ from tools.multimodal_tools import BrowserSearchTool, create_browser_search_tool
35
+
36
+ def print_section(title: str) -> None:
37
+ """Print a section title with formatting."""
38
+ print("\n" + "=" * 80)
39
+ print(f" {title} ".center(80, "="))
40
+ print("=" * 80 + "\n")
41
+
42
+ def print_results(results: List[Dict[str, Any]], max_snippet_length: int = 150) -> None:
43
+ """
44
+ Print search results in a readable format.
45
+
46
+ Args:
47
+ results: List of search result dictionaries
48
+ max_snippet_length: Maximum length for snippet display
49
+ """
50
+ if not results:
51
+ print("No results found.")
52
+ return
53
+
54
+ for i, result in enumerate(results):
55
+ print(f"Result {i+1}:")
56
+ print(f" Title: {result.get('title', 'No title')}")
57
+ print(f" Link: {result.get('link', 'No link')}")
58
+
59
+ # Format snippet for display
60
+ snippet = result.get('snippet', 'No snippet')
61
+ if len(snippet) > max_snippet_length:
62
+ snippet = snippet[:max_snippet_length] + "..."
63
+ print(f" Snippet: {snippet}")
64
+
65
+ # Print source and relevance if available
66
+ if 'source' in result:
67
+ print(f" Source: {result['source']}")
68
+ if 'relevance_score' in result:
69
+ print(f" Relevance: {result['relevance_score']}")
70
+
71
+ print()
72
+
73
+ def example_browser_search() -> None:
74
+ """
75
+ Demonstrate the BrowserSearchTool for direct browser-based searches.
76
+
77
+ This tool is useful for:
78
+ - Visual content that needs to be seen directly
79
+ - Interactive exploration of search results
80
+ - Searching specific websites with their native interfaces
81
+ """
82
+ print_section("BrowserSearchTool Example")
83
+
84
+ # Initialize the BrowserSearchTool
85
+ browser_tool = create_browser_search_tool()
86
+ print("BrowserSearchTool initialized")
87
+
88
+ # Example 1: General web search
89
+ print("\n1. General web search (Google):")
90
+ query = "latest advancements in quantum computing"
91
+ results = browser_tool.search(query, site="google")
92
+ print_results(results)
93
+ print("To use these results, you would use the browser_action tool to open the URL and interact with the search results directly.")
94
+
95
+ # Example 2: Academic search on arXiv
96
+ print("\n2. Academic search on arXiv:")
97
+ academic_query = "transformer models in natural language processing"
98
+ results = browser_tool.search(academic_query, site="arxiv")
99
+ print_results(results)
100
+ print("The browser_action tool would allow you to view papers, download PDFs, and explore citations.")
101
+
102
+ # Example 3: Wikipedia search for factual information
103
+ print("\n3. Wikipedia search for factual information:")
104
+ factual_query = "history of artificial intelligence"
105
+ results = browser_tool.search(factual_query, site="wikipedia")
106
+ print_results(results)
107
+ print("Using browser_action would let you navigate through Wikipedia articles and follow links to related topics.")
108
+
109
+ # Example 4: Direct URL visit
110
+ print("\n4. Direct URL visit:")
111
+ url = "https://arxiv.org/abs/2303.08774" # Example arXiv paper
112
+ result = browser_tool.direct_visit(url)
113
+ print(f"Title: {result.get('title')}")
114
+ print(f"Link: {result.get('link')}")
115
+ print(f"Instructions: {result.get('instructions')}")
116
+
117
+ def example_library_search() -> None:
118
+ """
119
+ Demonstrate the LibrarySearchTool for searches using imported libraries.
120
+
121
+ This tool is useful for:
122
+ - Programmatic searches without requiring a browser
123
+ - Automatically routing between general and academic sources
124
+ - Getting structured results that can be processed programmatically
125
+ """
126
+ print_section("LibrarySearchTool Example")
127
+
128
+ # Initialize the LibrarySearchTool
129
+ library_tool = create_library_search()
130
+ print("LibrarySearchTool initialized")
131
+
132
+ # Example 1: General factual query (routes to DuckDuckGo)
133
+ print("\n1. General factual query (routes to DuckDuckGo):")
134
+ factual_query = "population of Tokyo in 2023"
135
+ print(f"Query: '{factual_query}'")
136
+ print("Analyzing query type...")
137
+ is_academic = library_tool._is_academic_query(factual_query)
138
+ print(f"Is academic query? {is_academic}")
139
+ print("Routing to appropriate search provider...")
140
+ results = library_tool.search(factual_query)
141
+ print_results(results)
142
+
143
+ # Example 2: Academic query (routes to arXiv)
144
+ print("\n2. Academic query (routes to arXiv):")
145
+ academic_query = "recent papers on large language models"
146
+ print(f"Query: '{academic_query}'")
147
+ print("Analyzing query type...")
148
+ is_academic = library_tool._is_academic_query(academic_query)
149
+ print(f"Is academic query? {is_academic}")
150
+ print("Routing to appropriate search provider...")
151
+ results = library_tool.search(academic_query)
152
+ print_results(results)
153
+
154
+ # Example 3: Processing and analyzing results
155
+ print("\n3. Processing and analyzing results:")
156
+ query = "climate change impacts"
157
+ results = library_tool.search(query)
158
+
159
+ # Demonstrate how to extract and use specific information
160
+ if results:
161
+ # Count sources
162
+ sources = {}
163
+ for result in results:
164
+ source = result.get('source', 'unknown')
165
+ sources[source] = sources.get(source, 0) + 1
166
+
167
+ print(f"Query: '{query}'")
168
+ print(f"Total results: {len(results)}")
169
+ print(f"Results by source: {sources}")
170
+
171
+ # Find highest relevance result
172
+ if any('relevance_score' in r for r in results):
173
+ most_relevant = max(results, key=lambda r: r.get('relevance_score', 0))
174
+ print(f"Most relevant result: {most_relevant.get('title')} (Score: {most_relevant.get('relevance_score')})")
175
+
176
+ # Extract all unique domains from results
177
+ domains = set()
178
+ for result in results:
179
+ link = result.get('link', '')
180
+ if link.startswith('http'):
181
+ from urllib.parse import urlparse
182
+ domain = urlparse(link).netloc
183
+ domains.add(domain)
184
+
185
+ print(f"Unique domains in results: {domains}")
186
+
187
+ def example_api_search() -> None:
188
+ """
189
+ Demonstrate the ApiSearchTool for searches using external API services.
190
+
191
+ This tool is useful for:
192
+ - Complex reasoning queries that benefit from AI-powered search
193
+ - Getting high-quality, synthesized results for complex topics
194
+ - Accessing real-time information from search engines
195
+
196
+ Note: Requires API keys for Perplexity and/or Serper
197
+ """
198
+ print_section("ApiSearchTool Example")
199
+
200
+ # Check if API keys are available
201
+ perplexity_key = os.environ.get("PERPLEXITY_API_KEY")
202
+ serper_key = os.environ.get("SERPER_API_KEY")
203
+
204
+ if not perplexity_key and not serper_key:
205
+ print("Warning: No API keys found for Perplexity or Serper.")
206
+ print("Set PERPLEXITY_API_KEY and/or SERPER_API_KEY environment variables to use ApiSearchTool.")
207
+ print("Continuing with limited functionality...\n")
208
+
209
+ # Initialize the ApiSearchTool
210
+ api_tool = create_api_search()
211
+ print("ApiSearchTool initialized")
212
+
213
+ # Example 1: Complex reasoning query (routes to Perplexity if available)
214
+ print("\n1. Complex reasoning query (routes to Perplexity if available):")
215
+ complex_query = "Explain the implications of quantum computing on modern cryptography"
216
+ print(f"Query: '{complex_query}'")
217
+ print("Analyzing query complexity...")
218
+ is_complex = api_tool._is_complex_query(complex_query)
219
+ print(f"Is complex query? {is_complex}")
220
+ print("Routing to appropriate API service...")
221
+
222
+ try:
223
+ results = api_tool.search(complex_query)
224
+ print_results(results)
225
+
226
+ # Show how to access full content if available
227
+ if results and 'full_content' in results[0]:
228
+ print("Full content is available for this result.")
229
+ print("First 200 characters of full content:")
230
+ print(results[0]['full_content'][:200] + "...")
231
+ except Exception as e:
232
+ print(f"Error: {str(e)}")
233
+ print("This may be due to missing API keys or API service issues.")
234
+
235
+ # Example 2: Simple factual query (routes to Serper if available)
236
+ print("\n2. Simple factual query (routes to Serper if available):")
237
+ factual_query = "height of Mount Everest"
238
+ print(f"Query: '{factual_query}'")
239
+ print("Analyzing query complexity...")
240
+ is_complex = api_tool._is_complex_query(factual_query)
241
+ print(f"Is complex query? {is_complex}")
242
+ print("Routing to appropriate API service...")
243
+
244
+ try:
245
+ results = api_tool.search(factual_query)
246
+ print_results(results)
247
+ except Exception as e:
248
+ print(f"Error: {str(e)}")
249
+ print("This may be due to missing API keys or API service issues.")
250
+
251
+ # Example 3: Working with citations (if available from Perplexity)
252
+ print("\n3. Working with citations (if available from Perplexity):")
253
+ citation_query = "recent breakthroughs in fusion energy research"
254
+
255
+ try:
256
+ results = api_tool.search(citation_query)
257
+
258
+ for result in results:
259
+ if 'citations' in result:
260
+ print(f"Result includes {len(result['citations'])} citations:")
261
+ for i, citation in enumerate(result['citations'][:3]): # Show first 3 citations
262
+ print(f" Citation {i+1}: {citation.get('title', 'No title')} - {citation.get('url', 'No URL')}")
263
+ except Exception as e:
264
+ print(f"Error: {str(e)}")
265
+ print("This may be due to missing API keys or API service issues.")
266
+
267
+ def example_unified_search() -> None:
268
+ """
269
+ Demonstrate how to use all three search tools together through the tool registry.
270
+
271
+ This example shows:
272
+ - How to use the tool registry to access all search tools
273
+ - How to choose the right tool based on query type
274
+ - How to combine results from multiple tools
275
+ """
276
+ print_section("Unified Search Example")
277
+
278
+ # Create a tool registry with all search tools
279
+ registry = create_default_registry()
280
+ print("Tool registry created with the following search tools:")
281
+ search_tools = [tool for tool in registry.list_tools() if 'search' in tool]
282
+ for tool in search_tools:
283
+ print(f"- {tool}")
284
+
285
+ # Example 1: Using the registry to execute specific tools
286
+ print("\n1. Using the registry to execute specific tools:")
287
+ query = "renewable energy technologies"
288
+
289
+ print(f"\nQuery: '{query}'")
290
+ print("Executing with different search tools:")
291
+
292
+ # Try different search tools and compare results
293
+ results_by_tool = {}
294
+ for tool_name in ['duckduckgo_search', 'library_search', 'enhanced_web_search']:
295
+ if registry.get_tool(tool_name):
296
+ try:
297
+ print(f"\nExecuting {tool_name}...")
298
+ results = registry.execute_tool(tool_name, query=query)
299
+ results_by_tool[tool_name] = results
300
+ print(f"Got {len(results)} results from {tool_name}")
301
+ except Exception as e:
302
+ print(f"Error with {tool_name}: {str(e)}")
303
+
304
+ # Example 2: Using the unified_search function from tool_registry
305
+ print("\n2. Using the unified_search function:")
306
+ from agent.tool_registry import unified_search
307
+
308
+ query = "impact of artificial intelligence on healthcare"
309
+ print(f"\nQuery: '{query}'")
310
+ print("Executing unified_search...")
311
+
312
+ try:
313
+ unified_results = unified_search(registry, query)
314
+ print(f"Got results from the following providers: {unified_results['metadata']['providers_used']}")
315
+ print(f"Total results: {len(unified_results['results'])}")
316
+ print("\nTop 2 results:")
317
+ print_results(unified_results['results'][:2])
318
+
319
+ # Show execution times
320
+ print("Execution times:")
321
+ for provider, time in unified_results['metadata']['execution_times'].items():
322
+ print(f"- {provider}: {time:.2f} seconds")
323
+ except Exception as e:
324
+ print(f"Error with unified_search: {str(e)}")
325
+
326
+ # Example 3: Combining and ranking results from multiple tools
327
+ print("\n3. Combining and ranking results from multiple tools:")
328
+
329
+ # Function to combine results from multiple tools
330
+ def combine_and_rank_results(results_by_tool):
331
+ all_results = []
332
+ for tool, results in results_by_tool.items():
333
+ # Add tool name to each result if not already present
334
+ for result in results:
335
+ if 'source' not in result:
336
+ result['source'] = tool
337
+ all_results.append(result)
338
+
339
+ # Sort by relevance score if available
340
+ if all_results and 'relevance_score' in all_results[0]:
341
+ all_results.sort(key=lambda x: x.get('relevance_score', 0), reverse=True)
342
+
343
+ return all_results
344
+
345
+ combined_results = combine_and_rank_results(results_by_tool)
346
+ print(f"Combined {len(combined_results)} results from {len(results_by_tool)} tools")
347
+ print("\nTop 3 results after combining and ranking:")
348
+ print_results(combined_results[:3])
349
+
350
+ def main():
351
+ """Run all examples."""
352
+ print_section("Unified Search Examples")
353
+ print("This script demonstrates how to use all three search tools:")
354
+ print("1. BrowserSearchTool - For direct browser-based searches")
355
+ print("2. LibrarySearchTool - For searches using imported libraries")
356
+ print("3. ApiSearchTool - For searches using external API services")
357
+
358
+ # Run individual examples
359
+ example_browser_search()
360
+ example_library_search()
361
+ example_api_search()
362
+ example_unified_search()
363
+
364
+ print_section("Conclusion")
365
+ print("This example demonstrated how to use all three search tools:")
366
+ print("1. BrowserSearchTool - Best for visual content and interactive exploration")
367
+ print("2. LibrarySearchTool - Best for programmatic searches without API keys")
368
+ print("3. ApiSearchTool - Best for complex reasoning queries with high-quality results")
369
+ print("\nThese tools can be used individually or together through the tool registry.")
370
+ print("Choose the right tool based on your specific search needs and available resources.")
371
+
372
+ if __name__ == "__main__":
373
+ main()