alyxsis commited on
Commit
bc683fb
·
verified ·
1 Parent(s): 41ffb08

Search: fall through to other engines when DuckDuckGo is throttled; better broader-query hint

Browse files
Files changed (2) hide show
  1. Modules/Web_Search.py +179 -64
  2. README.md +1 -1
Modules/Web_Search.py CHANGED
@@ -5,6 +5,8 @@ import sys
5
  import threading
6
  import time
7
  from collections import OrderedDict
 
 
8
  from datetime import datetime
9
  from typing import Annotated, Any
10
 
@@ -23,8 +25,8 @@ TOOL_SUMMARY = (
23
  )
24
 
25
  _SAFESEARCH_LEVEL = "off"
26
- _DDGS_TIMEOUT_SECONDS = 10
27
- _SEARCH_DEADLINE_SECONDS = 25.0
28
  _MAX_PAGES_PER_BACKEND = 5
29
  MAX_RESULTS_CAP = 20
30
 
@@ -35,7 +37,7 @@ DATE_FILTER_CHOICES = ["any", "day", "week", "month", "year"]
35
  # Preferred order per type. Names the installed ddgs does not offer for a type
36
  # (e.g. ddgs 9.x has no Bing *text* engine) are dropped at runtime.
37
  _AUTO_ORDER = {
38
- "text": ["duckduckgo", "bing", "brave", "yahoo"],
39
  "news": ["duckduckgo", "bing", "yahoo"],
40
  "images": ["duckduckgo", "bing"],
41
  "videos": ["duckduckgo"],
@@ -166,6 +168,8 @@ class _SearchRun:
166
  self.backend_idx = 0
167
  self.page = 1
168
  self.exhausted = False
 
 
169
  self.created = time.monotonic()
170
  self.lock = threading.Lock()
171
 
@@ -175,8 +179,12 @@ _CACHE_LOCK = threading.Lock()
175
  _CACHE_TTL_SECONDS = 600.0
176
  _CACHE_MAX_ENTRIES = 256
177
 
 
 
 
178
 
179
- def _get_run(key: tuple) -> tuple[_SearchRun, bool]:
 
180
  with _CACHE_LOCK:
181
  now = time.monotonic()
182
  for k in [k for k, v in _CACHE.items() if now - v.created > _CACHE_TTL_SECONDS]:
@@ -184,62 +192,123 @@ def _get_run(key: tuple) -> tuple[_SearchRun, bool]:
184
  run = _CACHE.get(key)
185
  if run is not None:
186
  _CACHE.move_to_end(key)
187
- return run, True
188
  run = _SearchRun()
189
  _CACHE[key] = run
190
  while len(_CACHE) > _CACHE_MAX_ENTRIES:
191
  _CACHE.popitem(last=False)
192
- return run, False
 
 
 
 
 
 
 
 
 
 
 
 
 
193
 
194
 
195
- def _collect(query: str, stype: str, total_needed: int, timelimit: str | None, backends: list[str], explicit: bool) -> list[dict[str, Any]]:
196
- """Walk backends (and their result pages) until total_needed unique results are gathered."""
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
197
  key = (query.lower(), stype, timelimit, tuple(backends))
198
- run, _cached = _get_run(key)
 
199
  with run.lock:
200
  if len(run.results) >= total_needed or run.exhausted:
201
- return list(run.results)
202
  if not SEARCH_LIMITER.acquire():
203
  raise RuntimeError(RATE_LIMIT_MESSAGE.format(limit=SEARCH_LIMITER.per_minute).removeprefix("Error: "))
204
- started = time.monotonic()
205
- last_error: Exception | None = None
206
- with DDGS(timeout=_DDGS_TIMEOUT_SECONDS) as ddgs:
207
- while run.backend_idx < len(backends) and len(run.results) < total_needed:
208
- if time.monotonic() - started > _SEARCH_DEADLINE_SECONDS:
209
- _log("deadline reached; returning what was gathered")
210
- break
211
- backend = backends[run.backend_idx]
212
- t0 = time.perf_counter()
213
- try:
214
- items = _call(ddgs, stype, query, backend, timelimit, run.page)
215
- except Exception as exc:
216
- _log(f"{stype} backend={backend} page={run.page} failed after {time.perf_counter() - t0:.2f}s: {str(exc)[:120]}")
217
- if not _is_no_results(exc):
218
- last_error = exc
219
- items = []
220
- added = 0
221
- for item in items:
222
- marker = _unique_key(item, stype)
223
- if marker and marker not in run.seen:
224
- run.seen.add(marker)
225
- run.results.append(item)
226
- added += 1
227
- if items:
228
- _log(f"{stype} backend={backend} page={run.page} results={len(items)} new={added} in {time.perf_counter() - t0:.2f}s")
229
- if added == 0 or run.page >= _MAX_PAGES_PER_BACKEND:
230
- run.backend_idx += 1
231
- run.page = 1
232
- else:
233
- run.page += 1
234
- if run.backend_idx >= len(backends):
235
- run.exhausted = True
236
- results = list(run.results)
237
- if not results:
 
 
 
238
  with _CACHE_LOCK:
239
  _CACHE.pop(key, None) # never cache an empty (possibly rate-limited) answer
240
- if last_error is not None and (explicit or not _is_no_results(last_error)):
241
- raise RuntimeError(_normalize_error(last_error))
242
- return results
 
 
 
 
 
 
 
 
 
 
 
243
 
244
 
245
  def _format_result(item: dict[str, Any], stype: str, index: int) -> list[str]:
@@ -323,9 +392,35 @@ def _fetchable_url(item: dict[str, Any], stype: str) -> str:
323
  return ""
324
 
325
 
 
 
 
 
 
 
 
 
 
 
326
  def _broader_query(query: str) -> str:
327
- words = [w for w in query.split() if not re.match(r"^-?\w+:", w)] # drop site:, filetype: ...
328
- return " ".join(words[:4]) or query
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
329
 
330
 
331
  @autodoc(summary=TOOL_SUMMARY)
@@ -373,33 +468,50 @@ def Web_Search(
373
  explicit = (backend or "auto").strip().lower() != "auto"
374
  timelimit = _resolve_timelimit(date_filter, search_type)
375
  used_fallback = False
 
376
 
377
  try:
378
- raw = _collect(query, search_type, total_needed, timelimit, backends, explicit)
379
  except Exception as exc:
380
- return done(f"Error: {exc}")
 
 
381
 
382
- if not raw and search_type == "news":
383
  text_backends, _ = _backend_plan("text", backend)
384
  try:
385
- raw = _collect(query, "text", total_needed, _resolve_timelimit(date_filter, "text"), text_backends, False)
386
  except Exception:
387
- raw = []
388
- if raw:
389
- used_fallback = True
390
- search_type = "text"
 
 
 
 
391
 
392
  if not raw:
393
  fallback_note = " (also tried 'text' search as fallback)" if original_type == "news" else ""
394
- hint = ["", "Try a broader search:"]
395
- hint.append(f' Web_Search(query="{_broader_query(query)}")')
 
 
 
 
 
 
 
396
  if timelimit:
397
- hint.append(f" or drop the date filter (date_filter=\"any\"; it is currently \"{date_filter}\").")
398
  if explicit:
399
- hint.append(' or use backend="auto" to try every available search engine.')
400
- if backend_note:
401
- hint.insert(0, f"Note: {backend_note}")
402
- return done(f"No {original_type} results found for query: {query}{fallback_note}\n" + "\n".join(hint))
 
 
 
403
 
404
  paginated = raw[actual_offset: actual_offset + max_results]
405
  if not paginated:
@@ -425,6 +537,9 @@ def Web_Search(
425
  lines.append("Note: News search returned no results, so general web results are shown instead.")
426
  if backend_note:
427
  lines.append(f"Note: {backend_note}")
 
 
 
428
  if date_filter not in ("any", "", "none") and timelimit:
429
  lines.append(f"Date filter: {date_filter}")
430
  lines.append(f"{pagination_info} (results {start_num}-{end_num} of ~{total_available}+ available)")
 
5
  import threading
6
  import time
7
  from collections import OrderedDict
8
+ from concurrent.futures import ThreadPoolExecutor
9
+ from concurrent.futures import TimeoutError as FutureTimeout
10
  from datetime import datetime
11
  from typing import Annotated, Any
12
 
 
25
  )
26
 
27
  _SAFESEARCH_LEVEL = "off"
28
+ _ENGINE_TIMEOUT_SECONDS = 4.0 # per engine request; a slow or throttled engine is skipped
29
+ _SEARCH_BUDGET_SECONDS = 12.0 # whole tool call, including the news->web fallback
30
  _MAX_PAGES_PER_BACKEND = 5
31
  MAX_RESULTS_CAP = 20
32
 
 
37
  # Preferred order per type. Names the installed ddgs does not offer for a type
38
  # (e.g. ddgs 9.x has no Bing *text* engine) are dropped at runtime.
39
  _AUTO_ORDER = {
40
+ "text": ["duckduckgo", "bing", "yahoo", "brave", "mojeek", "startpage", "google", "wikipedia"],
41
  "news": ["duckduckgo", "bing", "yahoo"],
42
  "images": ["duckduckgo", "bing"],
43
  "videos": ["duckduckgo"],
 
168
  self.backend_idx = 0
169
  self.page = 1
170
  self.exhausted = False
171
+ self.answered_by: list[str] = [] # engines that contributed results, in order
172
+ self.skipped: dict[str, str] = {} # engine -> why it gave nothing
173
  self.created = time.monotonic()
174
  self.lock = threading.Lock()
175
 
 
179
  _CACHE_TTL_SECONDS = 600.0
180
  _CACHE_MAX_ENTRIES = 256
181
 
182
+ # Engine calls run here so each one can be abandoned after _ENGINE_TIMEOUT_SECONDS,
183
+ # even if ddgs itself blocks longer (its internal pool waits for slow engines).
184
+ _ENGINE_POOL = ThreadPoolExecutor(max_workers=16, thread_name_prefix="engine")
185
 
186
+
187
+ def _get_run(key: tuple) -> _SearchRun:
188
  with _CACHE_LOCK:
189
  now = time.monotonic()
190
  for k in [k for k, v in _CACHE.items() if now - v.created > _CACHE_TTL_SECONDS]:
 
192
  run = _CACHE.get(key)
193
  if run is not None:
194
  _CACHE.move_to_end(key)
195
+ return run
196
  run = _SearchRun()
197
  _CACHE[key] = run
198
  while len(_CACHE) > _CACHE_MAX_ENTRIES:
199
  _CACHE.popitem(last=False)
200
+ return run
201
+
202
+
203
+ def _engine_call(stype: str, query: str, backend: str, timelimit: str | None, page: int, timeout: float) -> list[dict[str, Any]]:
204
+ def work() -> list[dict[str, Any]]:
205
+ with DDGS(timeout=max(1, int(round(timeout)))) as ddgs:
206
+ return _call(ddgs, stype, query, backend, timelimit, page)
207
+
208
+ future = _ENGINE_POOL.submit(work)
209
+ try:
210
+ return future.result(timeout=timeout)
211
+ except FutureTimeout as exc:
212
+ future.cancel()
213
+ raise TimeoutError(f"{backend} timed out after {timeout:.0f}s") from exc
214
 
215
 
216
+ def _skip_reason(exc: Exception) -> str:
217
+ lowered = str(exc).lower()
218
+ if isinstance(exc, TimeoutError) or "timed out" in lowered or "timeout" in lowered:
219
+ return "timed out"
220
+ if _is_no_results(exc):
221
+ return "no results, possibly rate-limited"
222
+ if "ratelimit" in lowered or "rate limit" in lowered or "429" in lowered:
223
+ return "rate-limited"
224
+ return f"error ({type(exc).__name__})"
225
+
226
+
227
+ def _snapshot(run: _SearchRun) -> _SearchRun:
228
+ snap = _SearchRun()
229
+ snap.results = list(run.results)
230
+ snap.answered_by = list(run.answered_by)
231
+ snap.skipped = dict(run.skipped)
232
+ snap.exhausted = run.exhausted
233
+ return snap
234
+
235
+
236
+ def _collect(
237
+ query: str,
238
+ stype: str,
239
+ total_needed: int,
240
+ timelimit: str | None,
241
+ backends: list[str],
242
+ explicit: bool,
243
+ deadline: float,
244
+ ) -> _SearchRun:
245
+ """Walk engines in order (and their result pages) until total_needed unique results
246
+ are gathered, the engines run out, or the shared time budget is spent.
247
+
248
+ An engine that times out, is rate-limited, errors, or returns nothing is skipped
249
+ and the next one is tried. Returns a snapshot (results, answered_by, skipped).
250
+ """
251
  key = (query.lower(), stype, timelimit, tuple(backends))
252
+ run = _get_run(key)
253
+ last_error: Exception | None = None
254
  with run.lock:
255
  if len(run.results) >= total_needed or run.exhausted:
256
+ return _snapshot(run)
257
  if not SEARCH_LIMITER.acquire():
258
  raise RuntimeError(RATE_LIMIT_MESSAGE.format(limit=SEARCH_LIMITER.per_minute).removeprefix("Error: "))
259
+ while run.backend_idx < len(backends) and len(run.results) < total_needed:
260
+ remaining = deadline - time.monotonic()
261
+ if remaining < 1.0:
262
+ _log("time budget spent; returning what was gathered")
263
+ break
264
+ backend = backends[run.backend_idx]
265
+ t0 = time.perf_counter()
266
+ try:
267
+ items = _engine_call(stype, query, backend, timelimit, run.page, min(_ENGINE_TIMEOUT_SECONDS, remaining))
268
+ except Exception as exc:
269
+ reason = _skip_reason(exc)
270
+ _log(f"{stype} backend={backend} page={run.page} skipped after {time.perf_counter() - t0:.2f}s: {reason}")
271
+ if run.page == 1:
272
+ run.skipped[backend] = reason
273
+ if not _is_no_results(exc):
274
+ last_error = exc
275
+ items = []
276
+ added = 0
277
+ for item in items:
278
+ marker = _unique_key(item, stype)
279
+ if marker and marker not in run.seen:
280
+ run.seen.add(marker)
281
+ run.results.append(item)
282
+ added += 1
283
+ if items:
284
+ _log(f"{stype} backend={backend} page={run.page} results={len(items)} new={added} in {time.perf_counter() - t0:.2f}s")
285
+ if added and backend not in run.answered_by:
286
+ run.answered_by.append(backend)
287
+ if added == 0 or run.page >= _MAX_PAGES_PER_BACKEND:
288
+ run.backend_idx += 1
289
+ run.page = 1
290
+ else:
291
+ run.page += 1
292
+ if run.backend_idx >= len(backends):
293
+ run.exhausted = True
294
+ snap = _snapshot(run)
295
+ if not snap.results:
296
  with _CACHE_LOCK:
297
  _CACHE.pop(key, None) # never cache an empty (possibly rate-limited) answer
298
+ if explicit and last_error is not None:
299
+ raise RuntimeError(f"backend '{backends[0]}': {_normalize_error(last_error)}")
300
+ return snap
301
+
302
+
303
+ def _engine_note(snap: _SearchRun, explicit: bool) -> str:
304
+ if not snap.answered_by:
305
+ return ""
306
+ answered = ", ".join(snap.answered_by)
307
+ skipped = [f"{name}: {why}" for name, why in snap.skipped.items() if name not in snap.answered_by]
308
+ if skipped and not explicit:
309
+ shown = "; ".join(skipped[:3]) + (f"; +{len(skipped) - 3} more" if len(skipped) > 3 else "")
310
+ return f"Answered by {answered} (skipped {shown})."
311
+ return f"Answered by {answered}."
312
 
313
 
314
  def _format_result(item: dict[str, Any], stype: str, index: int) -> list[str]:
 
392
  return ""
393
 
394
 
395
+ _STOPWORDS = {
396
+ "a", "an", "the", "in", "on", "of", "for", "to", "and", "with", "at", "by", "from", "is", "are",
397
+ "what", "how", "why", "when", "where", "who", "which", "does", "do", "can", "near", "about",
398
+ }
399
+
400
+
401
+ def _norm(text: str) -> str:
402
+ return " ".join(text.lower().split())
403
+
404
+
405
  def _broader_query(query: str) -> str:
406
+ """A genuinely broader query, or "" when there is nothing sensible to drop.
407
+
408
+ Steps: drop operators (site:, filetype:, -exclusions), quotes and OR; keep at most
409
+ four words; if that changes nothing and the query has 3+ words, drop the last word.
410
+ Never returns the original query.
411
+ """
412
+ words = [w for w in query.split() if not re.match(r"^-|^\w+:", w) and w.upper() != "OR"]
413
+ words = [w.strip('"\'') for w in words]
414
+ words = [w for w in words if w]
415
+ content = [w for w in words if w.lower() not in _STOPWORDS]
416
+ if content and len(content) < len(words):
417
+ words = content
418
+ candidate = " ".join(words[:4])
419
+ if _norm(candidate) == _norm(query) and len(words) >= 3:
420
+ candidate = " ".join(words[: min(4, len(words)) - 1])
421
+ if not candidate or _norm(candidate) == _norm(query):
422
+ return ""
423
+ return candidate
424
 
425
 
426
  @autodoc(summary=TOOL_SUMMARY)
 
468
  explicit = (backend or "auto").strip().lower() != "auto"
469
  timelimit = _resolve_timelimit(date_filter, search_type)
470
  used_fallback = False
471
+ deadline = time.monotonic() + _SEARCH_BUDGET_SECONDS
472
 
473
  try:
474
+ snap = _collect(query, search_type, total_needed, timelimit, backends, explicit, deadline)
475
  except Exception as exc:
476
+ hint = ' Try backend="auto" to fall through to other search engines.' if explicit else ""
477
+ return done(f"Error: {str(exc).rstrip('.')}.{hint}")
478
+ tried = dict(snap.skipped)
479
 
480
+ if not snap.results and search_type == "news":
481
  text_backends, _ = _backend_plan("text", backend)
482
  try:
483
+ text_snap = _collect(query, "text", total_needed, _resolve_timelimit(date_filter, "text"), text_backends, False, deadline)
484
  except Exception:
485
+ text_snap = None
486
+ if text_snap is not None:
487
+ tried.update({f"{k} (text)": v for k, v in text_snap.skipped.items()})
488
+ if text_snap.results:
489
+ snap = text_snap
490
+ used_fallback = True
491
+ search_type = "text"
492
+ raw = snap.results
493
 
494
  if not raw:
495
  fallback_note = " (also tried 'text' search as fallback)" if original_type == "news" else ""
496
+ hint: list[str] = []
497
+ if backend_note:
498
+ hint.append(f"Note: {backend_note}")
499
+ if tried:
500
+ hint.append("Engines tried: " + "; ".join(f"{k}: {v}" for k, v in tried.items()) + ".")
501
+ suggestions: list[str] = []
502
+ broader = _broader_query(query)
503
+ if broader:
504
+ suggestions.append(f' Web_Search(query="{broader}")')
505
  if timelimit:
506
+ suggestions.append(f" drop the date filter (date_filter=\"any\"; it is currently \"{date_filter}\")")
507
  if explicit:
508
+ suggestions.append(' use backend="auto" to try every available search engine')
509
+ if any("rate-limited" in v or "timed out" in v for v in tried.values()):
510
+ suggestions.append(" wait a few seconds and retry; the search engines may be throttling this server")
511
+ if not suggestions:
512
+ suggestions.append(" rephrase the query with different words")
513
+ body = "\n".join(hint + ["", "Try:"] + suggestions)
514
+ return done(f"No {original_type} results found for query: {query}{fallback_note}\n{body}")
515
 
516
  paginated = raw[actual_offset: actual_offset + max_results]
517
  if not paginated:
 
537
  lines.append("Note: News search returned no results, so general web results are shown instead.")
538
  if backend_note:
539
  lines.append(f"Note: {backend_note}")
540
+ engine_note = _engine_note(snap, explicit)
541
+ if engine_note:
542
+ lines.append(f"Note: {engine_note}")
543
  if date_filter not in ("any", "", "none") and timelimit:
544
  lines.append(f"Date filter: {date_filter}")
545
  lines.append(f"{pagination_info} (results {start_num}-{end_num} of ~{total_available}+ available)")
README.md CHANGED
@@ -43,7 +43,7 @@ On this Space the MCP tools are named `Tools_Web_Search` and `Tools_Web_Fetch`.
43
  `Web_Search(query, max_results=5, page=1, offset=0, search_type="text", backend="auto", date_filter="any")`
44
 
45
  - `search_type`: `text` | `news` | `images` | `videos` | `books`. Empty news searches fall back to web results and say so.
46
- - `backend`: `auto` | `duckduckgo` | `bing` | `brave` | `yahoo` | `wikipedia`. A backend the search type does not support is replaced with a working one, and the output notes the swap.
47
  - `date_filter`: `any` | `day` | `week` | `month` | `year`.
48
  - `max_results` is 1–20. `offset` overrides `page`. Continuations come from a 10-minute result cache, so pages stay consistent.
49
 
 
43
  `Web_Search(query, max_results=5, page=1, offset=0, search_type="text", backend="auto", date_filter="any")`
44
 
45
  - `search_type`: `text` | `news` | `images` | `videos` | `books`. Empty news searches fall back to web results and say so.
46
+ - `backend`: `auto` | `duckduckgo` | `bing` | `brave` | `yahoo` | `wikipedia`. `auto` tries engines in turn: text uses DuckDuckGo, Yahoo, Brave, Mojeek, Startpage, Google, then Wikipedia. Each engine gets 4 s, and the whole call has a 12 s budget. A slow, throttled, or empty engine is skipped, and the output names the engine that answered. A backend the search type does not support is replaced with a working one, and the output notes the swap.
47
  - `date_filter`: `any` | `day` | `week` | `month` | `year`.
48
  - `max_results` is 1–20. `offset` overrides `page`. Continuations come from a 10-minute result cache, so pages stay consistent.
49