import os import requests import trafilatura from errors import get_logger, GenerAIError, ErrorCode, fmt_exc log = get_logger("scraper") # Istanza SearXNG — configurabile via Secret SEARXNG_URL SEARXNG_URL = os.environ.get("SEARXNG_URL", "https://searx.be") _active_instance = SEARXNG_URL PUBLIC_INSTANCES = [ "https://searx.be", "https://search.bus-hit.me", "https://searxng.site", ] # Motori attivi dentro SearXNG — brave, bing (Edge), torch, duckduckgo come fallback SEARXNG_ENGINES = os.environ.get("SEARXNG_ENGINES", "brave,bing,torch,duckduckgo") def get_active_instance() -> str: return _active_instance def set_active_instance(url: str): global _active_instance _active_instance = url.rstrip("/") log.info("Istanza SearXNG cambiata: %s", _active_instance) def get_engines() -> str: return SEARXNG_ENGINES def set_engines(engines: str): global SEARXNG_ENGINES SEARXNG_ENGINES = engines log.info("Motori SearXNG aggiornati: %s", SEARXNG_ENGINES) def _searxng_search(query: str, max_results: int) -> list[dict]: """Cerca con SearXNG usando più motori. Prova più istanze se la prima fallisce.""" active = get_active_instance() instances = [active] + [i for i in PUBLIC_INSTANCES if i != active] engines = get_engines() for instance in instances: try: resp = requests.get( f"{instance}/search", params={"q": query, "format": "json", "language": "it", "engines": engines}, timeout=8, headers={"User-Agent": "GenerAI/1.0"}, ) if resp.status_code == 200: data = resp.json() hits = data.get("results", [])[:max_results * 2] if hits: log.info("SearXNG (%s) [%s] → %d risultati", instance, engines, len(hits)) return hits # Nessun risultato con questi motori → riprova senza filtro motori resp2 = requests.get( f"{instance}/search", params={"q": query, "format": "json", "language": "it"}, timeout=8, headers={"User-Agent": "GenerAI/1.0"}, ) if resp2.status_code == 200: hits2 = resp2.json().get("results", [])[:max_results * 2] log.info("SearXNG (%s) [fallback tutti motori] → %d risultati", instance, len(hits2)) return hits2 except Exception as e: log.warning("SearXNG %s fallita: %s", instance, fmt_exc(e)) return [] def search_and_extract(query: str, max_results: int = 3) -> list[dict]: """Cerca con SearXNG ed estrae testo pulito. Returns [] on total failure.""" log.info("Ricerca web per: %r", query) hits = _searxng_search(query, max_results) if not hits: log.warning("[%s] Nessun risultato SearXNG per: %r", ErrorCode.WEB_NO_RESULTS.value, query) return [] results = [] for hit in hits: if len(results) >= max_results: break # SearXNG include già uno snippet — usalo se il fetch fallisce url = hit.get("url", "") title = hit.get("title", "") snippet = hit.get("content", "") if not url: continue log.debug("Fetching: %s", url) text = None try: downloaded = trafilatura.fetch_url(url, timeout=10) if downloaded: text = trafilatura.extract( downloaded, include_links=False, include_images=False, include_tables=False, no_fallback=False, ) except Exception as e: log.warning("[%s] fetch fallito per %s — %s", ErrorCode.WEB_FETCH_FAILED.value, url, fmt_exc(e)) # Fallback allo snippet SearXNG se trafilatura non estrae nulla if not text or len(text) < 100: text = snippet if not text or len(text) < 50: log.debug("Testo troppo corto per: %s", url) continue log.info("Estratti %d chars da: %s", len(text), url) results.append({ "title": title, "url": url, "text": text[:2000], }) if not results: log.warning("[%s] Nessun testo utile estratto per: %r", ErrorCode.WEB_NO_RESULTS.value, query) return results