GenerAI / scraper.py
amogaddy's picture
Sync aggiornamento
61adc39 verified
Raw History Blame
4.49 kB
import os
import requests
import trafilatura
from errors import get_logger, GenerAIError, ErrorCode, fmt_exc
log = get_logger("scraper")
# Istanza SearXNG — configurabile via Secret SEARXNG_URL
SEARXNG_URL = os.environ.get("SEARXNG_URL", "https://searx.be")
_active_instance = SEARXNG_URL
PUBLIC_INSTANCES = [
"https://searx.be",
"https://search.bus-hit.me",
"https://searxng.site",
]
# Motori attivi dentro SearXNG — brave, bing (Edge), torch, duckduckgo come fallback
SEARXNG_ENGINES = os.environ.get("SEARXNG_ENGINES", "brave,bing,torch,duckduckgo")
def get_active_instance() -> str:
return _active_instance
def set_active_instance(url: str):
global _active_instance
_active_instance = url.rstrip("/")
log.info("Istanza SearXNG cambiata: %s", _active_instance)
def get_engines() -> str:
return SEARXNG_ENGINES
def set_engines(engines: str):
global SEARXNG_ENGINES
SEARXNG_ENGINES = engines
log.info("Motori SearXNG aggiornati: %s", SEARXNG_ENGINES)
def _searxng_search(query: str, max_results: int) -> list[dict]:
"""Cerca con SearXNG usando più motori. Prova più istanze se la prima fallisce."""
active = get_active_instance()
instances = [active] + [i for i in PUBLIC_INSTANCES if i != active]
engines = get_engines()
for instance in instances:
try:
resp = requests.get(
f"{instance}/search",
params={"q": query, "format": "json", "language": "it", "engines": engines},
timeout=8,
headers={"User-Agent": "GenerAI/1.0"},
)
if resp.status_code == 200:
data = resp.json()
hits = data.get("results", [])[:max_results * 2]
if hits:
log.info("SearXNG (%s) [%s] → %d risultati", instance, engines, len(hits))
return hits
# Nessun risultato con questi motori → riprova senza filtro motori
resp2 = requests.get(
f"{instance}/search",
params={"q": query, "format": "json", "language": "it"},
timeout=8,
headers={"User-Agent": "GenerAI/1.0"},
)
if resp2.status_code == 200:
hits2 = resp2.json().get("results", [])[:max_results * 2]
log.info("SearXNG (%s) [fallback tutti motori] → %d risultati", instance, len(hits2))
return hits2
except Exception as e:
log.warning("SearXNG %s fallita: %s", instance, fmt_exc(e))
return []
def search_and_extract(query: str, max_results: int = 3) -> list[dict]:
"""Cerca con SearXNG ed estrae testo pulito. Returns [] on total failure."""
log.info("Ricerca web per: %r", query)
hits = _searxng_search(query, max_results)
if not hits:
log.warning("[%s] Nessun risultato SearXNG per: %r", ErrorCode.WEB_NO_RESULTS.value, query)
return []
results = []
for hit in hits:
if len(results) >= max_results:
break
# SearXNG include già uno snippet — usalo se il fetch fallisce
url = hit.get("url", "")
title = hit.get("title", "")
snippet = hit.get("content", "")
if not url:
continue
log.debug("Fetching: %s", url)
text = None
try:
downloaded = trafilatura.fetch_url(url, timeout=10)
if downloaded:
text = trafilatura.extract(
downloaded,
include_links=False,
include_images=False,
include_tables=False,
no_fallback=False,
)
except Exception as e:
log.warning("[%s] fetch fallito per %s — %s", ErrorCode.WEB_FETCH_FAILED.value, url, fmt_exc(e))
# Fallback allo snippet SearXNG se trafilatura non estrae nulla
if not text or len(text) < 100:
text = snippet
if not text or len(text) < 50:
log.debug("Testo troppo corto per: %s", url)
continue
log.info("Estratti %d chars da: %s", len(text), url)
results.append({
"title": title,
"url": url,
"text": text[:2000],
})
if not results:
log.warning("[%s] Nessun testo utile estratto per: %r", ErrorCode.WEB_NO_RESULTS.value, query)
return results