Download web_researcher.py from Vdphacker/Cyber_IA: direct link, hf CLI and curl.
- Browser
- Download file 2.19 kB
-
https://huggingface.co/spaces/Vdphacker/Cyber_IA/resolve/main/web_researcher.py
- Command line
-
hf download hf://spaces/Vdphacker/Cyber_IA/web_researcher.py
-
curl -L -o web_researcher.py https://huggingface.co/spaces/Vdphacker/Cyber_IA/resolve/main/web_researcher.py
2.19 kB
| import os | |
| from scrapegraphai.graphs import SmartScraperGraph | |
| from langchain_huggingface import HuggingFaceEndpoint | |
| # Liste des sites autorisés (à étendre) | |
| ALLOWED_SITES = [ | |
| "https://www.ssi.gouv.fr", # ANSSI (France) | |
| "https://www.cert.ssi.gouv.fr", # CERT-FR | |
| "https://csrc.nist.gov", # NIST (USA) | |
| "https://www.sans.org", # SANS Institute | |
| "https://attack.mitre.org", # MITRE ATT&CK | |
| "https://owasp.org", # OWASP | |
| "https://www.cisa.gov", # CISA (USA) | |
| ] | |
| def rechercher_sur_web(question: str) -> str: | |
| """ | |
| Recherche la réponse sur les sites de cybersécurité autorisés. | |
| ScrapeGraph-AI respecte automatiquement le robots.txt. | |
| """ | |
| results = [] | |
| for base_url in ALLOWED_SITES: | |
| try: | |
| # ScrapeGraph vérifie robots.txt avant de crawler | |
| graph_config = { | |
| "llm": { | |
| "model": "ollama/llama3", # ou utilise Hugging Face Endpoint | |
| "temperature": 0, | |
| "format": "json", | |
| "base_url": "http://localhost:11434", # si tu utilises Ollama | |
| }, | |
| "verbose": False, | |
| "headless": True, | |
| "respect_robots_txt": True, # 🔑 clé de ton exigence | |
| } | |
| # On cherche via le moteur de recherche interne du site (si possible) | |
| # Sinon, on scrape la page d'accueil (moins efficace) | |
| search_url = f"{base_url}/?s={question.replace(' ', '+')}" # ajuste selon le site | |
| smart_scraper = SmartScraperGraph( | |
| prompt=f"Trouve des informations techniques sur : {question}", | |
| source=search_url, | |
| config=graph_config | |
| ) | |
| result = smart_scraper.run() | |
| if result and "content" in result: | |
| results.append(f"Source: {base_url}\n{result['content'][:1000]}...") | |
| except Exception as e: | |
| print(f"⚠️ Erreur avec {base_url}: {e}") | |
| continue | |
| return "\n\n".join(results) if results else "Aucune source fiable trouvée." |