Download ingest.py from Vdphacker/CyberAI: direct link, hf CLI and curl.
- Browser
- Download file 2.25 kB
-
https://huggingface.co/spaces/Vdphacker/CyberAI/resolve/main/ingest.py
- Command line
-
hf download hf://spaces/Vdphacker/CyberAI/ingest.py
-
curl -L -o ingest.py https://huggingface.co/spaces/Vdphacker/CyberAI/resolve/main/ingest.py
2.25 kB
| # ingest.py | |
| import json | |
| import os | |
| import time | |
| import requests | |
| from bs4 import BeautifulSoup | |
| import chromadb | |
| from chromadb.utils import embedding_functions | |
| def simple_extract_text(url: str) -> str: | |
| """Extrait le texte propre d'une page web.""" | |
| try: | |
| headers = {'User-Agent': 'CyberWise Assistant (educational bot)'} | |
| response = requests.get(url, headers=headers, timeout=15) | |
| response.raise_for_status() | |
| soup = BeautifulSoup(response.content, 'html.parser') | |
| for script in soup(["script", "style", "nav", "footer", "header", "noscript"]): | |
| script.decompose() | |
| text = soup.get_text(separator=' ', strip=True) | |
| return ' '.join(text.split()) | |
| except Exception as e: | |
| print(f" ❌ Échec extraction {url}: {e}") | |
| return "" | |
| def run_ingest(persist_directory="./chroma_db"): | |
| if os.path.exists(persist_directory): | |
| print("✅ Base vectorielle déjà présente.") | |
| return | |
| print("🔍 Indexation des sources de confiance...") | |
| with open("trusted_sources.json", "r", encoding="utf-8") as f: | |
| sources = json.load(f) | |
| embedding_function = embedding_functions.SentenceTransformerEmbeddingFunction( | |
| model_name="all-MiniLM-L6-v2" | |
| ) | |
| client = chromadb.PersistentClient(path=persist_directory) | |
| collection = client.get_or_create_collection( | |
| name="cyberwise", | |
| embedding_function=embedding_function | |
| ) | |
| documents = [] | |
| metadatas = [] | |
| ids = [] | |
| for i, src in enumerate(sources): | |
| print(f" → {src['url']}") | |
| text = simple_extract_text(src['url']) | |
| if text and len(text) > 100: | |
| # Découper en chunks de 500 caractères | |
| chunks = [text[j:j+500] for j in range(0, len(text), 500)] | |
| for k, chunk in enumerate(chunks): | |
| documents.append(chunk) | |
| metadatas.append({"source": src['url']}) | |
| ids.append(f"doc_{i}_{k}") | |
| time.sleep(1) | |
| if documents: | |
| collection.add(documents=documents, metadatas=metadatas, ids=ids) | |
| print(f"✅ {len(documents)} extraits indexés dans 'chroma_db'.") | |
| else: | |
| raise RuntimeError("Aucun texte valide n’a été extrait des sources.") |