# ingest.py import json import os import time import requests from bs4 import BeautifulSoup import chromadb from chromadb.utils import embedding_functions def simple_extract_text(url: str) -> str: """Extrait le texte propre d'une page web.""" try: headers = {'User-Agent': 'CyberWise Assistant (educational bot)'} response = requests.get(url, headers=headers, timeout=15) response.raise_for_status() soup = BeautifulSoup(response.content, 'html.parser') for script in soup(["script", "style", "nav", "footer", "header", "noscript"]): script.decompose() text = soup.get_text(separator=' ', strip=True) return ' '.join(text.split()) except Exception as e: print(f" ❌ Échec extraction {url}: {e}") return "" def run_ingest(persist_directory="./chroma_db"): if os.path.exists(persist_directory): print("✅ Base vectorielle déjà présente.") return print("🔍 Indexation des sources de confiance...") with open("trusted_sources.json", "r", encoding="utf-8") as f: sources = json.load(f) embedding_function = embedding_functions.SentenceTransformerEmbeddingFunction( model_name="all-MiniLM-L6-v2" ) client = chromadb.PersistentClient(path=persist_directory) collection = client.get_or_create_collection( name="cyberwise", embedding_function=embedding_function ) documents = [] metadatas = [] ids = [] for i, src in enumerate(sources): print(f" → {src['url']}") text = simple_extract_text(src['url']) if text and len(text) > 100: # Découper en chunks de 500 caractères chunks = [text[j:j+500] for j in range(0, len(text), 500)] for k, chunk in enumerate(chunks): documents.append(chunk) metadatas.append({"source": src['url']}) ids.append(f"doc_{i}_{k}") time.sleep(1) if documents: collection.add(documents=documents, metadatas=metadatas, ids=ids) print(f"✅ {len(documents)} extraits indexés dans 'chroma_db'.") else: raise RuntimeError("Aucun texte valide n’a été extrait des sources.")