CyberAI / ingest.py
Vdphacker's picture
Update ingest.py
97b9aa0 verified
Raw History Blame Contribute Delete
2.25 kB
# ingest.py
import json
import os
import time
import requests
from bs4 import BeautifulSoup
import chromadb
from chromadb.utils import embedding_functions
def simple_extract_text(url: str) -> str:
"""Extrait le texte propre d'une page web."""
try:
headers = {'User-Agent': 'CyberWise Assistant (educational bot)'}
response = requests.get(url, headers=headers, timeout=15)
response.raise_for_status()
soup = BeautifulSoup(response.content, 'html.parser')
for script in soup(["script", "style", "nav", "footer", "header", "noscript"]):
script.decompose()
text = soup.get_text(separator=' ', strip=True)
return ' '.join(text.split())
except Exception as e:
print(f" ❌ Échec extraction {url}: {e}")
return ""
def run_ingest(persist_directory="./chroma_db"):
if os.path.exists(persist_directory):
print("✅ Base vectorielle déjà présente.")
return
print("🔍 Indexation des sources de confiance...")
with open("trusted_sources.json", "r", encoding="utf-8") as f:
sources = json.load(f)
embedding_function = embedding_functions.SentenceTransformerEmbeddingFunction(
model_name="all-MiniLM-L6-v2"
)
client = chromadb.PersistentClient(path=persist_directory)
collection = client.get_or_create_collection(
name="cyberwise",
embedding_function=embedding_function
)
documents = []
metadatas = []
ids = []
for i, src in enumerate(sources):
print(f" → {src['url']}")
text = simple_extract_text(src['url'])
if text and len(text) > 100:
# Découper en chunks de 500 caractères
chunks = [text[j:j+500] for j in range(0, len(text), 500)]
for k, chunk in enumerate(chunks):
documents.append(chunk)
metadatas.append({"source": src['url']})
ids.append(f"doc_{i}_{k}")
time.sleep(1)
if documents:
collection.add(documents=documents, metadatas=metadatas, ids=ids)
print(f"✅ {len(documents)} extraits indexés dans 'chroma_db'.")
else:
raise RuntimeError("Aucun texte valide n’a été extrait des sources.")