File size: 2,248 Bytes
97b9aa0 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 | # ingest.py
import json
import os
import time
import requests
from bs4 import BeautifulSoup
import chromadb
from chromadb.utils import embedding_functions
def simple_extract_text(url: str) -> str:
"""Extrait le texte propre d'une page web."""
try:
headers = {'User-Agent': 'CyberWise Assistant (educational bot)'}
response = requests.get(url, headers=headers, timeout=15)
response.raise_for_status()
soup = BeautifulSoup(response.content, 'html.parser')
for script in soup(["script", "style", "nav", "footer", "header", "noscript"]):
script.decompose()
text = soup.get_text(separator=' ', strip=True)
return ' '.join(text.split())
except Exception as e:
print(f" ❌ Échec extraction {url}: {e}")
return ""
def run_ingest(persist_directory="./chroma_db"):
if os.path.exists(persist_directory):
print("✅ Base vectorielle déjà présente.")
return
print("🔍 Indexation des sources de confiance...")
with open("trusted_sources.json", "r", encoding="utf-8") as f:
sources = json.load(f)
embedding_function = embedding_functions.SentenceTransformerEmbeddingFunction(
model_name="all-MiniLM-L6-v2"
)
client = chromadb.PersistentClient(path=persist_directory)
collection = client.get_or_create_collection(
name="cyberwise",
embedding_function=embedding_function
)
documents = []
metadatas = []
ids = []
for i, src in enumerate(sources):
print(f" → {src['url']}")
text = simple_extract_text(src['url'])
if text and len(text) > 100:
# Découper en chunks de 500 caractères
chunks = [text[j:j+500] for j in range(0, len(text), 500)]
for k, chunk in enumerate(chunks):
documents.append(chunk)
metadatas.append({"source": src['url']})
ids.append(f"doc_{i}_{k}")
time.sleep(1)
if documents:
collection.add(documents=documents, metadatas=metadatas, ids=ids)
print(f"✅ {len(documents)} extraits indexés dans 'chroma_db'.")
else:
raise RuntimeError("Aucun texte valide n’a été extrait des sources.") |