File size: 2,248 Bytes
97b9aa0
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
# ingest.py
import json
import os
import time
import requests
from bs4 import BeautifulSoup
import chromadb
from chromadb.utils import embedding_functions

def simple_extract_text(url: str) -> str:
    """Extrait le texte propre d'une page web."""
    try:
        headers = {'User-Agent': 'CyberWise Assistant (educational bot)'}
        response = requests.get(url, headers=headers, timeout=15)
        response.raise_for_status()
        soup = BeautifulSoup(response.content, 'html.parser')
        for script in soup(["script", "style", "nav", "footer", "header", "noscript"]):
            script.decompose()
        text = soup.get_text(separator=' ', strip=True)
        return ' '.join(text.split())
    except Exception as e:
        print(f"  ❌ Échec extraction {url}: {e}")
        return ""

def run_ingest(persist_directory="./chroma_db"):
    if os.path.exists(persist_directory):
        print("✅ Base vectorielle déjà présente.")
        return

    print("🔍 Indexation des sources de confiance...")
    with open("trusted_sources.json", "r", encoding="utf-8") as f:
        sources = json.load(f)

    embedding_function = embedding_functions.SentenceTransformerEmbeddingFunction(
        model_name="all-MiniLM-L6-v2"
    )
    client = chromadb.PersistentClient(path=persist_directory)
    collection = client.get_or_create_collection(
        name="cyberwise",
        embedding_function=embedding_function
    )

    documents = []
    metadatas = []
    ids = []

    for i, src in enumerate(sources):
        print(f"  → {src['url']}")
        text = simple_extract_text(src['url'])
        if text and len(text) > 100:
            # Découper en chunks de 500 caractères
            chunks = [text[j:j+500] for j in range(0, len(text), 500)]
            for k, chunk in enumerate(chunks):
                documents.append(chunk)
                metadatas.append({"source": src['url']})
                ids.append(f"doc_{i}_{k}")
        time.sleep(1)

    if documents:
        collection.add(documents=documents, metadatas=metadatas, ids=ids)
        print(f"✅ {len(documents)} extraits indexés dans 'chroma_db'.")
    else:
        raise RuntimeError("Aucun texte valide n’a été extrait des sources.")