Vdphacker commited on
Commit
97b9aa0
·
verified ·
1 Parent(s): fe40f3e

Update ingest.py

Browse files
Files changed (1) hide show
  1. ingest.py +63 -39
ingest.py CHANGED
@@ -1,39 +1,63 @@
1
- # ingest.py (version appelable en fonction)
2
- import json
3
- import os
4
- import time
5
- from langchain_community.document_loaders import WebBaseLoader
6
- from langchain_text_splitters import RecursiveCharacterTextSplitter
7
- from langchain_community.vectorstores import Chroma
8
- from langchain_community.embeddings import HuggingFaceEmbeddings
9
-
10
- def run_ingest(persist_directory: str = "./chroma_db"):
11
- # Charger les sources
12
- with open("trusted_sources.json", "r", encoding="utf-8") as f:
13
- sources = json.load(f)
14
-
15
- docs = []
16
- for src in sources:
17
- try:
18
- print(f"📄 Chargement : {src['url']}")
19
- loader = WebBaseLoader(src['url'])
20
- data = loader.load()
21
- for d in data:
22
- d.metadata["source"] = src['url']
23
- docs.extend(data)
24
- time.sleep(1)
25
- except Exception as e:
26
- print(f"❌ Erreur sur {src['url']}: {e}")
27
-
28
- if not docs:
29
- raise RuntimeError("Aucun document chargé. Vérifiez les URLs.")
30
-
31
- text_splitter = RecursiveCharacterTextSplitter(chunk_size=500, chunk_overlap=50)
32
- splits = text_splitter.split_documents(docs)
33
-
34
- embedding_model = HuggingFaceEmbeddings(model_name="sentence-transformers/all-MiniLM-L6-v2")
35
- Chroma.from_documents(
36
- documents=splits,
37
- embedding=embedding_model,
38
- persist_directory=persist_directory
39
- )
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # ingest.py
2
+ import json
3
+ import os
4
+ import time
5
+ import requests
6
+ from bs4 import BeautifulSoup
7
+ import chromadb
8
+ from chromadb.utils import embedding_functions
9
+
10
+ def simple_extract_text(url: str) -> str:
11
+ """Extrait le texte propre d'une page web."""
12
+ try:
13
+ headers = {'User-Agent': 'CyberWise Assistant (educational bot)'}
14
+ response = requests.get(url, headers=headers, timeout=15)
15
+ response.raise_for_status()
16
+ soup = BeautifulSoup(response.content, 'html.parser')
17
+ for script in soup(["script", "style", "nav", "footer", "header", "noscript"]):
18
+ script.decompose()
19
+ text = soup.get_text(separator=' ', strip=True)
20
+ return ' '.join(text.split())
21
+ except Exception as e:
22
+ print(f" ❌ Échec extraction {url}: {e}")
23
+ return ""
24
+
25
+ def run_ingest(persist_directory="./chroma_db"):
26
+ if os.path.exists(persist_directory):
27
+ print("✅ Base vectorielle déjà présente.")
28
+ return
29
+
30
+ print("🔍 Indexation des sources de confiance...")
31
+ with open("trusted_sources.json", "r", encoding="utf-8") as f:
32
+ sources = json.load(f)
33
+
34
+ embedding_function = embedding_functions.SentenceTransformerEmbeddingFunction(
35
+ model_name="all-MiniLM-L6-v2"
36
+ )
37
+ client = chromadb.PersistentClient(path=persist_directory)
38
+ collection = client.get_or_create_collection(
39
+ name="cyberwise",
40
+ embedding_function=embedding_function
41
+ )
42
+
43
+ documents = []
44
+ metadatas = []
45
+ ids = []
46
+
47
+ for i, src in enumerate(sources):
48
+ print(f" → {src['url']}")
49
+ text = simple_extract_text(src['url'])
50
+ if text and len(text) > 100:
51
+ # Découper en chunks de 500 caractères
52
+ chunks = [text[j:j+500] for j in range(0, len(text), 500)]
53
+ for k, chunk in enumerate(chunks):
54
+ documents.append(chunk)
55
+ metadatas.append({"source": src['url']})
56
+ ids.append(f"doc_{i}_{k}")
57
+ time.sleep(1)
58
+
59
+ if documents:
60
+ collection.add(documents=documents, metadatas=metadatas, ids=ids)
61
+ print(f"✅ {len(documents)} extraits indexés dans 'chroma_db'.")
62
+ else:
63
+ raise RuntimeError("Aucun texte valide n’a été extrait des sources.")