Spaces:
Sleeping
Sleeping
Update src/ingest.py
Browse files- src/ingest.py +4 -4
src/ingest.py
CHANGED
|
@@ -6,7 +6,7 @@ from pathlib import Path
|
|
| 6 |
|
| 7 |
import chromadb
|
| 8 |
from chromadb.utils import embedding_functions
|
| 9 |
-
from sentence_transformers import SentenceTransformer
|
| 10 |
from tqdm import tqdm
|
| 11 |
|
| 12 |
from src.readers import read_file, READERS
|
|
@@ -266,9 +266,9 @@ def ingest_documents(cfg: dict = None, documents_dir: str = None) -> int:
|
|
| 266 |
documents = [c.get("text", "") for c in batch]
|
| 267 |
metadatas = [c.get("metadata", {}) for c in batch]
|
| 268 |
|
| 269 |
-
emb_model = SentenceTransformer("all-MiniLM-L6-v2")
|
| 270 |
-
# print("computing embeddings")
|
| 271 |
-
precomputed_embeddings = emb_model.encode(documents, show_progress_bar=True).tolist()
|
| 272 |
|
| 273 |
# print("adding docs")
|
| 274 |
try:
|
|
|
|
| 6 |
|
| 7 |
import chromadb
|
| 8 |
from chromadb.utils import embedding_functions
|
| 9 |
+
# from sentence_transformers import SentenceTransformer
|
| 10 |
from tqdm import tqdm
|
| 11 |
|
| 12 |
from src.readers import read_file, READERS
|
|
|
|
| 266 |
documents = [c.get("text", "") for c in batch]
|
| 267 |
metadatas = [c.get("metadata", {}) for c in batch]
|
| 268 |
|
| 269 |
+
# emb_model = SentenceTransformer("all-MiniLM-L6-v2")
|
| 270 |
+
# # print("computing embeddings")
|
| 271 |
+
# precomputed_embeddings = emb_model.encode(documents, show_progress_bar=True).tolist()
|
| 272 |
|
| 273 |
# print("adding docs")
|
| 274 |
try:
|