Nelly-43 commited on
Commit
7a17e0a
·
verified ·
1 Parent(s): 23f526e

Update src/ingest.py

Browse files
Files changed (1) hide show
  1. src/ingest.py +4 -4
src/ingest.py CHANGED
@@ -6,7 +6,7 @@ from pathlib import Path
6
 
7
  import chromadb
8
  from chromadb.utils import embedding_functions
9
- from sentence_transformers import SentenceTransformer
10
  from tqdm import tqdm
11
 
12
  from src.readers import read_file, READERS
@@ -266,9 +266,9 @@ def ingest_documents(cfg: dict = None, documents_dir: str = None) -> int:
266
  documents = [c.get("text", "") for c in batch]
267
  metadatas = [c.get("metadata", {}) for c in batch]
268
 
269
- emb_model = SentenceTransformer("all-MiniLM-L6-v2")
270
- # print("computing embeddings")
271
- precomputed_embeddings = emb_model.encode(documents, show_progress_bar=True).tolist()
272
 
273
  # print("adding docs")
274
  try:
 
6
 
7
  import chromadb
8
  from chromadb.utils import embedding_functions
9
+ # from sentence_transformers import SentenceTransformer
10
  from tqdm import tqdm
11
 
12
  from src.readers import read_file, READERS
 
266
  documents = [c.get("text", "") for c in batch]
267
  metadatas = [c.get("metadata", {}) for c in batch]
268
 
269
+ # emb_model = SentenceTransformer("all-MiniLM-L6-v2")
270
+ # # print("computing embeddings")
271
+ # precomputed_embeddings = emb_model.encode(documents, show_progress_bar=True).tolist()
272
 
273
  # print("adding docs")
274
  try: