Charlotte-Works-AI / utils.py
asareen27's picture
fix bm25
c60314b
Raw History Blame Contribute Delete
10.4 kB
# -*- coding: utf-8 -*-
"""main.ipynb
Automatically generated by Colab.
Original file is located at
https://colab.research.google.com/drive/1Af557ozVRks1onzjmGX5c7t2LbnGmVHe
# Hello there, and welcome to the Charlotte Works AI Assistant!
Thank you so much for deciding to use the Charlotte Works AI Asssitant!
### **First time here?** Watch the YouTube Video to get a step-by-step guide.
To get started, just run all of the cells within this Notebook. You can do that by looking at the menu bar and the top and clicking on Runtime → Run all, pressing Ctrl + F9, or clicking on the little ▶️ icon next to every cell.
Depending on Wi-Fi speeds, the entire process may take 5-10 minutes to complete.
If you have any questions or feedback, please don't hesitate to reach out to [Arnav Sareen](mailto:asareen@charlotteworks.com). Thank you again!
"""
import os
import dspy
import chromadb
from dspy.retrieve.chromadb_rm import ChromadbRM
import random as rand
import json
from inspect import getsource
import pymupdf
import numpy as np
from tokenizers import Tokenizer
from semantic_text_splitter import TextSplitter
from chromadb.api.models import Collection
import chromadb.utils.embedding_functions as embedding_functions
from re import sub as re_sub
from rank_bm25 import BM25Okapi
from os import listdir
from os.path import isfile, join
# from FlagEmbedding import FlagReranker
from cohere import Client as cohere_client
from extract_text import extract_pdf_to_text
from typing import List
MAX_TOKENS = 1000
TOKENIZER = Tokenizer.from_pretrained("bert-base-uncased")
SPLITTER = TextSplitter.from_huggingface_tokenizer(TOKENIZER, MAX_TOKENS)
SENTENCE_TRANSFORMER = embedding_functions.SentenceTransformerEmbeddingFunction(model_name="BAAI/bge-small-en-v1.5", trust_remote_code=True)
class ZeroShot(dspy.Module):
"""
Provide answer to question
"""
def __init__(self):
super().__init__()
self.prog = dspy.Predict("question -> answer")
def forward(self, question):
return self.prog(question="In the video game Watch Dogs 2, " + question)
# Use PyMuPDF to extract text from a document
def extract_text_from_document(document_path: str) -> str:
pdf_document = pymupdf.open(document_path)
text = ""
for page in pdf_document:
text += page.get_text()
return text
def get_files(path: str) -> dict:
filenames = [f for f in listdir(path) if isfile(join(path, f))]
document_paths = [f'{path}{filename}' for filename in filenames]
files = {}
for file_name, file_path in zip(filenames, document_paths):
print(f'File Path: {file_path}')
print(f'File Name: {file_name}')
if file_name.endswith(".json"):
with open(file_path, encoding="utf8") as f:
data = json.load(f)
files[file_name] = data
elif file_name.endswith('.pdf'):
# If it ends in PDF, first check to make sure a next file of the same does not exist:
if os.path.exists(f"{path}/{file_name.split('.pdf')[0]}.txt"):
pass
else:
# If it doesn't exist, we can extract the text
text = extract_pdf_to_text(files=[file_path])
text = re_sub(r"Page \d+ of \d+", "", text) #Remove the "Page _ of _ text"
text = re_sub(r"Attachment \d+", "", text) # Remove the "Attachment _"
text = text.replace("\n", " ").strip() # Remove newlines + white space
#Save the extracted text to a text file so we can save on parsing operations
with open(f"{path}/{file_name.split('.pdf')[0]}.txt", mode='w', encoding="utf8") as f:
f.write(text)
#Add to the files array
files[file_name] = text
elif file_name.endswith(".txt"):
# Ensure we are not double-adding files if we already added them to the array
if files.get(f'{file_name.split(".txt")[0]}.pdf'):
continue
with open(file_path, mode='r', encoding='utf8') as f:
data = f.read()
files[file_name] = data
else:
continue
return files
# See if a document is already present within the ChromaDB collection
def validate_document(collection: Collection, document_id: str) -> bool:
return len(collection.get(document_id)["ids"]) > 0
# Verify and embed a document into the ChromaDB collection
def embed_document(collection: Collection, document_ids: list, text_list: list) -> None:
unique_documents = []
unique_ids = []
for document, id in zip(text_list,document_ids):
if not validate_document(collection, id):
unique_documents.append(document)
unique_ids.append(id)
if len(unique_documents) != 0 and len(unique_ids) != 0:
collection.add(documents=unique_documents, ids=unique_ids)
# Custom retriever for BM25
class BM25(dspy.Retrieve):
def __init__(self, texts: List[str], k:int = 3):
super().__init__(k=k)
self.tokenizer = Tokenizer.from_pretrained("bert-base-uncased")
self.bm25_splitter = TextSplitter.from_huggingface_tokenizer(self.tokenizer, 200)
self.corpus = np.empty(0)
for doc in texts:
if len(doc) > 200:
# If we have a rather large document, we would want to tokenize first into more digestable chunks
self.corpus = np.append(self.corpus, self.bm25_splitter.chunks(doc))
else:
self.corpus = np.append(self.corpus, doc)
split_text = [doc.lower().split(" ") for doc in self.corpus]
self.bm25 = BM25Okapi(split_text)
def forward(self, query:str) -> dspy.Prediction:
tokenized_query = query.lower().split(" ")
response = self.bm25.get_top_n(tokenized_query, self.corpus, n=self.k)
return dspy.Prediction(
passages=response
)
def chunk_documents(raw_files: dict, splitter: TextSplitter) -> dict:
files = {}
for file_name, file_text in raw_files.items():
if file_name.endswith('.pdf') or file_name.endswith('.txt'):
content = splitter.chunks(file_text)
elif file_name.endswith('.json'):
content = [question["context"] for question in file_text]
new_file_name = file_name.split(".")[0]
files[new_file_name] = content
return files
def chroma_embed(collection: Collection, documents: dict) -> None:
for document_name, document in documents.items():
embed_document(collection, document_ids=[f'{document_name}_{i}' for i in range(len(document))], text_list=document)
print(f"Successfully added {document_name} to ChromaDB collection")
def init_chromadb():
# Create the embedding function, using the all-mpnet-base-v2 model
# sentence_transformer_ef = embedding_functions.SentenceTransformerEmbeddingFunction(model_name="all-mpnet-base-v2")
# Create chromadb client and intialize a context collection
chroma_client = chromadb.PersistentClient(
path='chroma/'
)
collection = chroma_client.get_or_create_collection(name="documents", metadata={"hnsw:space": "cosine"}, embedding_function=SENTENCE_TRANSFORMER)
max_tokens = 1000
all_files = get_files("./data/")
extracted_documents = chunk_documents(all_files, SPLITTER)
chroma_embed(collection, documents=extracted_documents)
def init_bm25():
all_files = get_files("./data/")
extracted_documents = chunk_documents(all_files, SPLITTER)
bm25 = BM25(extracted_documents.values())
return bm25
#Create a training and test dataset
# qca_dataset = []
# for qca in all_files['dataset.json']:
# # Tell DSPy that the 'question' field is the input. Any other fields are labels and/or metadata.
# qca_dataset.append(dspy.Example(question=qca["question"], answer=qca["answer"], golden_context=qca["context"]).with_inputs("question"))
# rand.seed(2024)
# rand.shuffle(qca_dataset)
# train_set = qca_dataset[: int(0.7 * len(qca_dataset))]
# test_set = qca_dataset[int(0.7 * len(qca_dataset)) :]
# len(train_set), len(test_set)
class HyDESignature(dspy.Signature):
"""Given the provided question, generate a short passage with keywords"""
question = dspy.InputField()
passage = dspy.OutputField()
# TODO: We probably want HyDE to have some sort of conversation history
class HyDE(dspy.Module):
def __init__(self):
super().__init__()
self.generate_answer = dspy.ChainOfThought(HyDESignature)
def forward(self, question):
response = self.generate_answer(question=question)
return dspy.Prediction(rationale=response.rationale, passage=response.passage)
class GenerateAnswer(dspy.Signature):
"""You are a helpful AI assistant that answers questions given the provided context and your own knowledge"""
context = dspy.InputField(desc="may contain relevant facts")
question = dspy.InputField()
answer = dspy.OutputField()
#TODO: We need to think about the best instances to use HyDE and not; Hybrid Search feels like the best route to go here...
from time import sleep
class RAG(dspy.Module):
def __init__(self, bm25_retriever, cohere):
super().__init__()
self.generate_answer = dspy.ChainOfThought(GenerateAnswer)
self.bm25 = bm25_retriever
self.cohere = cohere
self.retrieve = ChromadbRM("documents", "./chroma", k=5, embedding_function=SENTENCE_TRANSFORMER)
# The forward method is what actually sets up and sends the query to the LLM
def forward(self, question):
hyDE_retrieval = HyDE().forward(question)
print(f"HyDE: {hyDE_retrieval.passage}")
chroma_context = [item["long_text"] for item in self.retrieve(hyDE_retrieval.passage)]
print(f"ChromaDB Retrieval: {chroma_context}")
bm25_context = self.bm25.forward(hyDE_retrieval.passage).passages
print(f"BM25 Retreival: {bm25_context}")
all_context = list(set(chroma_context + bm25_context))
cohere_context = self.cohere.rerank(model='rerank-english-v3.0', query=question, documents=all_context, top_n=5)
print(f"Cohere Reranker: {cohere_context}")
reranked_indexes = [item.index for item in cohere_context.results]
reranked_context = [all_context[i] for i in reranked_indexes]
# sleep(.1)
# context = [item["long_text"] for item in self.retrieve(question)]
prediction = self.generate_answer(context=reranked_context, question=question)
print(prediction)
return dspy.Prediction(context=reranked_context, rationale=prediction.rationale, answer=prediction.answer)
#