# -*- coding: utf-8 -*- """main.ipynb Automatically generated by Colab. Original file is located at https://colab.research.google.com/drive/1Af557ozVRks1onzjmGX5c7t2LbnGmVHe # Hello there, and welcome to the Charlotte Works AI Assistant! Thank you so much for deciding to use the Charlotte Works AI Asssitant! ### **First time here?** Watch the YouTube Video to get a step-by-step guide. To get started, just run all of the cells within this Notebook. You can do that by looking at the menu bar and the top and clicking on Runtime → Run all, pressing Ctrl + F9, or clicking on the little ▶️ icon next to every cell. Depending on Wi-Fi speeds, the entire process may take 5-10 minutes to complete. If you have any questions or feedback, please don't hesitate to reach out to [Arnav Sareen](mailto:asareen@charlotteworks.com). Thank you again! """ import os import dspy import chromadb from dspy.retrieve.chromadb_rm import ChromadbRM import random as rand import json from inspect import getsource import pymupdf import numpy as np from tokenizers import Tokenizer from semantic_text_splitter import TextSplitter from chromadb.api.models import Collection import chromadb.utils.embedding_functions as embedding_functions from re import sub as re_sub from rank_bm25 import BM25Okapi from os import listdir from os.path import isfile, join # from FlagEmbedding import FlagReranker from cohere import Client as cohere_client from extract_text import extract_pdf_to_text from typing import List MAX_TOKENS = 1000 TOKENIZER = Tokenizer.from_pretrained("bert-base-uncased") SPLITTER = TextSplitter.from_huggingface_tokenizer(TOKENIZER, MAX_TOKENS) SENTENCE_TRANSFORMER = embedding_functions.SentenceTransformerEmbeddingFunction(model_name="BAAI/bge-small-en-v1.5", trust_remote_code=True) class ZeroShot(dspy.Module): """ Provide answer to question """ def __init__(self): super().__init__() self.prog = dspy.Predict("question -> answer") def forward(self, question): return self.prog(question="In the video game Watch Dogs 2, " + question) # Use PyMuPDF to extract text from a document def extract_text_from_document(document_path: str) -> str: pdf_document = pymupdf.open(document_path) text = "" for page in pdf_document: text += page.get_text() return text def get_files(path: str) -> dict: filenames = [f for f in listdir(path) if isfile(join(path, f))] document_paths = [f'{path}{filename}' for filename in filenames] files = {} for file_name, file_path in zip(filenames, document_paths): print(f'File Path: {file_path}') print(f'File Name: {file_name}') if file_name.endswith(".json"): with open(file_path, encoding="utf8") as f: data = json.load(f) files[file_name] = data elif file_name.endswith('.pdf'): # If it ends in PDF, first check to make sure a next file of the same does not exist: if os.path.exists(f"{path}/{file_name.split('.pdf')[0]}.txt"): pass else: # If it doesn't exist, we can extract the text text = extract_pdf_to_text(files=[file_path]) text = re_sub(r"Page \d+ of \d+", "", text) #Remove the "Page _ of _ text" text = re_sub(r"Attachment \d+", "", text) # Remove the "Attachment _" text = text.replace("\n", " ").strip() # Remove newlines + white space #Save the extracted text to a text file so we can save on parsing operations with open(f"{path}/{file_name.split('.pdf')[0]}.txt", mode='w', encoding="utf8") as f: f.write(text) #Add to the files array files[file_name] = text elif file_name.endswith(".txt"): # Ensure we are not double-adding files if we already added them to the array if files.get(f'{file_name.split(".txt")[0]}.pdf'): continue with open(file_path, mode='r', encoding='utf8') as f: data = f.read() files[file_name] = data else: continue return files # See if a document is already present within the ChromaDB collection def validate_document(collection: Collection, document_id: str) -> bool: return len(collection.get(document_id)["ids"]) > 0 # Verify and embed a document into the ChromaDB collection def embed_document(collection: Collection, document_ids: list, text_list: list) -> None: unique_documents = [] unique_ids = [] for document, id in zip(text_list,document_ids): if not validate_document(collection, id): unique_documents.append(document) unique_ids.append(id) if len(unique_documents) != 0 and len(unique_ids) != 0: collection.add(documents=unique_documents, ids=unique_ids) # Custom retriever for BM25 class BM25(dspy.Retrieve): def __init__(self, texts: List[str], k:int = 3): super().__init__(k=k) self.tokenizer = Tokenizer.from_pretrained("bert-base-uncased") self.bm25_splitter = TextSplitter.from_huggingface_tokenizer(self.tokenizer, 200) self.corpus = np.empty(0) for doc in texts: if len(doc) > 200: # If we have a rather large document, we would want to tokenize first into more digestable chunks self.corpus = np.append(self.corpus, self.bm25_splitter.chunks(doc)) else: self.corpus = np.append(self.corpus, doc) split_text = [doc.lower().split(" ") for doc in self.corpus] self.bm25 = BM25Okapi(split_text) def forward(self, query:str) -> dspy.Prediction: tokenized_query = query.lower().split(" ") response = self.bm25.get_top_n(tokenized_query, self.corpus, n=self.k) return dspy.Prediction( passages=response ) def chunk_documents(raw_files: dict, splitter: TextSplitter) -> dict: files = {} for file_name, file_text in raw_files.items(): if file_name.endswith('.pdf') or file_name.endswith('.txt'): content = splitter.chunks(file_text) elif file_name.endswith('.json'): content = [question["context"] for question in file_text] new_file_name = file_name.split(".")[0] files[new_file_name] = content return files def chroma_embed(collection: Collection, documents: dict) -> None: for document_name, document in documents.items(): embed_document(collection, document_ids=[f'{document_name}_{i}' for i in range(len(document))], text_list=document) print(f"Successfully added {document_name} to ChromaDB collection") def init_chromadb(): # Create the embedding function, using the all-mpnet-base-v2 model # sentence_transformer_ef = embedding_functions.SentenceTransformerEmbeddingFunction(model_name="all-mpnet-base-v2") # Create chromadb client and intialize a context collection chroma_client = chromadb.PersistentClient( path='chroma/' ) collection = chroma_client.get_or_create_collection(name="documents", metadata={"hnsw:space": "cosine"}, embedding_function=SENTENCE_TRANSFORMER) max_tokens = 1000 all_files = get_files("./data/") extracted_documents = chunk_documents(all_files, SPLITTER) chroma_embed(collection, documents=extracted_documents) def init_bm25(): all_files = get_files("./data/") extracted_documents = chunk_documents(all_files, SPLITTER) bm25 = BM25(extracted_documents.values()) return bm25 #Create a training and test dataset # qca_dataset = [] # for qca in all_files['dataset.json']: # # Tell DSPy that the 'question' field is the input. Any other fields are labels and/or metadata. # qca_dataset.append(dspy.Example(question=qca["question"], answer=qca["answer"], golden_context=qca["context"]).with_inputs("question")) # rand.seed(2024) # rand.shuffle(qca_dataset) # train_set = qca_dataset[: int(0.7 * len(qca_dataset))] # test_set = qca_dataset[int(0.7 * len(qca_dataset)) :] # len(train_set), len(test_set) class HyDESignature(dspy.Signature): """Given the provided question, generate a short passage with keywords""" question = dspy.InputField() passage = dspy.OutputField() # TODO: We probably want HyDE to have some sort of conversation history class HyDE(dspy.Module): def __init__(self): super().__init__() self.generate_answer = dspy.ChainOfThought(HyDESignature) def forward(self, question): response = self.generate_answer(question=question) return dspy.Prediction(rationale=response.rationale, passage=response.passage) class GenerateAnswer(dspy.Signature): """You are a helpful AI assistant that answers questions given the provided context and your own knowledge""" context = dspy.InputField(desc="may contain relevant facts") question = dspy.InputField() answer = dspy.OutputField() #TODO: We need to think about the best instances to use HyDE and not; Hybrid Search feels like the best route to go here... from time import sleep class RAG(dspy.Module): def __init__(self, bm25_retriever, cohere): super().__init__() self.generate_answer = dspy.ChainOfThought(GenerateAnswer) self.bm25 = bm25_retriever self.cohere = cohere self.retrieve = ChromadbRM("documents", "./chroma", k=5, embedding_function=SENTENCE_TRANSFORMER) # The forward method is what actually sets up and sends the query to the LLM def forward(self, question): hyDE_retrieval = HyDE().forward(question) print(f"HyDE: {hyDE_retrieval.passage}") chroma_context = [item["long_text"] for item in self.retrieve(hyDE_retrieval.passage)] print(f"ChromaDB Retrieval: {chroma_context}") bm25_context = self.bm25.forward(hyDE_retrieval.passage).passages print(f"BM25 Retreival: {bm25_context}") all_context = list(set(chroma_context + bm25_context)) cohere_context = self.cohere.rerank(model='rerank-english-v3.0', query=question, documents=all_context, top_n=5) print(f"Cohere Reranker: {cohere_context}") reranked_indexes = [item.index for item in cohere_context.results] reranked_context = [all_context[i] for i in reranked_indexes] # sleep(.1) # context = [item["long_text"] for item in self.retrieve(question)] prediction = self.generate_answer(context=reranked_context, question=question) print(prediction) return dspy.Prediction(context=reranked_context, rationale=prediction.rationale, answer=prediction.answer) #