Spaces:
Build error
Build error
Download utils.py from aesareen/Charlotte-Works-AI: direct link, hf CLI and curl.
- Browser
- Download file 10.4 kB
-
https://huggingface.co/spaces/aesareen/Charlotte-Works-AI/resolve/main/utils.py
- Command line
-
hf download hf://spaces/aesareen/Charlotte-Works-AI/utils.py
-
curl -L -o utils.py https://huggingface.co/spaces/aesareen/Charlotte-Works-AI/resolve/main/utils.py
10.4 kB
| # -*- coding: utf-8 -*- | |
| """main.ipynb | |
| Automatically generated by Colab. | |
| Original file is located at | |
| https://colab.research.google.com/drive/1Af557ozVRks1onzjmGX5c7t2LbnGmVHe | |
| # Hello there, and welcome to the Charlotte Works AI Assistant! | |
| Thank you so much for deciding to use the Charlotte Works AI Asssitant! | |
| ### **First time here?** Watch the YouTube Video to get a step-by-step guide. | |
| To get started, just run all of the cells within this Notebook. You can do that by looking at the menu bar and the top and clicking on Runtime → Run all, pressing Ctrl + F9, or clicking on the little ▶️ icon next to every cell. | |
| Depending on Wi-Fi speeds, the entire process may take 5-10 minutes to complete. | |
| If you have any questions or feedback, please don't hesitate to reach out to [Arnav Sareen](mailto:asareen@charlotteworks.com). Thank you again! | |
| """ | |
| import os | |
| import dspy | |
| import chromadb | |
| from dspy.retrieve.chromadb_rm import ChromadbRM | |
| import random as rand | |
| import json | |
| from inspect import getsource | |
| import pymupdf | |
| import numpy as np | |
| from tokenizers import Tokenizer | |
| from semantic_text_splitter import TextSplitter | |
| from chromadb.api.models import Collection | |
| import chromadb.utils.embedding_functions as embedding_functions | |
| from re import sub as re_sub | |
| from rank_bm25 import BM25Okapi | |
| from os import listdir | |
| from os.path import isfile, join | |
| # from FlagEmbedding import FlagReranker | |
| from cohere import Client as cohere_client | |
| from extract_text import extract_pdf_to_text | |
| from typing import List | |
| MAX_TOKENS = 1000 | |
| TOKENIZER = Tokenizer.from_pretrained("bert-base-uncased") | |
| SPLITTER = TextSplitter.from_huggingface_tokenizer(TOKENIZER, MAX_TOKENS) | |
| SENTENCE_TRANSFORMER = embedding_functions.SentenceTransformerEmbeddingFunction(model_name="BAAI/bge-small-en-v1.5", trust_remote_code=True) | |
| class ZeroShot(dspy.Module): | |
| """ | |
| Provide answer to question | |
| """ | |
| def __init__(self): | |
| super().__init__() | |
| self.prog = dspy.Predict("question -> answer") | |
| def forward(self, question): | |
| return self.prog(question="In the video game Watch Dogs 2, " + question) | |
| # Use PyMuPDF to extract text from a document | |
| def extract_text_from_document(document_path: str) -> str: | |
| pdf_document = pymupdf.open(document_path) | |
| text = "" | |
| for page in pdf_document: | |
| text += page.get_text() | |
| return text | |
| def get_files(path: str) -> dict: | |
| filenames = [f for f in listdir(path) if isfile(join(path, f))] | |
| document_paths = [f'{path}{filename}' for filename in filenames] | |
| files = {} | |
| for file_name, file_path in zip(filenames, document_paths): | |
| print(f'File Path: {file_path}') | |
| print(f'File Name: {file_name}') | |
| if file_name.endswith(".json"): | |
| with open(file_path, encoding="utf8") as f: | |
| data = json.load(f) | |
| files[file_name] = data | |
| elif file_name.endswith('.pdf'): | |
| # If it ends in PDF, first check to make sure a next file of the same does not exist: | |
| if os.path.exists(f"{path}/{file_name.split('.pdf')[0]}.txt"): | |
| pass | |
| else: | |
| # If it doesn't exist, we can extract the text | |
| text = extract_pdf_to_text(files=[file_path]) | |
| text = re_sub(r"Page \d+ of \d+", "", text) #Remove the "Page _ of _ text" | |
| text = re_sub(r"Attachment \d+", "", text) # Remove the "Attachment _" | |
| text = text.replace("\n", " ").strip() # Remove newlines + white space | |
| #Save the extracted text to a text file so we can save on parsing operations | |
| with open(f"{path}/{file_name.split('.pdf')[0]}.txt", mode='w', encoding="utf8") as f: | |
| f.write(text) | |
| #Add to the files array | |
| files[file_name] = text | |
| elif file_name.endswith(".txt"): | |
| # Ensure we are not double-adding files if we already added them to the array | |
| if files.get(f'{file_name.split(".txt")[0]}.pdf'): | |
| continue | |
| with open(file_path, mode='r', encoding='utf8') as f: | |
| data = f.read() | |
| files[file_name] = data | |
| else: | |
| continue | |
| return files | |
| # See if a document is already present within the ChromaDB collection | |
| def validate_document(collection: Collection, document_id: str) -> bool: | |
| return len(collection.get(document_id)["ids"]) > 0 | |
| # Verify and embed a document into the ChromaDB collection | |
| def embed_document(collection: Collection, document_ids: list, text_list: list) -> None: | |
| unique_documents = [] | |
| unique_ids = [] | |
| for document, id in zip(text_list,document_ids): | |
| if not validate_document(collection, id): | |
| unique_documents.append(document) | |
| unique_ids.append(id) | |
| if len(unique_documents) != 0 and len(unique_ids) != 0: | |
| collection.add(documents=unique_documents, ids=unique_ids) | |
| # Custom retriever for BM25 | |
| class BM25(dspy.Retrieve): | |
| def __init__(self, texts: List[str], k:int = 3): | |
| super().__init__(k=k) | |
| self.tokenizer = Tokenizer.from_pretrained("bert-base-uncased") | |
| self.bm25_splitter = TextSplitter.from_huggingface_tokenizer(self.tokenizer, 200) | |
| self.corpus = np.empty(0) | |
| for doc in texts: | |
| if len(doc) > 200: | |
| # If we have a rather large document, we would want to tokenize first into more digestable chunks | |
| self.corpus = np.append(self.corpus, self.bm25_splitter.chunks(doc)) | |
| else: | |
| self.corpus = np.append(self.corpus, doc) | |
| split_text = [doc.lower().split(" ") for doc in self.corpus] | |
| self.bm25 = BM25Okapi(split_text) | |
| def forward(self, query:str) -> dspy.Prediction: | |
| tokenized_query = query.lower().split(" ") | |
| response = self.bm25.get_top_n(tokenized_query, self.corpus, n=self.k) | |
| return dspy.Prediction( | |
| passages=response | |
| ) | |
| def chunk_documents(raw_files: dict, splitter: TextSplitter) -> dict: | |
| files = {} | |
| for file_name, file_text in raw_files.items(): | |
| if file_name.endswith('.pdf') or file_name.endswith('.txt'): | |
| content = splitter.chunks(file_text) | |
| elif file_name.endswith('.json'): | |
| content = [question["context"] for question in file_text] | |
| new_file_name = file_name.split(".")[0] | |
| files[new_file_name] = content | |
| return files | |
| def chroma_embed(collection: Collection, documents: dict) -> None: | |
| for document_name, document in documents.items(): | |
| embed_document(collection, document_ids=[f'{document_name}_{i}' for i in range(len(document))], text_list=document) | |
| print(f"Successfully added {document_name} to ChromaDB collection") | |
| def init_chromadb(): | |
| # Create the embedding function, using the all-mpnet-base-v2 model | |
| # sentence_transformer_ef = embedding_functions.SentenceTransformerEmbeddingFunction(model_name="all-mpnet-base-v2") | |
| # Create chromadb client and intialize a context collection | |
| chroma_client = chromadb.PersistentClient( | |
| path='chroma/' | |
| ) | |
| collection = chroma_client.get_or_create_collection(name="documents", metadata={"hnsw:space": "cosine"}, embedding_function=SENTENCE_TRANSFORMER) | |
| max_tokens = 1000 | |
| all_files = get_files("./data/") | |
| extracted_documents = chunk_documents(all_files, SPLITTER) | |
| chroma_embed(collection, documents=extracted_documents) | |
| def init_bm25(): | |
| all_files = get_files("./data/") | |
| extracted_documents = chunk_documents(all_files, SPLITTER) | |
| bm25 = BM25(extracted_documents.values()) | |
| return bm25 | |
| #Create a training and test dataset | |
| # qca_dataset = [] | |
| # for qca in all_files['dataset.json']: | |
| # # Tell DSPy that the 'question' field is the input. Any other fields are labels and/or metadata. | |
| # qca_dataset.append(dspy.Example(question=qca["question"], answer=qca["answer"], golden_context=qca["context"]).with_inputs("question")) | |
| # rand.seed(2024) | |
| # rand.shuffle(qca_dataset) | |
| # train_set = qca_dataset[: int(0.7 * len(qca_dataset))] | |
| # test_set = qca_dataset[int(0.7 * len(qca_dataset)) :] | |
| # len(train_set), len(test_set) | |
| class HyDESignature(dspy.Signature): | |
| """Given the provided question, generate a short passage with keywords""" | |
| question = dspy.InputField() | |
| passage = dspy.OutputField() | |
| # TODO: We probably want HyDE to have some sort of conversation history | |
| class HyDE(dspy.Module): | |
| def __init__(self): | |
| super().__init__() | |
| self.generate_answer = dspy.ChainOfThought(HyDESignature) | |
| def forward(self, question): | |
| response = self.generate_answer(question=question) | |
| return dspy.Prediction(rationale=response.rationale, passage=response.passage) | |
| class GenerateAnswer(dspy.Signature): | |
| """You are a helpful AI assistant that answers questions given the provided context and your own knowledge""" | |
| context = dspy.InputField(desc="may contain relevant facts") | |
| question = dspy.InputField() | |
| answer = dspy.OutputField() | |
| #TODO: We need to think about the best instances to use HyDE and not; Hybrid Search feels like the best route to go here... | |
| from time import sleep | |
| class RAG(dspy.Module): | |
| def __init__(self, bm25_retriever, cohere): | |
| super().__init__() | |
| self.generate_answer = dspy.ChainOfThought(GenerateAnswer) | |
| self.bm25 = bm25_retriever | |
| self.cohere = cohere | |
| self.retrieve = ChromadbRM("documents", "./chroma", k=5, embedding_function=SENTENCE_TRANSFORMER) | |
| # The forward method is what actually sets up and sends the query to the LLM | |
| def forward(self, question): | |
| hyDE_retrieval = HyDE().forward(question) | |
| print(f"HyDE: {hyDE_retrieval.passage}") | |
| chroma_context = [item["long_text"] for item in self.retrieve(hyDE_retrieval.passage)] | |
| print(f"ChromaDB Retrieval: {chroma_context}") | |
| bm25_context = self.bm25.forward(hyDE_retrieval.passage).passages | |
| print(f"BM25 Retreival: {bm25_context}") | |
| all_context = list(set(chroma_context + bm25_context)) | |
| cohere_context = self.cohere.rerank(model='rerank-english-v3.0', query=question, documents=all_context, top_n=5) | |
| print(f"Cohere Reranker: {cohere_context}") | |
| reranked_indexes = [item.index for item in cohere_context.results] | |
| reranked_context = [all_context[i] for i in reranked_indexes] | |
| # sleep(.1) | |
| # context = [item["long_text"] for item in self.retrieve(question)] | |
| prediction = self.generate_answer(context=reranked_context, question=question) | |
| print(prediction) | |
| return dspy.Prediction(context=reranked_context, rationale=prediction.rationale, answer=prediction.answer) | |
| # | |