YOUR_AI_CHAT / app.py
risper7's picture
Update app.py
932f97c verified
Raw History Blame Contribute Delete
3.79 kB
import os
from torch import cuda, bfloat16
import torch
import transformers
from transformers import AutoTokenizer, AutoModelForCausalLM
from langchain_huggingface import HuggingFaceEmbeddings, HuggingFacePipeline
from langchain.chains import RetrievalQA
from langchain.prompts import PromptTemplate
from langchain_chroma import Chroma
from huggingface_hub import login
from accelerate import disk_offload
import chromadb
import gradio as gr
from transformers import pipeline
# Login to Hugging Face Hub
hf_token = os.getenv("HF_TOKEN")
login(token=hf_token)
# Model configuration and pipeline setup
model_id='google/gemma-1.1-2b-it'
device = f'cuda:{cuda.current_device()}' if cuda.is_available() else 'cpu'
bnb_config = transformers.BitsAndBytesConfig(
load_in_4bit=True,
bnb_4bit_quant_type='nf4',
bnb_4bit_use_double_quant=True,
bnb_4bit_compute_dtype=bfloat16
)
model_config = transformers.AutoConfig.from_pretrained(
model_id,
trust_remote_code=True,
max_new_tokens=1024,
)
model = AutoModelForCausalLM.from_pretrained(
model_id,
trust_remote_code=True,
config=model_config,
#quantization_config=bnb_config,
device_map='auto',
token=hf_token
)
#model.to('cpu')
offload_dir = './disk_offload'
model = disk_offload(model, offload_dir)
tokenizer = AutoTokenizer.from_pretrained(model_id, token=hf_token)
query_pipeline = pipeline(
"text-generation",
model=model,
tokenizer=tokenizer,
torch_dtype=torch.float16 if device == 'cuda' else torch.float32,
device_map='auto',
max_length=6000,
max_new_tokens=500,
do_sample=True,
temperature = 0.7,
top_k=50,
top_p=0.85,
repetition_penalty=1.2
)
# Define the embedding function using Hugging Face embeddings
embedding_function = HuggingFaceEmbeddings(model_name="sentence-transformers/all-MiniLM-L6-v2")
# Set up the vector store with ChromaDB
client = chromadb.PersistentClient(path="risper7/YOUR_AI_CHAT/vectordata1")
vector_store = Chroma(
client=client,
collection_name="AI_data",
embedding_function=embedding_function
)
# Set up retriever from the vector store
retriever = vector_store.as_retriever(
search_type="mmr",
search_kwargs={"k": 3, "fetch_k": 10}
)
# Define prompt template
template = """
You are an AI and Machine Learning expert. Analyze the given question and respond with a concise, accurate answer based strictly on the provided context.
If the context does not contain enough information, answer to the best of your knowledge while maintaining clarity and correctness.
Don't add any additional details that are not related to the question and context.
Question:
{question}
Context:
{context}
Answer:
"""
PROMPT = PromptTemplate(input_variables=["context", "query"], template=template)
# Set up LLM for the QA chain
llm = HuggingFacePipeline(pipeline=query_pipeline)
# Create the RetrievalQA chain
qa_chain = RetrievalQA.from_chain_type(
llm, retriever=retriever, chain_type_kwargs={"prompt": PROMPT}
)
# Function to get the answer from the QA chain
def get_answer(query):
result = qa_chain.invoke({"query": query})
answer = result['result'].split("Answer:")[1].strip()
return answer
def chat_function(user_input, history):
if history is None:
history = []
query = user_input
response = get_answer(query)
history.append((user_input, response))
return history, history
iface = gr.Interface(
fn=chat_function,
inputs=[
gr.Textbox(placeholder="Ask me anything about AI...", label="Enter your question here"),
gr.State()
],
outputs=[
gr.Chatbot(height=300),
gr.State()
],
title="Your AI Chat"
)
# Launch the Gradio interface
if __name__ == "__main__":
iface.launch(share=True,debug=True)