import os from torch import cuda, bfloat16 import torch import transformers from transformers import AutoTokenizer, AutoModelForCausalLM from langchain_huggingface import HuggingFaceEmbeddings, HuggingFacePipeline from langchain.chains import RetrievalQA from langchain.prompts import PromptTemplate from langchain_chroma import Chroma from huggingface_hub import login from accelerate import disk_offload import chromadb import gradio as gr from transformers import pipeline # Login to Hugging Face Hub hf_token = os.getenv("HF_TOKEN") login(token=hf_token) # Model configuration and pipeline setup model_id='google/gemma-1.1-2b-it' device = f'cuda:{cuda.current_device()}' if cuda.is_available() else 'cpu' bnb_config = transformers.BitsAndBytesConfig( load_in_4bit=True, bnb_4bit_quant_type='nf4', bnb_4bit_use_double_quant=True, bnb_4bit_compute_dtype=bfloat16 ) model_config = transformers.AutoConfig.from_pretrained( model_id, trust_remote_code=True, max_new_tokens=1024, ) model = AutoModelForCausalLM.from_pretrained( model_id, trust_remote_code=True, config=model_config, #quantization_config=bnb_config, device_map='auto', token=hf_token ) #model.to('cpu') offload_dir = './disk_offload' model = disk_offload(model, offload_dir) tokenizer = AutoTokenizer.from_pretrained(model_id, token=hf_token) query_pipeline = pipeline( "text-generation", model=model, tokenizer=tokenizer, torch_dtype=torch.float16 if device == 'cuda' else torch.float32, device_map='auto', max_length=6000, max_new_tokens=500, do_sample=True, temperature = 0.7, top_k=50, top_p=0.85, repetition_penalty=1.2 ) # Define the embedding function using Hugging Face embeddings embedding_function = HuggingFaceEmbeddings(model_name="sentence-transformers/all-MiniLM-L6-v2") # Set up the vector store with ChromaDB client = chromadb.PersistentClient(path="risper7/YOUR_AI_CHAT/vectordata1") vector_store = Chroma( client=client, collection_name="AI_data", embedding_function=embedding_function ) # Set up retriever from the vector store retriever = vector_store.as_retriever( search_type="mmr", search_kwargs={"k": 3, "fetch_k": 10} ) # Define prompt template template = """ You are an AI and Machine Learning expert. Analyze the given question and respond with a concise, accurate answer based strictly on the provided context. If the context does not contain enough information, answer to the best of your knowledge while maintaining clarity and correctness. Don't add any additional details that are not related to the question and context. Question: {question} Context: {context} Answer: """ PROMPT = PromptTemplate(input_variables=["context", "query"], template=template) # Set up LLM for the QA chain llm = HuggingFacePipeline(pipeline=query_pipeline) # Create the RetrievalQA chain qa_chain = RetrievalQA.from_chain_type( llm, retriever=retriever, chain_type_kwargs={"prompt": PROMPT} ) # Function to get the answer from the QA chain def get_answer(query): result = qa_chain.invoke({"query": query}) answer = result['result'].split("Answer:")[1].strip() return answer def chat_function(user_input, history): if history is None: history = [] query = user_input response = get_answer(query) history.append((user_input, response)) return history, history iface = gr.Interface( fn=chat_function, inputs=[ gr.Textbox(placeholder="Ask me anything about AI...", label="Enter your question here"), gr.State() ], outputs=[ gr.Chatbot(height=300), gr.State() ], title="Your AI Chat" ) # Launch the Gradio interface if __name__ == "__main__": iface.launch(share=True,debug=True)