Spaces:
Runtime error
Runtime error
Download app.py from risper7/YOUR_AI_CHAT: direct link, hf CLI and curl.
- Browser
- Download file 3.79 kB
-
https://huggingface.co/spaces/risper7/YOUR_AI_CHAT/resolve/main/app.py
- Command line
-
hf download hf://spaces/risper7/YOUR_AI_CHAT/app.py
-
curl -L -o app.py https://huggingface.co/spaces/risper7/YOUR_AI_CHAT/resolve/main/app.py
3.79 kB
| import os | |
| from torch import cuda, bfloat16 | |
| import torch | |
| import transformers | |
| from transformers import AutoTokenizer, AutoModelForCausalLM | |
| from langchain_huggingface import HuggingFaceEmbeddings, HuggingFacePipeline | |
| from langchain.chains import RetrievalQA | |
| from langchain.prompts import PromptTemplate | |
| from langchain_chroma import Chroma | |
| from huggingface_hub import login | |
| from accelerate import disk_offload | |
| import chromadb | |
| import gradio as gr | |
| from transformers import pipeline | |
| # Login to Hugging Face Hub | |
| hf_token = os.getenv("HF_TOKEN") | |
| login(token=hf_token) | |
| # Model configuration and pipeline setup | |
| model_id='google/gemma-1.1-2b-it' | |
| device = f'cuda:{cuda.current_device()}' if cuda.is_available() else 'cpu' | |
| bnb_config = transformers.BitsAndBytesConfig( | |
| load_in_4bit=True, | |
| bnb_4bit_quant_type='nf4', | |
| bnb_4bit_use_double_quant=True, | |
| bnb_4bit_compute_dtype=bfloat16 | |
| ) | |
| model_config = transformers.AutoConfig.from_pretrained( | |
| model_id, | |
| trust_remote_code=True, | |
| max_new_tokens=1024, | |
| ) | |
| model = AutoModelForCausalLM.from_pretrained( | |
| model_id, | |
| trust_remote_code=True, | |
| config=model_config, | |
| #quantization_config=bnb_config, | |
| device_map='auto', | |
| token=hf_token | |
| ) | |
| #model.to('cpu') | |
| offload_dir = './disk_offload' | |
| model = disk_offload(model, offload_dir) | |
| tokenizer = AutoTokenizer.from_pretrained(model_id, token=hf_token) | |
| query_pipeline = pipeline( | |
| "text-generation", | |
| model=model, | |
| tokenizer=tokenizer, | |
| torch_dtype=torch.float16 if device == 'cuda' else torch.float32, | |
| device_map='auto', | |
| max_length=6000, | |
| max_new_tokens=500, | |
| do_sample=True, | |
| temperature = 0.7, | |
| top_k=50, | |
| top_p=0.85, | |
| repetition_penalty=1.2 | |
| ) | |
| # Define the embedding function using Hugging Face embeddings | |
| embedding_function = HuggingFaceEmbeddings(model_name="sentence-transformers/all-MiniLM-L6-v2") | |
| # Set up the vector store with ChromaDB | |
| client = chromadb.PersistentClient(path="risper7/YOUR_AI_CHAT/vectordata1") | |
| vector_store = Chroma( | |
| client=client, | |
| collection_name="AI_data", | |
| embedding_function=embedding_function | |
| ) | |
| # Set up retriever from the vector store | |
| retriever = vector_store.as_retriever( | |
| search_type="mmr", | |
| search_kwargs={"k": 3, "fetch_k": 10} | |
| ) | |
| # Define prompt template | |
| template = """ | |
| You are an AI and Machine Learning expert. Analyze the given question and respond with a concise, accurate answer based strictly on the provided context. | |
| If the context does not contain enough information, answer to the best of your knowledge while maintaining clarity and correctness. | |
| Don't add any additional details that are not related to the question and context. | |
| Question: | |
| {question} | |
| Context: | |
| {context} | |
| Answer: | |
| """ | |
| PROMPT = PromptTemplate(input_variables=["context", "query"], template=template) | |
| # Set up LLM for the QA chain | |
| llm = HuggingFacePipeline(pipeline=query_pipeline) | |
| # Create the RetrievalQA chain | |
| qa_chain = RetrievalQA.from_chain_type( | |
| llm, retriever=retriever, chain_type_kwargs={"prompt": PROMPT} | |
| ) | |
| # Function to get the answer from the QA chain | |
| def get_answer(query): | |
| result = qa_chain.invoke({"query": query}) | |
| answer = result['result'].split("Answer:")[1].strip() | |
| return answer | |
| def chat_function(user_input, history): | |
| if history is None: | |
| history = [] | |
| query = user_input | |
| response = get_answer(query) | |
| history.append((user_input, response)) | |
| return history, history | |
| iface = gr.Interface( | |
| fn=chat_function, | |
| inputs=[ | |
| gr.Textbox(placeholder="Ask me anything about AI...", label="Enter your question here"), | |
| gr.State() | |
| ], | |
| outputs=[ | |
| gr.Chatbot(height=300), | |
| gr.State() | |
| ], | |
| title="Your AI Chat" | |
| ) | |
| # Launch the Gradio interface | |
| if __name__ == "__main__": | |
| iface.launch(share=True,debug=True) | |