Spaces:
Runtime error
Runtime error
| import gradio as gr | |
| import copy | |
| import time | |
| import ctypes #to run on C api directly | |
| import llama_cpp | |
| from llama_cpp import Llama | |
| from huggingface_hub import hf_hub_download #load from huggingfaces | |
| llm = Llama(model_path= hf_hub_download(repo_id="TheBloke/orca_mini_3B-GGML", filename="orca-mini-3b.ggmlv3.q4_1.bin"), n_ctx=2048) #download model from hf/ n_ctx=2048 for high ccontext length | |
| history = [] | |
| pre_prompt = " The user and the AI are having a conversation : " | |
| def generate_text(input_text, history): | |
| print("history ",history) | |
| print("input ", input_text) | |
| if history == []: | |
| input_text_with_history = f"{pre_prompt}"+ "\n" + f"Q: {input_text} " + "\n" +" A:" | |
| else: | |
| input_text_with_history = f"{history[-1][1]}"+ "\n" | |
| input_text_with_history += f"Q: {input_text}" + "\n" +" A:" | |
| print("new input", input_text_with_history) | |
| output = llm(input_text_with_history, max_tokens=1024, stop=["Q:", "\n"], stream=True) | |
| for out in output: | |
| stream = copy.deepcopy(out) | |
| print(stream["choices"][0]["text"]) | |
| input_text_with_history += stream["choices"][0]["text"] | |
| yield stream["choices"][0]["text"] | |
| history =["init",input_text_with_history] | |
| demo = gr.ChatInterface(generate_text) | |
| demo.queue(concurrency_count=1, max_size=5) | |
| demo.launch() | |