Spaces:
Runtime error
Runtime error
| import gradio as gr | |
| import time | |
| import ctypes #to run on C api directly | |
| import llama_cpp | |
| from llama_cpp import Llama | |
| from huggingface_hub import hf_hub_download #load from huggingfaces | |
| llm = Llama(model_path= hf_hub_download(repo_id="TheBloke/airoboros-l2-13b-gpt4-m2.0-GGML", filename="airoboros-l2-13b-gpt4-m2.0.ggmlv3.q6_K.bin"), n_ctx=2048) #download model from hf/ n_ctx=2048 for high ccontext length | |
| history = [] | |
| def generate_text(input_text, history): | |
| print("history ",history) | |
| print("input ", input) | |
| print("last history: ", history[-1]) | |
| full_conversation = f"{history} Q: {input_text} \n A:" | |
| print("full convo", full_conversation) | |
| temp_conversation = f"{history[-1]} Q: {input_text} \n A:" | |
| print("temp convo", temp_conversation) | |
| output = llm(full_conversation, max_tokens=1024, stop=["Q:", "\n"], echo=True) | |
| response = output['choices'][0]['text'] | |
| return response | |
| demo = gr.ChatInterface(generate_text) | |
| demo.queue(concurrency_count=1, max_size=5) | |
| demo.launch() | |