File size: 2,309 Bytes
3514447
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
from __future__ import annotations
import gradio as gr
import subprocess
from typing import Iterable

#### I think everything you need if you are not using gradio start here and ends below at the next ####
# You may also need the very first line of this file, but it gets mad if you move that line anywhere beside the very top of the file.

from huggingface_hub import hf_hub_download
from llama_cpp import Llama
from llama_cpp import LlamaRAMCache

hf_hub_download(repo_id="TheBloke/Llama-2-7B-chat-GGML", filename="llama-2-7b-chat.ggmlv3.q4_K_M.bin", local_dir=".")

llm = Llama(model_path="./llama-2-7b-chat.ggmlv3.q4_K_M.bin", rms_norm_eps=1e-5)

#cache = LlamaRAMCache(capacity_bytes=2 << 30)
#llm.set_cache(cache)

ins = '''<s>[INST] <<SYS>>
You are a helpful, respectful and honest assistant. Always answer as helpfully as possible, while being safe.  Your answers should not include any harmful, unethical, racist, sexist, toxic, dangerous, or illegal content. Please ensure that your responses are socially unbiased and positive in nature.
If a question does not make any sense, or is not factually coherent, explain why instead of answering something not correct. If you don't know the answer to a question, please don't share false information.
<</SYS>>
{} [/INST]
'''

def generate(instruction): 
    result = ""
    ans = llm(ins.format(instruction), stop=['USER:'], stream=False, max_tokens=512)
    print(ans['choices'][0]['text'])
    return str(ans['choices'][0]['text'])

# test print with hardcoded prompt
print(generate("What year was Microsoft founded?"))
#### The stuff below this is not necessary if you have some other interface or just want to see if this prints out an answer. ####

with gr.Blocks() as demo:
    with gr.Column():
        with gr.Row():
            with gr.Column(scale=3):
                instruction = gr.Textbox(placeholder="Enter your question here", label="Question", elem_id="q-input")

                with gr.Box():
                    gr.Markdown("Answer")
                    output = gr.Markdown(elem_id="q-output")
                submit = gr.Button("Generate")

    submit.click(generate, inputs=[instruction], outputs=output)
    #instruction.submit(generate, inputs=[instruction], outputs=[output])

demo.queue(concurrency_count=1).launch(debug=False)