Download app.py from majevince/LLMM_Test: direct link, hf CLI and curl.
- Browser
- Download file 2.31 kB
-
https://huggingface.co/spaces/majevince/LLMM_Test/resolve/main/app.py
- Command line
-
hf download hf://spaces/majevince/LLMM_Test/app.py
-
curl -L -o app.py https://huggingface.co/spaces/majevince/LLMM_Test/resolve/main/app.py
2.31 kB
| from __future__ import annotations | |
| import gradio as gr | |
| import subprocess | |
| from typing import Iterable | |
| #### I think everything you need if you are not using gradio start here and ends below at the next #### | |
| # You may also need the very first line of this file, but it gets mad if you move that line anywhere beside the very top of the file. | |
| from huggingface_hub import hf_hub_download | |
| from llama_cpp import Llama | |
| from llama_cpp import LlamaRAMCache | |
| hf_hub_download(repo_id="TheBloke/Llama-2-7B-chat-GGML", filename="llama-2-7b-chat.ggmlv3.q4_K_M.bin", local_dir=".") | |
| llm = Llama(model_path="./llama-2-7b-chat.ggmlv3.q4_K_M.bin", rms_norm_eps=1e-5) | |
| #cache = LlamaRAMCache(capacity_bytes=2 << 30) | |
| #llm.set_cache(cache) | |
| ins = '''<s>[INST] <<SYS>> | |
| You are a helpful, respectful and honest assistant. Always answer as helpfully as possible, while being safe. Your answers should not include any harmful, unethical, racist, sexist, toxic, dangerous, or illegal content. Please ensure that your responses are socially unbiased and positive in nature. | |
| If a question does not make any sense, or is not factually coherent, explain why instead of answering something not correct. If you don't know the answer to a question, please don't share false information. | |
| <</SYS>> | |
| {} [/INST] | |
| ''' | |
| def generate(instruction): | |
| result = "" | |
| ans = llm(ins.format(instruction), stop=['USER:'], stream=False, max_tokens=512) | |
| print(ans['choices'][0]['text']) | |
| return str(ans['choices'][0]['text']) | |
| # test print with hardcoded prompt | |
| print(generate("What year was Microsoft founded?")) | |
| #### The stuff below this is not necessary if you have some other interface or just want to see if this prints out an answer. #### | |
| with gr.Blocks() as demo: | |
| with gr.Column(): | |
| with gr.Row(): | |
| with gr.Column(scale=3): | |
| instruction = gr.Textbox(placeholder="Enter your question here", label="Question", elem_id="q-input") | |
| with gr.Box(): | |
| gr.Markdown("Answer") | |
| output = gr.Markdown(elem_id="q-output") | |
| submit = gr.Button("Generate") | |
| submit.click(generate, inputs=[instruction], outputs=output) | |
| #instruction.submit(generate, inputs=[instruction], outputs=[output]) | |
| demo.queue(concurrency_count=1).launch(debug=False) |