File size: 697 Bytes
5d62acd | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 | import sys
from llama_cpp import Llama
if len(sys.argv) < 2:
print("Model path not provided as argument")
print("Eg. Usage: $ python chatapp.py path/to/model.gguf")
sys.exit(1)
llm = Llama(
model_path=sys.argv[1],
n_ctx=512,
n_threads=4,
n_gpu_layers=1,
verbose=False
)
print("Chat with Llama (type 'exit' to quit)\n")
while True:
user_input = input("You: ")
if user_input.lower() in ["exit", "quit"]: break
prompt = f"### Human: {user_input}\n### Assistant:"
output = llm(
prompt,
max_tokens=100,
stop=["###", "### Human:", "\n###"]
)
response = output["choices"][0]["text"].strip()
print("Bot:", response)
|