File size: 925 Bytes
809c683 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 | import os
import uvicorn
from fastapi import FastAPI, Request
from ctransformers import AutoModelForCausalLM
# Set the cache directory environment variable
os.environ["HF_HOME"] = "/app/.cache"
app = FastAPI()
# Set gpu_layers to the number of layers to offload to GPU. Set to 0 if no GPU acceleration is available on your system.
llm = AutoModelForCausalLM.from_pretrained("BekiTila/Amharic-llama-for-agriculture-chat-bot-v0.1.gguf", model_file="Amharic-llama-for-agriculture-chat-bot-v0.1.gguf", model_type="llama", gpu_layers=0)
# Create the text generation pipeline
@app.post("/generate")
async def generate_response(request: Request):
data = await request.json()
prompt = data['input']
instruction = f"### Instruction:\n{prompt}\n\n### Response:\n"
result = llm(instruction)
return {"response": result}
if __name__ == "__main__":
uvicorn.run(app, host="0.0.0.0", port=8000)
|