File size: 925 Bytes
809c683
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
import os
import uvicorn
from fastapi import FastAPI, Request
from ctransformers import AutoModelForCausalLM




# Set the cache directory environment variable
os.environ["HF_HOME"] = "/app/.cache"

app = FastAPI()


# Set gpu_layers to the number of layers to offload to GPU. Set to 0 if no GPU acceleration is available on your system.
llm = AutoModelForCausalLM.from_pretrained("BekiTila/Amharic-llama-for-agriculture-chat-bot-v0.1.gguf", model_file="Amharic-llama-for-agriculture-chat-bot-v0.1.gguf", model_type="llama", gpu_layers=0)



# Create the text generation pipeline

@app.post("/generate")
async def generate_response(request: Request):
    data = await request.json()
    prompt = data['input']
    instruction = f"### Instruction:\n{prompt}\n\n### Response:\n"
    result = llm(instruction)
    
    
    return {"response": result}

if __name__ == "__main__":
    uvicorn.run(app, host="0.0.0.0", port=8000)