Temporary / app.py
BekiTila's picture
Create app.py
809c683 verified
Raw History Blame Contribute Delete
925 Bytes
import os
import uvicorn
from fastapi import FastAPI, Request
from ctransformers import AutoModelForCausalLM
# Set the cache directory environment variable
os.environ["HF_HOME"] = "/app/.cache"
app = FastAPI()
# Set gpu_layers to the number of layers to offload to GPU. Set to 0 if no GPU acceleration is available on your system.
llm = AutoModelForCausalLM.from_pretrained("BekiTila/Amharic-llama-for-agriculture-chat-bot-v0.1.gguf", model_file="Amharic-llama-for-agriculture-chat-bot-v0.1.gguf", model_type="llama", gpu_layers=0)
# Create the text generation pipeline
@app.post("/generate")
async def generate_response(request: Request):
data = await request.json()
prompt = data['input']
instruction = f"### Instruction:\n{prompt}\n\n### Response:\n"
result = llm(instruction)
return {"response": result}
if __name__ == "__main__":
uvicorn.run(app, host="0.0.0.0", port=8000)