import os import uvicorn from fastapi import FastAPI, Request from ctransformers import AutoModelForCausalLM # Set the cache directory environment variable os.environ["HF_HOME"] = "/app/.cache" app = FastAPI() # Set gpu_layers to the number of layers to offload to GPU. Set to 0 if no GPU acceleration is available on your system. llm = AutoModelForCausalLM.from_pretrained("BekiTila/Amharic-llama-for-agriculture-chat-bot-v0.1.gguf", model_file="Amharic-llama-for-agriculture-chat-bot-v0.1.gguf", model_type="llama", gpu_layers=0) # Create the text generation pipeline @app.post("/generate") async def generate_response(request: Request): data = await request.json() prompt = data['input'] instruction = f"### Instruction:\n{prompt}\n\n### Response:\n" result = llm(instruction) return {"response": result} if __name__ == "__main__": uvicorn.run(app, host="0.0.0.0", port=8000)